using a json library makes my script lag for 10 minutes

oh okay · July 30, 2019

this is what I am doing in my script:

Receive array of items over the internet (filesize: 500kb) (this goes fine)

then loop through these elements in order to sort them (this takes an insanely long time)

here is the code of my scrape class

package RietFlipper;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.stream.Collectors;

import org.dreambot.api.methods.MethodProvider;
import org.json.JSONObject;


public class Scraper {
	private static URLConnection con;
	private static InputStream is;
	private static InputStreamReader isr;
	private static BufferedReader br;


	public static ArrayList<String> getData(int minimumPrice, int maximumPrice, boolean ismember) {
		try {
			URL url = new URL(
					"https://storage.googleapis.com/osbuddy-exchange/summary.json");
			con = url.openConnection();
			is = con.getInputStream();
			isr = new InputStreamReader(is);
			br = new BufferedReader(isr);
			ArrayList <String> itemName = new ArrayList<String>();
			ArrayList <Integer> itemPrices = new ArrayList<Integer>();
			String a  = br.lines().collect(Collectors.joining());

			JSONObject obj = new JSONObject(a);
			obj.keys().forEachRemaining(key -> {
		        Object value = obj.get(key);
		       // obj.getJSONObject(key).getBoolean("members");
		        if(minimumPrice == 0 && maximumPrice == 0) {
		        	itemName.add(obj.getJSONObject(key).getString("name"));
		        }else {
		        if(obj.getJSONObject(key).getBoolean("members") == ismember && obj.getJSONObject(key).getInt("overall_average") > minimumPrice && obj.getJSONObject(key).getInt("overall_average") < maximumPrice) {
		        	MethodProvider.log("Key: " + key  + " - Value: " + value);
		        itemName.add(obj.getJSONObject(key).getString("name"));
		        itemPrices.add(obj.getJSONObject(key).getInt("overall_average"));
		        }
		        }
		        
		        
		    });
			
			
			return itemName;

		} catch (Exception e) {
			e.printStackTrace();
			MethodProvider.log(e.toString());
		} finally {
			try {
				if (br != null) {
					br.close();
				} else if (isr != null) {
					isr.close();
				} else if (is != null) {
					is.close();
				}
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		return null;
	}
}

Articron · July 30, 2019

Here's my OSBuddy stuff over HTTP. Try using this, and if that still doesn't work, post back.

Example use:

OSBuddy.init(); //call in onstart
Optional<OSBuddyItem> item = OSBuddy.getItem(p -> p.getId() == 4151);
if (item.isPresent()) {
   System.out.println(item.get().getName());
   System.out.println(item.get().getSellAverage());
   //etc
}

package dreambot.org.articron.networking.http.osrs.osbuddy;

public class OSBuddyItem {

    private String name;
    private final int id, buyAverage, buyQuantity, sellAverage,sellQuantity,overallAverage,overallQuantity;

    public OSBuddyItem(int id, String name, int buyAverage, int buyQuantity, int sellAverage, int sellQuantity, int overallAverage, int overallQuantity) {
        this.name = name;
        this.id = id;
        this.buyAverage = buyAverage;
        this.buyQuantity = buyQuantity;
        this.sellAverage = sellAverage;
        this.sellQuantity = sellQuantity;
        this.overallAverage = overallAverage;
        this.overallQuantity = overallQuantity;
    }


    public int getBuyAverage() {
        return buyAverage;
    }

    public int getBuyQuantity() {
        return buyQuantity;
    }

    public int getSellAverage() {
        return sellAverage;
    }

    public String getName() {
        return name;
    }

    public int getId() {
        return id;
    }

    public int getSellQuantity() {
        return sellQuantity;
    }

    public int getOverallAverage() {
        return overallAverage;
    }

    public int getOverallQuantity() {
        return overallQuantity;
    }
}

package dreambot.org.articron.networking.http.osrs.osbuddy;

import dreambot.org.articron.json.JsonObject;
import dreambot.org.articron.networking.http.HTTPRequest;
import dreambot.org.articron.networking.http.HTTPResponse;
import dreambot.org.script.world.looting.LootItem;

import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.function.Predicate;

public class OSBuddy {

    private static final String TRADEABLE_ITEMS = "https://storage.googleapis.com/osbuddy-exchange/summary.json";

    private static List<OSBuddyItem> OSBuddyItemList;

    public static void init() {
        OSBuddyItemList = new LinkedList<>();
        HTTPResponse itemRequest = new HTTPRequest(TRADEABLE_ITEMS).read();
        JsonObject items = itemRequest.getJsonResponse().asObject();
        for (String id : items.names()) {
            JsonObject item = items.get(id).asObject();
            int itemId          = item.get("id").asInt();
            String itemName     = item.get("name").asString();
            int buyAverage      = item.get("buy_average").asInt();
            int buyQuantity     = item.get("buy_quantity").asInt();
            int sellAverage     = item.get("sell_average").asInt();
            int sellQuantity    = item.get("sell_quantity").asInt();
            int overallAverage  = item.get("overall_average").asInt();
            int overallQuantity = item.get("overall_quantity").asInt();
            OSBuddyItemList.add(new OSBuddyItem(itemId,itemName,buyAverage,buyQuantity,sellAverage,sellQuantity,overallAverage,overallQuantity));

        }
    }


    public static int getItemId(String itemName) {
        return getItem(p -> p.getName().equals(itemName)).map(OSBuddyItem::getId).orElse(-1);
    }

    public static String getItemName(int itemId) {
        return getItem(p -> p.getId() == itemId).map(OSBuddyItem::getName).orElse(null);
    }

    public static Optional<OSBuddyItem> getItem(Predicate<OSBuddyItem> predicate) {
        return OSBuddyItemList.stream().filter(predicate).findFirst();
    }


}

package dreambot.org.articron.networking.http;

import java.io.IOException;
import java.io.PrintStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

public class HTTPRequest {

    private final String url, method;

    private Map<String,String> parameterMap = new HashMap<>();

    public HTTPRequest(String url, String method) {
        this.url = url;
        this.method = method;
    }

    public HTTPRequest(String url) {
        this(url,"GET");
    }

    public HTTPRequest put(String k, String v) {
        parameterMap.put(k,v);
        return this;
    }

    public HTTPResponse read() {
        String baseUrl = url;
        if (method.equals("GET") || method.equals("get")) {
            int i = 0;
            for(Map.Entry<String,String> entry : parameterMap.entrySet()) {
                baseUrl = baseUrl.concat((i==0 ? "?" : "&") + entry.getKey() + "=" + entry.getValue());
                i++;
            }
        }
        try {
            System.out.println("URL: " + baseUrl);
            URL url = new URL(baseUrl);
            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
            connection.addRequestProperty("User-Agent",
                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36");
            if (method.equals("POST") || method.equals("post")) {
                String contentString = "";
                for (Map.Entry<String,String> entry : parameterMap.entrySet()) {
                    contentString = contentString.concat("&"+entry.getKey()+"="+entry.getValue());
                }
                connection.setDoOutput(true);
                PrintStream out = new PrintStream(connection.getOutputStream());
                out.print(contentString.substring(1));
            }
            return new HTTPResponse(connection).read();
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }

}

package dreambot.org.articron.networking.http;

import dreambot.org.articron.json.Json;
import dreambot.org.articron.json.JsonObject;
import dreambot.org.articron.json.JsonValue;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.util.LinkedList;
import java.util.List;

public class HTTPResponse {

    private HttpURLConnection connection;
    private int responseCode;
    private List<String> lineList;

    public HTTPResponse(HttpURLConnection connection) throws IOException {
        this.connection = connection;
        this.responseCode = connection.getResponseCode();
        this.lineList = new LinkedList<>();
    }

    public HTTPResponse read() throws IOException {
        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
        String line;
        while ((line = in.readLine()) != null) {
            lineList.add(line);
        }
        return this;
    }

    public List<String> getResponse() {
        return lineList;
    }

    public JsonValue getJsonResponse() {
        String raw = getRawResponse();
        return Json.parse(raw);
    }

    public int getIndexContaining(String string) {
        for(int i = 0; i < lineList.size();i++) {
            String line = lineList.get(i);
            if (line.contains(string))
                return i;
        }
        return -1;
    }

    public String getRawResponse() {
        return String.join("",lineList);
    }

    public int getResponseCode() {
        return responseCode;
    }
}

Leon · July 30, 2019

8 minutes ago, Articron said:

Here's my OSBuddy stuff over HTTP. Try using this, and if that still doesn't work, post back.

Example use:


OSBuddy.init(); //call in onstart
OSBuddyItem item = OSBuddy.getItem(item -> item.getID() == 4151);
System.out.println(item.getName());
System.out.println(item.getSellAverage());
//etc...


package dreambot.org.articron.networking.http.osrs.osbuddy;

public class OSBuddyItem {

    private String name;
    private final int id, buyAverage, buyQuantity, sellAverage,sellQuantity,overallAverage,overallQuantity;

    public OSBuddyItem(int id, String name, int buyAverage, int buyQuantity, int sellAverage, int sellQuantity, int overallAverage, int overallQuantity) {
        this.name = name;
        this.id = id;
        this.buyAverage = buyAverage;
        this.buyQuantity = buyQuantity;
        this.sellAverage = sellAverage;
        this.sellQuantity = sellQuantity;
        this.overallAverage = overallAverage;
        this.overallQuantity = overallQuantity;
    }


    public int getBuyAverage() {
        return buyAverage;
    }

    public int getBuyQuantity() {
        return buyQuantity;
    }

    public int getSellAverage() {
        return sellAverage;
    }

    public String getName() {
        return name;
    }

    public int getId() {
        return id;
    }

    public int getSellQuantity() {
        return sellQuantity;
    }

    public int getOverallAverage() {
        return overallAverage;
    }

    public int getOverallQuantity() {
        return overallQuantity;
    }
}


package dreambot.org.articron.networking.http.osrs.osbuddy;

import dreambot.org.articron.json.JsonObject;
import dreambot.org.articron.networking.http.HTTPRequest;
import dreambot.org.articron.networking.http.HTTPResponse;
import dreambot.org.script.world.looting.LootItem;

import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.function.Predicate;

public class OSBuddy {

    private static final String TRADEABLE_ITEMS = "https://storage.googleapis.com/osbuddy-exchange/summary.json";

    private static List<OSBuddyItem> OSBuddyItemList;

    public static void init() {
        OSBuddyItemList = new LinkedList<>();
        HTTPResponse itemRequest = new HTTPRequest(TRADEABLE_ITEMS).read();
        JsonObject items = itemRequest.getJsonResponse().asObject();
        for (String id : items.names()) {
            JsonObject item = items.get(id).asObject();
            int itemId          = item.get("id").asInt();
            String itemName     = item.get("name").asString();
            int buyAverage      = item.get("buy_average").asInt();
            int buyQuantity     = item.get("buy_quantity").asInt();
            int sellAverage     = item.get("sell_average").asInt();
            int sellQuantity    = item.get("sell_quantity").asInt();
            int overallAverage  = item.get("overall_average").asInt();
            int overallQuantity = item.get("overall_quantity").asInt();
            OSBuddyItemList.add(new OSBuddyItem(itemId,itemName,buyAverage,buyQuantity,sellAverage,sellQuantity,overallAverage,overallQuantity));

        }
    }


    public static int getItemId(String itemName) {
        return getItem(p -> p.getName().equals(itemName)).map(OSBuddyItem::getId).orElse(-1);
    }

    public static String getItemName(int itemId) {
        return getItem(p -> p.getId() == itemId).map(OSBuddyItem::getName).orElse(null);
    }

    public static Optional<OSBuddyItem> getItem(Predicate<OSBuddyItem> predicate) {
        return OSBuddyItemList.stream().filter(predicate).findFirst();
    }


}


package dreambot.org.articron.networking.http;

import java.io.IOException;
import java.io.PrintStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

public class HTTPRequest {

    private final String url, method;

    private Map<String,String> parameterMap = new HashMap<>();

    public HTTPRequest(String url, String method) {
        this.url = url;
        this.method = method;
    }

    public HTTPRequest(String url) {
        this(url,"GET");
    }

    public HTTPRequest put(String k, String v) {
        parameterMap.put(k,v);
        return this;
    }

    public HTTPResponse read() {
        String baseUrl = url;
        if (method.equals("GET") || method.equals("get")) {
            int i = 0;
            for(Map.Entry<String,String> entry : parameterMap.entrySet()) {
                baseUrl = baseUrl.concat((i==0 ? "?" : "&") + entry.getKey() + "=" + entry.getValue());
                i++;
            }
        }
        try {
            System.out.println("URL: " + baseUrl);
            URL url = new URL(baseUrl);
            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
            connection.addRequestProperty("User-Agent",
                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36");
            if (method.equals("POST") || method.equals("post")) {
                String contentString = "";
                for (Map.Entry<String,String> entry : parameterMap.entrySet()) {
                    contentString = contentString.concat("&"+entry.getKey()+"="+entry.getValue());
                }
                connection.setDoOutput(true);
                PrintStream out = new PrintStream(connection.getOutputStream());
                out.print(contentString.substring(1));
            }
            return new HTTPResponse(connection).read();
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }

}


package dreambot.org.articron.networking.http;

import dreambot.org.articron.json.Json;
import dreambot.org.articron.json.JsonObject;
import dreambot.org.articron.json.JsonValue;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.util.LinkedList;
import java.util.List;

public class HTTPResponse {

    private HttpURLConnection connection;
    private int responseCode;
    private List<String> lineList;

    public HTTPResponse(HttpURLConnection connection) throws IOException {
        this.connection = connection;
        this.responseCode = connection.getResponseCode();
        this.lineList = new LinkedList<>();
    }

    public HTTPResponse read() throws IOException {
        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
        String line;
        while ((line = in.readLine()) != null) {
            lineList.add(line);
        }
        return this;
    }

    public List<String> getResponse() {
        return lineList;
    }

    public JsonValue getJsonResponse() {
        String raw = getRawResponse();
        return Json.parse(raw);
    }

    public int getIndexContaining(String string) {
        for(int i = 0; i < lineList.size();i++) {
            String line = lineList.get(i);
            if (line.contains(string))
                return i;
        }
        return -1;
    }

    public String getRawResponse() {
        return String.join("",lineList);
    }

    public int getResponseCode() {
        return responseCode;
    }
}

I heard your bad

NovaGTX · July 30, 2019

4 hours ago, Leonn said:

I heard your bad

You're*

beezdul · July 31, 2019

10 hours ago, NovaGTX said:

You're*

Your are* 🤠

oh okay · July 31, 2019

23 hours ago, Articron said:

Here's my OSBuddy stuff over HTTP. Try using this, and if that still doesn't work, post back.

Example use:


OSBuddy.init(); //call in onstart
Optional<OSBuddyItem> item = OSBuddy.getItem(p -> p.getId() == 4151);
if (item.isPresent()) {
   System.out.println(item.get().getName());
   System.out.println(item.get().getSellAverage());
   //etc
}


package dreambot.org.articron.networking.http.osrs.osbuddy;

public class OSBuddyItem {

    private String name;
    private final int id, buyAverage, buyQuantity, sellAverage,sellQuantity,overallAverage,overallQuantity;

    public OSBuddyItem(int id, String name, int buyAverage, int buyQuantity, int sellAverage, int sellQuantity, int overallAverage, int overallQuantity) {
        this.name = name;
        this.id = id;
        this.buyAverage = buyAverage;
        this.buyQuantity = buyQuantity;
        this.sellAverage = sellAverage;
        this.sellQuantity = sellQuantity;
        this.overallAverage = overallAverage;
        this.overallQuantity = overallQuantity;
    }


    public int getBuyAverage() {
        return buyAverage;
    }

    public int getBuyQuantity() {
        return buyQuantity;
    }

    public int getSellAverage() {
        return sellAverage;
    }

    public String getName() {
        return name;
    }

    public int getId() {
        return id;
    }

    public int getSellQuantity() {
        return sellQuantity;
    }

    public int getOverallAverage() {
        return overallAverage;
    }

    public int getOverallQuantity() {
        return overallQuantity;
    }
}


package dreambot.org.articron.networking.http.osrs.osbuddy;

import dreambot.org.articron.json.JsonObject;
import dreambot.org.articron.networking.http.HTTPRequest;
import dreambot.org.articron.networking.http.HTTPResponse;
import dreambot.org.script.world.looting.LootItem;

import java.util.LinkedList;
import java.util.List;
import java.util.Optional;
import java.util.function.Predicate;

public class OSBuddy {

    private static final String TRADEABLE_ITEMS = "https://storage.googleapis.com/osbuddy-exchange/summary.json";

    private static List<OSBuddyItem> OSBuddyItemList;

    public static void init() {
        OSBuddyItemList = new LinkedList<>();
        HTTPResponse itemRequest = new HTTPRequest(TRADEABLE_ITEMS).read();
        JsonObject items = itemRequest.getJsonResponse().asObject();
        for (String id : items.names()) {
            JsonObject item = items.get(id).asObject();
            int itemId          = item.get("id").asInt();
            String itemName     = item.get("name").asString();
            int buyAverage      = item.get("buy_average").asInt();
            int buyQuantity     = item.get("buy_quantity").asInt();
            int sellAverage     = item.get("sell_average").asInt();
            int sellQuantity    = item.get("sell_quantity").asInt();
            int overallAverage  = item.get("overall_average").asInt();
            int overallQuantity = item.get("overall_quantity").asInt();
            OSBuddyItemList.add(new OSBuddyItem(itemId,itemName,buyAverage,buyQuantity,sellAverage,sellQuantity,overallAverage,overallQuantity));

        }
    }


    public static int getItemId(String itemName) {
        return getItem(p -> p.getName().equals(itemName)).map(OSBuddyItem::getId).orElse(-1);
    }

    public static String getItemName(int itemId) {
        return getItem(p -> p.getId() == itemId).map(OSBuddyItem::getName).orElse(null);
    }

    public static Optional<OSBuddyItem> getItem(Predicate<OSBuddyItem> predicate) {
        return OSBuddyItemList.stream().filter(predicate).findFirst();
    }


}


package dreambot.org.articron.networking.http;

import java.io.IOException;
import java.io.PrintStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;

public class HTTPRequest {

    private final String url, method;

    private Map<String,String> parameterMap = new HashMap<>();

    public HTTPRequest(String url, String method) {
        this.url = url;
        this.method = method;
    }

    public HTTPRequest(String url) {
        this(url,"GET");
    }

    public HTTPRequest put(String k, String v) {
        parameterMap.put(k,v);
        return this;
    }

    public HTTPResponse read() {
        String baseUrl = url;
        if (method.equals("GET") || method.equals("get")) {
            int i = 0;
            for(Map.Entry<String,String> entry : parameterMap.entrySet()) {
                baseUrl = baseUrl.concat((i==0 ? "?" : "&") + entry.getKey() + "=" + entry.getValue());
                i++;
            }
        }
        try {
            System.out.println("URL: " + baseUrl);
            URL url = new URL(baseUrl);
            HttpURLConnection connection = (HttpURLConnection) url.openConnection();
            connection.addRequestProperty("User-Agent",
                    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36");
            if (method.equals("POST") || method.equals("post")) {
                String contentString = "";
                for (Map.Entry<String,String> entry : parameterMap.entrySet()) {
                    contentString = contentString.concat("&"+entry.getKey()+"="+entry.getValue());
                }
                connection.setDoOutput(true);
                PrintStream out = new PrintStream(connection.getOutputStream());
                out.print(contentString.substring(1));
            }
            return new HTTPResponse(connection).read();
        } catch (IOException e) {
            e.printStackTrace();
            return null;
        }
    }

}


package dreambot.org.articron.networking.http;

import dreambot.org.articron.json.Json;
import dreambot.org.articron.json.JsonObject;
import dreambot.org.articron.json.JsonValue;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.util.LinkedList;
import java.util.List;

public class HTTPResponse {

    private HttpURLConnection connection;
    private int responseCode;
    private List<String> lineList;

    public HTTPResponse(HttpURLConnection connection) throws IOException {
        this.connection = connection;
        this.responseCode = connection.getResponseCode();
        this.lineList = new LinkedList<>();
    }

    public HTTPResponse read() throws IOException {
        BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
        String line;
        while ((line = in.readLine()) != null) {
            lineList.add(line);
        }
        return this;
    }

    public List<String> getResponse() {
        return lineList;
    }

    public JsonValue getJsonResponse() {
        String raw = getRawResponse();
        return Json.parse(raw);
    }

    public int getIndexContaining(String string) {
        for(int i = 0; i < lineList.size();i++) {
            String line = lineList.get(i);
            if (line.contains(string))
                return i;
        }
        return -1;
    }

    public String getRawResponse() {
        return String.join("",lineList);
    }

    public int getResponseCode() {
        return responseCode;
    }
}

what JSON library are you using?

Articron · July 31, 2019

1 hour ago, oh okay said:

what JSON library are you using?

It's called minimal-json.

Leon · July 31, 2019

1 hour ago, Articron said:

It's called minimal-json.

Nerd

oh okay · August 4, 2019

On 7/31/2019 at 8:20 PM, Articron said:

It's called minimal-json.

thanks man it works -Trester

now I can finish my entrana walker on topbot.org

Sign In

using a json library makes my script lag for 10 minutes

Recommended Posts

oh okay 6

Articron 746

Leon 50

NovaGTX 106

beezdul 56

oh okay 6

Articron 746

Leon 50

oh okay 6

Archived

Browse

Support

Important Information