From 6a869afb1dc2a0068493926881e945b203820167 Mon Sep 17 00:00:00 2001 From: anthorob Date: Tue, 24 Dec 2024 09:03:31 -0500 Subject: [PATCH] Add request caching on the WebModule --- .../command/crypto/CryptoPriceCommand.java | 8 +- .../bobert/netneutrality/http/WebModule.java | 415 +++++++++++------- .../http/server/ActionHandler.java | 6 +- .../bobert/netneutrality/utils/Settings.java | 3 +- 4 files changed, 276 insertions(+), 156 deletions(-) diff --git a/src/main/java/me/bobert/netneutrality/command/crypto/CryptoPriceCommand.java b/src/main/java/me/bobert/netneutrality/command/crypto/CryptoPriceCommand.java index 2fc8fed..f44fd99 100644 --- a/src/main/java/me/bobert/netneutrality/command/crypto/CryptoPriceCommand.java +++ b/src/main/java/me/bobert/netneutrality/command/crypto/CryptoPriceCommand.java @@ -20,11 +20,12 @@ import org.json.JSONObject; import java.text.NumberFormat; import java.util.Currency; import java.util.Locale; +import java.util.concurrent.TimeUnit; public class CryptoPriceCommand extends AbstractCommand { private final String exchange = "Coinbase"; - private final String baseURL = "https://min-api.cryptocompare.com/data/price"; + private final String baseURL = "https://min-api.cryptocompare.com/data/price?fsym=%s&tsyms=%s"; private final String exchangeApi = String.format("https://min-api.cryptocompare.com/data/v4/all/exchanges?e=%s", exchange); @Override @@ -68,7 +69,6 @@ public class CryptoPriceCommand extends AbstractCommand { @Override public CommandExecutionResult execute(CommandContext context) { - //TODO: implement cache Locale usa = new Locale("en", "US"); NumberFormat dollarFormat = NumberFormat.getCurrencyInstance(usa); dollarFormat.setMaximumFractionDigits(24); @@ -90,7 +90,7 @@ public class CryptoPriceCommand extends AbstractCommand { return; } - WebModule http = new WebModule(String.format("%s?fsym=%s&tsyms=%s", baseURL, symbol, finalCurrency)).request(); + WebModule http = new WebModule.Builder(String.format(baseURL, symbol, finalCurrency)).cache(30, TimeUnit.MINUTES).build().request(); JSONObject json = http.getAsObject(); if (json.isEmpty() || (json.has("Response") && json.getString("Response").equalsIgnoreCase("Error"))) { @@ -114,7 +114,7 @@ public class CryptoPriceCommand extends AbstractCommand { private boolean validateSymbol(String symbol) { try { - WebModule http = new WebModule(exchangeApi).request(); + WebModule http = new WebModule.Builder(exchangeApi).get().cache(1, TimeUnit.DAYS).build().request(); JSONObject json = http.getAsObject(); JSONObject pairs = json.getJSONObject("Data") diff --git a/src/main/java/me/bobert/netneutrality/http/WebModule.java b/src/main/java/me/bobert/netneutrality/http/WebModule.java index 04ed06d..7fbe042 100644 --- a/src/main/java/me/bobert/netneutrality/http/WebModule.java +++ b/src/main/java/me/bobert/netneutrality/http/WebModule.java @@ -6,6 +6,7 @@ package me.bobert.netneutrality.http; * https://discord.gg/qUa6zNFZsd */ +import com.google.gson.Gson; import io.sentry.Breadcrumb; import io.sentry.Sentry; import me.bobert.netneutrality.NetNeutrality; @@ -13,187 +14,305 @@ import me.bobert.netneutrality.core.LogHandler; import okhttp3.*; import org.json.JSONArray; import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.List; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; -import java.util.TreeMap; import java.util.concurrent.TimeUnit; public class WebModule { + private enum Method { + GET, POST + } - private final String url; - private final OkHttpClient client = new OkHttpClient.Builder() - .connectTimeout(5, TimeUnit.SECONDS) - .writeTimeout(5, TimeUnit.SECONDS) - .readTimeout(5, TimeUnit.SECONDS) - .build(); - private Response response; - //private final String userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 OPR/82.0.4227.43"; - private static final String userAgent = String.format("Net Neutrality/%s (Discord bot, tony@bobert.me)", NetNeutrality.VERSION); + private static final String USER_AGENT = "Net Neutrality/%s (Discord bot, discord.gg/trAqhQ5Wsr)"; + private static final Logger log = LoggerFactory.getLogger(WebModule.class); + private static final Map cache = Collections.synchronizedMap( + new LinkedHashMap<>(100, 0.75f, true) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > 50; // Limit cache to 50 entries + } + } + ); - public WebModule(String url) { - this.url = url; - } + private final Method method; + private final String url; + private final long cacheExpirationMillis; + private final OkHttpClient client; + private final Map headers; + private final Map params; + private final Request request; + private CachedResponse cachedResponse; - /** - * Send a request to the specified URL in the constructor. - *
Needs to be executed before any other method call - * - * @return an instance of the WebModule class with the Response of the Web Request - */ - public WebModule request() { - Sentry.addBreadcrumb(Breadcrumb.http(url, "get")); - Request request = new Request.Builder().url(this.url) - .addHeader("User-Agent", userAgent) - .build(); - try { - response = client.newCall(request).execute(); - Sentry.addBreadcrumb(Breadcrumb.http(url, "get", response.code())); - } catch (IOException e) { - Sentry.captureException(e); - } + private WebModule(Builder builder) { + this.method = builder.method; + this.url = builder.url; + this.client = builder.client; + this.headers = builder.headers; + this.params = builder.params; + this.cacheExpirationMillis = builder.timeUnit.toMillis(builder.timeout); + this.request = createRequest(); + } - return this; - } + /** + * Invalidates a cached response for the specified URL. + *

+ * This method removes any cached responses matching the given URL. + * + * @param url the URL of the cached response to invalidate + */ + public static void invalidateCache(String url) { + cache.keySet().removeIf(s -> s.contains(url)); + } - public WebModule request(Map headers) { - Sentry.addBreadcrumb(Breadcrumb.http(url, "get")); - Request.Builder builder = new Request.Builder().url(this.url) - .addHeader("User-Agent", userAgent); + private Request createRequest() { + Request.Builder builder = new Request.Builder().url(url); + builder.addHeader("User-Agent", String.format(USER_AGENT, NetNeutrality.VERSION)); + headers.forEach(builder::addHeader); - for (Map.Entry header : headers.entrySet()) { - builder.addHeader(header.getKey(), header.getValue()); - } + if (method == Method.POST && !params.isEmpty()) { + RequestBody body = RequestBody.create(new Gson().toJson(params), MediaType.parse("application/json; charset=utf-8")); + builder.post(body); + } - Request request = builder.build(); + return builder.build(); + } - try { - response = client.newCall(request).execute(); - Sentry.addBreadcrumb(Breadcrumb.http(url, "get", response.code())); - } catch (IOException e) { - Sentry.captureException(e); - } + /** + * Sends the HTTP request based on the configuration and retrieves the response. + *

+ * If the request is a GET request and caching is enabled, the response will be cached. + * Cached responses are reused until they expire. + *

+ * POST requests do not support caching. + * + * @return the {@code WebModule} instance, allowing further interaction with the response + * @throws RuntimeException if the request fails + */ + public WebModule request() { + String cacheKey = createCacheKey(method, url, headers); - return this; - } - - public WebModule post(TreeMap params, Map headers) { - Sentry.addBreadcrumb(Breadcrumb.http(url, "post")); - RequestBody body = RequestBody.create(createJson(params), MediaType.parse("application/json; charset=utf-8")); - Request.Builder builder = new Request.Builder().url(this.url) - .addHeader("User-Agent", userAgent); + if (method == Method.GET && cacheKey != null) { + CachedResponse cachedResponse = cache.get(cacheKey); - for (Map.Entry header : headers.entrySet()) { - builder.addHeader(header.getKey(), header.getValue()); - } - Request request = builder.post(body).build(); - try { - response = client.newCall(request).execute(); - Sentry.addBreadcrumb(Breadcrumb.http(url, "post", response.code())); - } catch (IOException e) { - Sentry.captureException(e); - } + if (cachedResponse != null && !cachedResponse.isExpired()) { + log.debug("Retrieving cached response: {}", cacheKey); + this.cachedResponse = cachedResponse; + return this; + } + } - return this; - } + try (Response response = client.newCall(request).execute()) { + Sentry.addBreadcrumb(Breadcrumb.http(url, method.name(), response.code())); + String body = response.body() != null ? response.body().string() : null; - public static String createJson(TreeMap params) { - StringBuilder json = new StringBuilder(); + this.cachedResponse = new CachedResponse( + response.code(), + response.headers(), + body, + System.currentTimeMillis() + cacheExpirationMillis + ); - for (Map.Entry param : params.entrySet()) { - String str = "\"" + param.getKey() + "\":\"" + param.getValue() + "\""; - json.append(str); - json.append(","); - } - json.insert(0, "{"); + if (method == Method.GET && cacheKey != null) { + log.debug("Caching request: {}", cacheKey); + cache.put(cacheKey, this.cachedResponse); + } + } catch (IOException e) { + Sentry.captureException(e); + throw new RuntimeException("Failed to execute request", e); + } - return (json.length() > 1 ? json.substring(0, json.length() - 1) + "}" : ""); - } + return this; + } - public static String createJsonObject(List key, List value) { - if (key.size() != value.size()) - throw new IllegalArgumentException("Key list size not equal to value list size"); + private String createCacheKey(Method method, String url, Map headers) { + if (method == Method.POST) { + return null; // no cache for post requests + } - StringBuilder json = new StringBuilder(); + StringBuilder keyBuilder = new StringBuilder(method.name()).append(":").append(url); + if (headers != null) { + headers.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> keyBuilder.append(entry.getKey()).append("=").append(entry.getValue()).append(";")); + } + return keyBuilder.toString(); + } - for (int index = 0; index < key.size();) { - json.append(String.format("\"%s\":\"%s\"", key.get(index), value.get(index))); - json.append(","); - } - json.insert(0, "{"); + public JSONObject getAsObject() { + try { + return new JSONObject(cachedResponse.getBody()); + } catch (NullPointerException e) { + LogHandler.log(e); + return new JSONObject(); + } + } - return (json.length() > 1 ? json.substring(0, json.length() - 1) + "}" : ""); - } + public JSONArray getAsArray() { + try { + return new JSONArray(cachedResponse.getBody()); + } catch (NullPointerException e) { + LogHandler.log(e); + return new JSONArray(); + } + } - public static String createJsonArray(List list) { - StringBuilder json = new StringBuilder(); + public String getRawData() { + return cachedResponse.getBody(); + } - for (String str : list) { - json.append("\"").append(str).append("\""); - json.append(","); - } - json.insert(0, "["); + public static class Builder { + private final String url; + private Method method = Method.GET; + private long timeout = 5; + private TimeUnit timeUnit = TimeUnit.MINUTES; + private OkHttpClient client; + private long connectTimeout; + private long readTimeout; + private long writeTimeout; + private final Map headers = new HashMap<>(); + private final Map params = new HashMap<>(); - return (json.length() > 1 ? json.substring(0, json.length() - 1) + "]" : ""); - } + /** + * Creates a new {@code Builder} instance for the specified URL. + * + * @param url the URL for the HTTP request + */ + public Builder(String url) { + this.url = url; + } - public static String createJsonArray(TreeMap users) { - StringBuilder json = new StringBuilder(); + /** + * Configures the request to use the HTTP GET method. + * + * @return the current {@code Builder} instance for chaining + */ + public Builder get() { + this.method = Method.GET; + return this; + } - for (Map.Entry param : users.entrySet()) { - String str = "{\"" + param.getKey() + "\":\"" + param.getValue() + "\"}"; - json.append(str); - json.append(","); - } - json.insert(0, "["); + /** + * Configures the request to use the HTTP POST method and sets the request parameters. + * + * @param params a map of key-value pairs to include in the POST request body + * @return the current {@code Builder} instance for chaining + */ + public Builder post(Map params) { + this.params.putAll(params); + this.method = Method.POST; + return this; + } - return (json.length() > 1 ? json.substring(0, json.length() - 1) + "]" : "[]"); - } + /** + * Adds a custom header to the request. + * + * @param key the name of the header + * @param value the value of the header + * @return the current {@code Builder} instance for chaining + */ + public Builder addHeader(String key, String value) { + this.headers.put(key, value); + return this; + } - - /** - * @return the response as a JSONObject - */ - public JSONObject getAsObject() { - if (response == null) - throw new RuntimeException("Request has not been sent"); + /** + * Configures caching for the request. + * + * @param timeout the cache expiration duration + * @param timeUnit the unit of time for the cache expiration + * @return the current {@code Builder} instance for chaining + */ + public Builder cache(long timeout, TimeUnit timeUnit) { + this.timeout = timeout; + this.timeUnit = timeUnit; + return this; + } - try { - ResponseBody body = response.body(); - return new JSONObject(body.string()); - } catch (IOException | NullPointerException e) { - LogHandler.log(e); - return new JSONObject(); - } - } - - /** - * Use if your response start with an array : [ ] - * - * @return the response as a JSONArray - */ - public JSONArray getAsArray() { - if (response == null) - throw new RuntimeException("Request has not been sent"); + /** + * Sets the connection timeout for the HTTP client. + * + * @param timeout the timeout duration + * @param timeUnit the unit of time for the timeout + * @return the current {@code Builder} instance for chaining + */ + public Builder connectTimeout(long timeout, TimeUnit timeUnit) { + this.connectTimeout = timeUnit.toMillis(timeout); + return this; + } - try { - ResponseBody body = response.body(); - return new JSONArray(body.string()); - } catch (IOException | NullPointerException e) { - LogHandler.log(e); - return new JSONArray(); - } - } - - /** - * @return the response as a String - */ - public String getRawData() throws IOException { - if (response == null) - throw new RuntimeException("Request has not been sent"); + /** + * Sets the read timeout for the HTTP client. + * + * @param timeout the timeout duration + * @param timeUnit the unit of time for the timeout + * @return the current {@code Builder} instance for chaining + */ + public Builder readTimeout(long timeout, TimeUnit timeUnit) { + this.readTimeout = timeUnit.toMillis(timeout); + return this; + } - return response.body().string(); - } + /** + * Sets the write timeout for the HTTP client. + * + * @param timeout the timeout duration + * @param timeUnit the unit of time for the timeout + * @return the current {@code Builder} instance for chaining + */ + public Builder writeTimeout(long timeout, TimeUnit timeUnit) { + this.writeTimeout = timeUnit.toMillis(timeout); + return this; + } + /** + * Builds and returns a configured {@link WebModule} instance. + * + * @return a new {@code WebModule} instance + */ + public WebModule build() { + this.client = new OkHttpClient.Builder() + .connectTimeout(connectTimeout, TimeUnit.MILLISECONDS) + .writeTimeout(writeTimeout, TimeUnit.MILLISECONDS) + .readTimeout(readTimeout, TimeUnit.MILLISECONDS) + .build(); + + return new WebModule(this); + } + } + + private static class CachedResponse { + private final int statusCode; + private final Headers headers; + private final String body; + private final long expirationTime; + + public CachedResponse(int statusCode, Headers headers, String body, long expirationTime) { + this.statusCode = statusCode; + this.headers = headers; + this.body = body; + this.expirationTime = expirationTime; + } + + public boolean isExpired() { + return System.currentTimeMillis() > expirationTime; + } + + public int getStatusCode() { + return statusCode; + } + + public Headers getHeaders() { + return headers; + } + + public String getBody() { + return body; + } + } } diff --git a/src/main/java/me/bobert/netneutrality/http/server/ActionHandler.java b/src/main/java/me/bobert/netneutrality/http/server/ActionHandler.java index e73bf19..ce5b38a 100644 --- a/src/main/java/me/bobert/netneutrality/http/server/ActionHandler.java +++ b/src/main/java/me/bobert/netneutrality/http/server/ActionHandler.java @@ -6,10 +6,10 @@ package me.bobert.netneutrality.http.server; * https://discord.gg/qUa6zNFZsd */ +import com.google.gson.Gson; import com.sun.net.httpserver.HttpExchange; import me.bobert.netneutrality.NetNeutrality; import me.bobert.netneutrality.core.GuildManager; -import me.bobert.netneutrality.http.WebModule; import net.dv8tion.jda.api.entities.Guild; import org.apache.logging.log4j.LogManager; import org.json.JSONObject; @@ -38,7 +38,7 @@ public class ActionHandler extends AbstractHandler { json.put("health", String.valueOf(NetNeutrality.getInstance().getJDA().getStatus())); json.put("ping", String.valueOf(NetNeutrality.getInstance().getJDA().getGatewayPing())); - sendJsonResponse(exchange, WebModule.createJson(json)); + sendJsonResponse(exchange, new Gson().toJson(json)); return; } @@ -77,7 +77,7 @@ public class ActionHandler extends AbstractHandler { json.put("status", String.valueOf(ok)); - sendJsonResponse(exchange, WebModule.createJson(json)); + sendJsonResponse(exchange, new Gson().toJson(json)); return; } diff --git a/src/main/java/me/bobert/netneutrality/utils/Settings.java b/src/main/java/me/bobert/netneutrality/utils/Settings.java index 6e9cde9..938ac50 100644 --- a/src/main/java/me/bobert/netneutrality/utils/Settings.java +++ b/src/main/java/me/bobert/netneutrality/utils/Settings.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.Map; import java.util.Properties; +import java.util.concurrent.TimeUnit; public class Settings { @@ -100,7 +101,7 @@ public class Settings { String gitlabProjectId = "22"; String latestReleaseUrl = String.format("%s/api/v4/projects/%s/releases/", gitlabHost, gitlabProjectId); - JSONArray json = new WebModule(latestReleaseUrl).request().getAsArray(); + JSONArray json = new WebModule.Builder(latestReleaseUrl).cache(1, TimeUnit.DAYS).build().request().getAsArray(); if (json.isNull(0)) { return "0.0.0"; }