diff --git a/OF DL/Entities/Config.cs b/OF DL/Entities/Config.cs index 0934422..a7f6699 100644 --- a/OF DL/Entities/Config.cs +++ b/OF DL/Entities/Config.cs @@ -57,6 +57,10 @@ namespace OF_DL.Entities [ToggleableConfig] public bool LimitDownloadRate { get; set; } = false; public int DownloadLimitInMbPerSec { get; set; } = 4; + public int ApiRateLimitRequests { get; set; } = 10; + public int ApiRateLimitWindowSeconds { get; set; } = 10; + public int MaxLogBodyLength { get; set; } = 2000; + public bool EnableJsonResponseBodyRawLogs { get; set; } = false; // Indicates if you want to download only on specific dates. [ToggleableConfig] diff --git a/OF DL/Entities/IDownloadConfig.cs b/OF DL/Entities/IDownloadConfig.cs index 21565d0..c8bc619 100644 --- a/OF DL/Entities/IDownloadConfig.cs +++ b/OF DL/Entities/IDownloadConfig.cs @@ -51,6 +51,12 @@ namespace OF_DL.Entities public LoggingLevel LoggingLevel { get; set; } bool IgnoreOwnMessages { get; set; } + + int ApiRateLimitRequests { get; set; } + int ApiRateLimitWindowSeconds { get; set; } + + int MaxLogBodyLength { get; set; } + bool EnableJsonResponseBodyRawLogs { get; set; } } } diff --git a/OF DL/Helpers/APIHelper.cs b/OF DL/Helpers/APIHelper.cs index 046c132..d2e863f 100644 --- a/OF DL/Helpers/APIHelper.cs +++ b/OF DL/Helpers/APIHelper.cs @@ -33,10 +33,16 @@ public class APIHelper : IAPIHelper private static DynamicRules? cachedDynamicRules; private const int MaxAttempts = 30; private const int DelayBetweenAttempts = 3000; + private static int ApiRateLimitCount = 10; + private static TimeSpan ApiRateLimitWindow = TimeSpan.FromSeconds(10); + private static readonly SemaphoreSlim ApiRateLimiter = new(1, 1); + private static readonly Queue ApiRequestTimes = new(); private static readonly ILogger DiagnosticLog = new LoggerConfiguration() .MinimumLevel.Debug() .WriteTo.File("logs/purchasedtab-diagnostics.txt", rollingInterval: RollingInterval.Day, retainedFileCountLimit: 7) .CreateLogger(); + private readonly int maxLogBodyLength; + private readonly bool enableRawJsonBodyLogs; static APIHelper() { @@ -51,6 +57,19 @@ public class APIHelper : IAPIHelper this.auth = auth; m_DBHelper = new DBHelper(downloadConfig); this.downloadConfig = downloadConfig; + maxLogBodyLength = downloadConfig.MaxLogBodyLength > 0 ? downloadConfig.MaxLogBodyLength : 2000; + enableRawJsonBodyLogs = downloadConfig.EnableJsonResponseBodyRawLogs; + if (downloadConfig.ApiRateLimitRequests > 0) + { + ApiRateLimitCount = downloadConfig.ApiRateLimitRequests; + } + if (downloadConfig.ApiRateLimitWindowSeconds > 0) + { + ApiRateLimitWindow = TimeSpan.FromSeconds(downloadConfig.ApiRateLimitWindowSeconds); + } + + Log.Debug("DiagnosticsEnabled={DiagnosticsEnabled} MaxLogBodyLength={MaxLogBodyLength} EnableRawJson={EnableRawJson} RawLogDir={RawDir}", + DiagnosticsEnabled, maxLogBodyLength, enableRawJsonBodyLogs, Path.Combine("logs", "diagnostics", "raw")); } @@ -122,20 +141,28 @@ public class APIHelper : IAPIHelper } - private async Task BuildHeaderAndExecuteRequests(Dictionary getParams, string endpoint, HttpClient client, ILogger? diagnosticLogger = null) + private async Task BuildHeaderAndExecuteRequests(Dictionary getParams, string endpoint, HttpClient client, ILogger? diagnosticLogger = null, string? rawLabel = null) { Log.Debug("Calling BuildHeaderAndExecuteRequests"); HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint); diagnosticLogger?.Information("Diag request | method={Method} url={Url} headers={Headers}", request.Method, request.RequestUri, FlattenHeaders(request.Headers)); + await EnforceApiRateLimitAsync(diagnosticLogger, request.RequestUri?.ToString()); using var response = await client.SendAsync(request); + if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests) + { + string respBody = await response.Content.ReadAsStringAsync(); + diagnosticLogger?.Warning("API 429 | url={Url} headers={Headers} respHeaders={RespHeaders} bodyPreview={Body}", request.RequestUri, FlattenHeaders(request.Headers), string.Join("; ", response.Headers.Select(h => $"{h.Key}={string.Join(",", h.Value)}")), TruncateForLog(respBody)); + Log.Warning("API 429 | url={Url}", request.RequestUri); + } diagnosticLogger?.Information("Diag response | status={StatusCode} reason={ReasonPhrase} requestUrl={Url}", response.StatusCode, response.ReasonPhrase, request.RequestUri); response.EnsureSuccessStatusCode(); string body = await response.Content.ReadAsStringAsync(); Log.Debug(body); - diagnosticLogger?.Information("Diag response body (truncated) | length={Length} body={Body}", body?.Length ?? 0, TruncateString(body, 2000)); + diagnosticLogger?.Information("Diag response body (truncated) | length={Length} body={Body}", body?.Length ?? 0, TruncateForLog(body)); + WriteRawBody(rawLabel ?? request.RequestUri?.ToString() ?? endpoint, body); return body; } @@ -190,6 +217,10 @@ public class APIHelper : IAPIHelper return string.Join("; ", headers.Select(h => $"{h.Key}={string.Join(",", h.Value)}")); } + private bool DiagnosticsEnabled => downloadConfig.LoggingLevel == LoggingLevel.Verbose; + + private ILogger? CurrentDiagnosticLogger => DiagnosticsEnabled ? DiagnosticLog : null; + private static string TruncateString(string? value, int maxLength = 1024) { if (string.IsNullOrEmpty(value)) @@ -199,6 +230,102 @@ public class APIHelper : IAPIHelper return value.Length <= maxLength ? value : $"{value[..maxLength]}...[truncated]"; } + private string TruncateForLog(string? value, int? overrideMax = null) + { + int max = overrideMax.HasValue ? Math.Min(overrideMax.Value, maxLogBodyLength) : maxLogBodyLength; + return TruncateString(value, max); + } + + private void DiagInfo(string messageTemplate, params object[] propertyValues) + { + if (!DiagnosticsEnabled) + { + return; + } + DiagnosticLog.Information(messageTemplate, propertyValues); + } + + private void DiagWarning(string messageTemplate, params object[] propertyValues) + { + if (!DiagnosticsEnabled) + { + return; + } + DiagnosticLog.Warning(messageTemplate, propertyValues); + } + + private string SanitizeForFile(string input) + { + var invalids = Path.GetInvalidFileNameChars(); + var safe = string.Concat(input.Select(ch => invalids.Contains(ch) ? '_' : ch)); + return string.IsNullOrWhiteSpace(safe) ? "response" : safe; + } + + private void WriteRawBody(string? label, string? body) + { + if (!enableRawJsonBodyLogs || string.IsNullOrEmpty(body)) + { + return; + } + try + { + string dir = Path.Combine("logs", "diagnostics", "raw"); + Directory.CreateDirectory(dir); + string namePart = !string.IsNullOrEmpty(label) ? SanitizeForFile(label) : "response"; + string fileName = $"{DateTime.UtcNow:yyyyMMddTHHmmssfff}-{namePart}.json"; + string fullPath = Path.Combine(dir, fileName); + File.WriteAllText(fullPath, body); + Log.Debug("Wrote raw body log to {Path}", fullPath); + } + catch (Exception ex) + { + Log.Debug("Failed to write raw body log: {Message}", ex.Message); + } + } + + private static async Task EnforceApiRateLimitAsync(ILogger? diagnosticLogger, string? context = null) + { + if (ApiRateLimitCount <= 0 || ApiRateLimitWindow.TotalSeconds <= 0) + { + return; + } + + while (true) + { + await ApiRateLimiter.WaitAsync(); + double waitSeconds = 0; + try + { + var now = DateTime.UtcNow; + while (ApiRequestTimes.Count > 0 && (now - ApiRequestTimes.Peek()) > ApiRateLimitWindow) + { + ApiRequestTimes.Dequeue(); + } + + if (ApiRequestTimes.Count < ApiRateLimitCount) + { + ApiRequestTimes.Enqueue(now); + return; + } + + waitSeconds = (ApiRateLimitWindow - (now - ApiRequestTimes.Peek())).TotalSeconds; + } + finally + { + ApiRateLimiter.Release(); + } + + if (waitSeconds < 0.1) + { + waitSeconds = 0.1; + } + + diagnosticLogger?.Information("API rate limit sleep | waitSeconds={WaitSeconds} currentCount={CurrentCount} context={Context}", waitSeconds, ApiRequestTimes.Count, context ?? string.Empty); + Log.Information("API rate limit sleep | waitSeconds={WaitSeconds} currentCount={CurrentCount} context={Context}", waitSeconds, ApiRequestTimes.Count, context ?? string.Empty); + await Task.Delay(TimeSpan.FromSeconds(waitSeconds)); + } + } + /// /// this one is used during initialization only @@ -254,6 +381,7 @@ public class APIHelper : IAPIHelper HttpClient client = new(); HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint); + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString()); using var response = await client.SendAsync(request); if (!response.IsSuccessStatusCode) @@ -287,6 +415,7 @@ public class APIHelper : IAPIHelper HttpClient client = new(); HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary(), endpoint); + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString()); using var response = await client.SendAsync(request); response.EnsureSuccessStatusCode(); @@ -683,6 +812,7 @@ public class APIHelper : IAPIHelper highlight_request.Headers.Add(keyValuePair.Key, keyValuePair.Value); } + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger); using var highlightResponse = await highlight_client.SendAsync(highlight_request); highlightResponse.EnsureSuccessStatusCode(); var highlightBody = await highlightResponse.Content.ReadAsStringAsync(); @@ -1895,6 +2025,7 @@ public class APIHelper : IAPIHelper { looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); } + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString()); using (var loopresponse = await loopclient.SendAsync(looprequest)) { loopresponse.EnsureSuccessStatusCode(); @@ -2124,6 +2255,7 @@ public class APIHelper : IAPIHelper { looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); } + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger); using (var loopresponse = await loopclient.SendAsync(looprequest)) { loopresponse.EnsureSuccessStatusCode(); @@ -2284,15 +2416,15 @@ public class APIHelper : IAPIHelper string initialQueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); var initialHeaders = GetDynamicHeaders("/api2/v2" + endpoint, initialQueryParams); - DiagnosticLog.Information("PurchasedTab start | ctx={Context} endpoint={Endpoint} folder={Folder} usersCount={UsersCount} limit={Limit} query={Query} headers={Headers}", diagContext, endpoint, folder, users?.Count ?? 0, post_limit, initialQueryParams, string.Join(", ", initialHeaders.Select(kvp => $"{kvp.Key}={kvp.Value}"))); + DiagInfo("PurchasedTab start | ctx={Context} endpoint={Endpoint} folder={Folder} usersCount={UsersCount} limit={Limit} query={Query} headers={Headers}", diagContext, endpoint, folder, users?.Count ?? 0, post_limit, initialQueryParams, string.Join(", ", initialHeaders.Select(kvp => $"{kvp.Key}={kvp.Value}"))); - var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), DiagnosticLog); - DiagnosticLog.Information("PurchasedTab initial response | ctx={Context} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, body?.Length ?? 0, TruncateString(body, 1500)); + var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, diagContext); + DiagInfo("PurchasedTab initial response | ctx={Context} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, body?.Length ?? 0, TruncateForLog(body)); purchased = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - DiagnosticLog.Information("PurchasedTab parsed initial | ctx={Context} items={Items} hasMore={HasMore}", diagContext, purchased?.list?.Count ?? 0, purchased?.hasMore ?? false); + DiagInfo("PurchasedTab parsed initial | ctx={Context} items={Items} hasMore={HasMore}", diagContext, purchased?.list?.Count ?? 0, purchased?.hasMore ?? false); if (purchased == null || purchased.list == null) { - DiagnosticLog.Warning("PurchasedTab parsed initial null list | ctx={Context} bodyPreview={BodyPreview}", diagContext, TruncateString(body, 1500)); + DiagWarning("PurchasedTab parsed initial null list | ctx={Context} bodyPreview={BodyPreview}", diagContext, TruncateForLog(body)); } if (purchased != null && purchased.hasMore) { @@ -2310,14 +2442,16 @@ public class APIHelper : IAPIHelper { looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); } - DiagnosticLog.Information("PurchasedTab page request | ctx={Context} url={Url} offset={Offset} limit={Limit} headers={Headers}", diagContext, looprequest.RequestUri, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit, FlattenHeaders(looprequest.Headers)); + DiagInfo("PurchasedTab page request | ctx={Context} url={Url} offset={Offset} limit={Limit} headers={Headers}", diagContext, looprequest.RequestUri, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit, FlattenHeaders(looprequest.Headers)); + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString()); using (var loopresponse = await loopclient.SendAsync(looprequest)) { - DiagnosticLog.Information("PurchasedTab page response | ctx={Context} status={Status} reason={Reason} url={Url}", diagContext, loopresponse.StatusCode, loopresponse.ReasonPhrase, looprequest.RequestUri); + DiagInfo("PurchasedTab page response | ctx={Context} status={Status} reason={Reason} url={Url}", diagContext, loopresponse.StatusCode, loopresponse.ReasonPhrase, looprequest.RequestUri); loopresponse.EnsureSuccessStatusCode(); var loopbody = await loopresponse.Content.ReadAsStringAsync(); newPurchased = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); - DiagnosticLog.Information("PurchasedTab page parsed | ctx={Context} offset={Offset} items={Items} hasMore={HasMore} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", newPurchased?.list?.Count ?? 0, newPurchased?.hasMore ?? false, loopbody?.Length ?? 0, TruncateString(loopbody, 1500)); + DiagInfo("PurchasedTab page parsed | ctx={Context} offset={Offset} items={Items} hasMore={HasMore} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", newPurchased?.list?.Count ?? 0, newPurchased?.hasMore ?? false, loopbody?.Length ?? 0, TruncateForLog(loopbody)); + WriteRawBody($"{diagContext}-page-{getParams["offset"]}", loopbody); } purchased.list.AddRange(newPurchased.list); if (!newPurchased.hasMore) @@ -2358,13 +2492,14 @@ public class APIHelper : IAPIHelper string resolvedUsername = $"Deleted User - {user.Key}"; if (userObject == null || userObject[user.Key.ToString()] == null || userObject[user.Key.ToString()]["username"] == null) { - DiagnosticLog.Warning("PurchasedTab user lookup missing username | ctx={Context} userId={UserId} rawUser={RawUser}", diagContext, user.Key, userObject != null ? TruncateString(userObject.ToString(), 500) : "null"); + DiagWarning("PurchasedTab user lookup missing username | ctx={Context} userId={UserId} rawUser={RawUser}", diagContext, user.Key, userObject != null ? TruncateForLog(userObject.ToString()) : "null"); } else { resolvedUsername = userObject[user.Key.ToString()]["username"].ToString(); - DiagnosticLog.Information("PurchasedTab user lookup success | ctx={Context} userId={UserId} username={Username} rawUser={RawUser}", diagContext, user.Key, resolvedUsername, TruncateString(userObject.ToString(), 1500)); + DiagInfo("PurchasedTab user lookup success | ctx={Context} userId={UserId} username={Username} rawUser={RawUser}", diagContext, user.Key, resolvedUsername, TruncateForLog(userObject.ToString())); } + WriteRawBody($"{diagContext}-user-{user.Key}", userObject?.ToString()); purchasedTabCollection.UserId = user.Key; purchasedTabCollection.Username = userObject is not null && !string.IsNullOrEmpty(userObject[user.Key.ToString()]["username"].ToString()) ? userObject[user.Key.ToString()]["username"].ToString() : $"Deleted User - {user.Key}"; string path = System.IO.Path.Combine(folder, purchasedTabCollection.Username); @@ -2372,11 +2507,12 @@ public class APIHelper : IAPIHelper { foreach (Purchased.List purchase in user.Value) { - DiagnosticLog.Information("PurchasedTab processing purchase | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} mediaNull={MediaNull} mediaCount={MediaCount} postedAt={PostedAt} createdAt={CreatedAt} fromUserId={FromUserId} authorId={AuthorId}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.media == null, purchase.media?.Count ?? 0, purchase.postedAt, purchase.createdAt, purchase.fromUser?.id, purchase.author?.id); - DiagnosticLog.Information("PurchasedTab purchase raw | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} rawPurchase={RawPurchase}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, TruncateString(JsonConvert.SerializeObject(purchase), 2000)); + DiagInfo("PurchasedTab processing purchase | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} mediaNull={MediaNull} mediaCount={MediaCount} postedAt={PostedAt} createdAt={CreatedAt} fromUserId={FromUserId} authorId={AuthorId}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.media == null, purchase.media?.Count ?? 0, purchase.postedAt, purchase.createdAt, purchase.fromUser?.id, purchase.author?.id); + DiagInfo("PurchasedTab purchase raw | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} rawPurchase={RawPurchase}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, TruncateForLog(JsonConvert.SerializeObject(purchase))); + WriteRawBody($"{diagContext}-purchase-{purchase.id}", JsonConvert.SerializeObject(purchase)); if (purchase.media == null) { - DiagnosticLog.Warning("PurchasedTab purchase has null media | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} rawPurchase={RawPurchase}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, TruncateString(JsonConvert.SerializeObject(purchase), 1500)); + DiagWarning("PurchasedTab purchase has null media | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} rawPurchase={RawPurchase}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, TruncateForLog(JsonConvert.SerializeObject(purchase))); } switch (purchase.responseType) { @@ -2661,6 +2797,7 @@ public class APIHelper : IAPIHelper request.Headers.Add("user-agent", auth.USER_AGENT); request.Headers.Add("Accept", "*/*"); request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.COOKIE};"); + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString()); using (var response = await client.SendAsync(request)) { response.EnsureSuccessStatusCode(); @@ -2706,6 +2843,7 @@ public class APIHelper : IAPIHelper request.Headers.Add("user-agent", auth.USER_AGENT); request.Headers.Add("Accept", "*/*"); request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.COOKIE};"); + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString()); using (var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead)) { response.EnsureSuccessStatusCode(); @@ -2762,6 +2900,7 @@ public class APIHelper : IAPIHelper Content = new StringContent(json, Encoding.UTF8, "application/json") }; + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString()); using var response = await client.SendAsync(request); Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}"); @@ -2841,6 +2980,7 @@ public class APIHelper : IAPIHelper Content = new StringContent(json, Encoding.UTF8, "application/json") }; + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString()); using var response = await client.SendAsync(request); if (!response.IsSuccessStatusCode) diff --git a/OF DL/Program.cs b/OF DL/Program.cs index f9a3a6b..ffa9747 100644 --- a/OF DL/Program.cs +++ b/OF DL/Program.cs @@ -35,6 +35,32 @@ public class Program private static Auth? auth = null; private static LoggingLevelSwitch levelSwitch = new LoggingLevelSwitch(); + private static int SafeGetInt(Akka.Configuration.Config hoconConfig, string path, int defaultValue) + { + try + { + var value = hoconConfig.GetString(path); + return string.IsNullOrWhiteSpace(value) ? defaultValue : hoconConfig.GetInt(path); + } + catch + { + return defaultValue; + } + } + + private static bool SafeGetBool(Akka.Configuration.Config hoconConfig, string path, bool defaultValue) + { + try + { + var value = hoconConfig.GetString(path); + return string.IsNullOrWhiteSpace(value) ? defaultValue : hoconConfig.GetBoolean(path); + } + catch + { + return defaultValue; + } + } + private static async Task LoadAuthFromBrowser() { bool runningInDocker = Environment.GetEnvironmentVariable("OFDL_DOCKER") != null; @@ -219,11 +245,15 @@ public class Program hoconConfig.AppendLine($" Timeout = {(jsonConfig.Timeout.HasValue ? jsonConfig.Timeout.Value : -1)}"); hoconConfig.AppendLine($" LimitDownloadRate = {jsonConfig.LimitDownloadRate.ToString().ToLower()}"); hoconConfig.AppendLine($" DownloadLimitInMbPerSec = {jsonConfig.DownloadLimitInMbPerSec}"); + hoconConfig.AppendLine($" ApiRateLimitRequests = {jsonConfig.ApiRateLimitRequests}"); + hoconConfig.AppendLine($" ApiRateLimitWindowSeconds = {jsonConfig.ApiRateLimitWindowSeconds}"); hoconConfig.AppendLine("}"); hoconConfig.AppendLine("# Logging/Debug Settings"); hoconConfig.AppendLine("Logging {"); hoconConfig.AppendLine($" LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\""); + hoconConfig.AppendLine($" MaxLogBodyLength = {jsonConfig.MaxLogBodyLength}"); + hoconConfig.AppendLine($" EnableJsonResponseBodyRawLogs = {jsonConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}"); hoconConfig.AppendLine("}"); File.WriteAllText("config.conf", hoconConfig.ToString()); @@ -318,9 +348,13 @@ public class Program Timeout = string.IsNullOrWhiteSpace(hoconConfig.GetString("Performance.Timeout")) ? -1 : hoconConfig.GetInt("Performance.Timeout"), LimitDownloadRate = hoconConfig.GetBoolean("Performance.LimitDownloadRate"), DownloadLimitInMbPerSec = hoconConfig.GetInt("Performance.DownloadLimitInMbPerSec"), + ApiRateLimitRequests = SafeGetInt(hoconConfig, "Performance.ApiRateLimitRequests", 10), + ApiRateLimitWindowSeconds = SafeGetInt(hoconConfig, "Performance.ApiRateLimitWindowSeconds", 10), // Logging/Debug Settings - LoggingLevel = Enum.Parse(hoconConfig.GetString("Logging.LoggingLevel"), true) + LoggingLevel = Enum.Parse(hoconConfig.GetString("Logging.LoggingLevel"), true), + MaxLogBodyLength = SafeGetInt(hoconConfig, "Logging.MaxLogBodyLength", 2000), + EnableJsonResponseBodyRawLogs = SafeGetBool(hoconConfig, "Logging.EnableJsonResponseBodyRawLogs", false) }; ValidateFileNameFormat(config.PaidPostFileNameFormat, "PaidPostFileNameFormat"); @@ -467,11 +501,15 @@ public class Program hoconConfig.AppendLine($" Timeout = {(jsonConfig.Timeout.HasValue ? jsonConfig.Timeout.Value : -1)}"); hoconConfig.AppendLine($" LimitDownloadRate = {jsonConfig.LimitDownloadRate.ToString().ToLower()}"); hoconConfig.AppendLine($" DownloadLimitInMbPerSec = {jsonConfig.DownloadLimitInMbPerSec}"); + hoconConfig.AppendLine($" ApiRateLimitRequests = {jsonConfig.ApiRateLimitRequests}"); + hoconConfig.AppendLine($" ApiRateLimitWindowSeconds = {jsonConfig.ApiRateLimitWindowSeconds}"); hoconConfig.AppendLine("}"); hoconConfig.AppendLine("# Logging/Debug Settings"); hoconConfig.AppendLine("Logging {"); hoconConfig.AppendLine($" LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\""); + hoconConfig.AppendLine($" MaxLogBodyLength = {jsonConfig.MaxLogBodyLength}"); + hoconConfig.AppendLine($" EnableJsonResponseBodyRawLogs = {jsonConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}"); hoconConfig.AppendLine("}"); File.WriteAllText("config.conf", hoconConfig.ToString()); @@ -3043,11 +3081,15 @@ public class Program hoconConfig.AppendLine($" Timeout = {(newConfig.Timeout.HasValue ? newConfig.Timeout.Value : -1)}"); hoconConfig.AppendLine($" LimitDownloadRate = {newConfig.LimitDownloadRate.ToString().ToLower()}"); hoconConfig.AppendLine($" DownloadLimitInMbPerSec = {newConfig.DownloadLimitInMbPerSec}"); + hoconConfig.AppendLine($" ApiRateLimitRequests = {newConfig.ApiRateLimitRequests}"); + hoconConfig.AppendLine($" ApiRateLimitWindowSeconds = {newConfig.ApiRateLimitWindowSeconds}"); hoconConfig.AppendLine("}"); hoconConfig.AppendLine("# Logging/Debug Settings"); hoconConfig.AppendLine("Logging {"); hoconConfig.AppendLine($" LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\""); + hoconConfig.AppendLine($" MaxLogBodyLength = {newConfig.MaxLogBodyLength}"); + hoconConfig.AppendLine($" EnableJsonResponseBodyRawLogs = {newConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}"); hoconConfig.AppendLine("}"); File.WriteAllText("config.conf", hoconConfig.ToString()); @@ -3205,11 +3247,15 @@ public class Program hoconConfig.AppendLine($" Timeout = {(newConfig.Timeout.HasValue ? newConfig.Timeout.Value : -1)}"); hoconConfig.AppendLine($" LimitDownloadRate = {newConfig.LimitDownloadRate.ToString().ToLower()}"); hoconConfig.AppendLine($" DownloadLimitInMbPerSec = {newConfig.DownloadLimitInMbPerSec}"); + hoconConfig.AppendLine($" ApiRateLimitRequests = {newConfig.ApiRateLimitRequests}"); + hoconConfig.AppendLine($" ApiRateLimitWindowSeconds = {newConfig.ApiRateLimitWindowSeconds}"); hoconConfig.AppendLine("}"); hoconConfig.AppendLine("# Logging/Debug Settings"); hoconConfig.AppendLine("Logging {"); hoconConfig.AppendLine($" LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\""); + hoconConfig.AppendLine($" MaxLogBodyLength = {newConfig.MaxLogBodyLength}"); + hoconConfig.AppendLine($" EnableJsonResponseBodyRawLogs = {newConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}"); hoconConfig.AppendLine("}"); File.WriteAllText("config.conf", hoconConfig.ToString());