Adding fallback for downloading ffmpeg when publishing

.gitea/workflows/publish-release.yml

@@ -52,9 +52,20 @@ jobs:
           echo "➤ Creating folder for CDM"
           mkdir -p cdm/devices/chrome_1610
 
-          echo "➤ Copying ffmpeg from user folder"
+          echo "➤ Copying ffmpeg"
+          if [ -f /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe ] && [ -f /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE ]; then
             cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
             cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
+          else
+            echo "➤ Downloading ffmpeg package"
+            mkdir -p /tmp/ffmpeg
+            if [ ! -f /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build.zip ]; then
+              curl -L https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-7.1.1-essentials_build.zip -o /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build.zip
+            fi
+            unzip -o /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build.zip -d /tmp/ffmpeg
+            cp /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
+            cp /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
+          fi
 
           echo "➤ Creating release zip"
           zip ../OFDLV${{ steps.version.outputs.version }}.zip OF\ DL.exe e_sqlite3.dll rules.json config.conf cdm ffmpeg.exe LICENSE.ffmpeg

OF DL/Entities/Config.cs

@@ -57,6 +57,10 @@ namespace OF_DL.Entities
         [ToggleableConfig]
         public bool LimitDownloadRate { get; set; } = false;
         public int DownloadLimitInMbPerSec { get; set; } = 4;
+        public int ApiRateLimitRequests { get; set; } = 10;
+        public int ApiRateLimitWindowSeconds { get; set; } = 10;
+        public int MaxLogBodyLength { get; set; } = 2000;
+        public bool EnableJsonResponseBodyRawLogs { get; set; } = false;
 
         // Indicates if you want to download only on specific dates.
         [ToggleableConfig]

OF DL/Entities/IDownloadConfig.cs

@@ -51,6 +51,12 @@ namespace OF_DL.Entities
         public LoggingLevel LoggingLevel { get; set; }
 
         bool IgnoreOwnMessages { get; set; }
+
+        int ApiRateLimitRequests { get; set; }
+        int ApiRateLimitWindowSeconds { get; set; }
+
+        int MaxLogBodyLength { get; set; }
+        bool EnableJsonResponseBodyRawLogs { get; set; }
     }
 
 }

OF DL/Helpers/APIHelper.cs

@@ -33,6 +33,16 @@ public class APIHelper : IAPIHelper
     private static DynamicRules? cachedDynamicRules;
     private const int MaxAttempts = 30;
     private const int DelayBetweenAttempts = 3000;
+    private static int ApiRateLimitCount = 10;
+    private static TimeSpan ApiRateLimitWindow = TimeSpan.FromSeconds(10);
+    private static readonly SemaphoreSlim ApiRateLimiter = new(1, 1);
+    private static readonly Queue<DateTime> ApiRequestTimes = new();
+    private static readonly ILogger DiagnosticLog = new LoggerConfiguration()
+        .MinimumLevel.Debug()
+        .WriteTo.File("logs/purchasedtab-diagnostics.txt", rollingInterval: RollingInterval.Day, retainedFileCountLimit: 7)
+        .CreateLogger();
+    private readonly int maxLogBodyLength;
+    private readonly bool enableRawJsonBodyLogs;
 
     static APIHelper()
     {
@@ -47,6 +57,19 @@ public class APIHelper : IAPIHelper
         this.auth = auth;
         m_DBHelper = new DBHelper(downloadConfig);
         this.downloadConfig = downloadConfig;
+        maxLogBodyLength = downloadConfig.MaxLogBodyLength > 0 ? downloadConfig.MaxLogBodyLength : 2000;
+        enableRawJsonBodyLogs = downloadConfig.EnableJsonResponseBodyRawLogs;
+        if (downloadConfig.ApiRateLimitRequests > 0)
+        {
+            ApiRateLimitCount = downloadConfig.ApiRateLimitRequests;
+        }
+        if (downloadConfig.ApiRateLimitWindowSeconds > 0)
+        {
+            ApiRateLimitWindow = TimeSpan.FromSeconds(downloadConfig.ApiRateLimitWindowSeconds);
+        }
+
+        Log.Debug("DiagnosticsEnabled={DiagnosticsEnabled} MaxLogBodyLength={MaxLogBodyLength} EnableRawJson={EnableRawJson} RawLogDir={RawDir}",
+            DiagnosticsEnabled, maxLogBodyLength, enableRawJsonBodyLogs, Path.Combine("logs", "diagnostics", "raw"));
     }
 
 
@@ -118,16 +141,28 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, ILogger? diagnosticLogger = null, string? rawLabel = null)
     {
         Log.Debug("Calling BuildHeaderAndExecuteRequests");
 
         HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
+        diagnosticLogger?.Information("Diag request | method={Method} url={Url} headers={Headers}", request.Method, request.RequestUri, FlattenHeaders(request.Headers));
+
+        await EnforceApiRateLimitAsync(diagnosticLogger, request.RequestUri?.ToString());
         using var response = await client.SendAsync(request);
+        if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
+        {
+            string respBody = await response.Content.ReadAsStringAsync();
+            diagnosticLogger?.Warning("API 429 | url={Url} headers={Headers} respHeaders={RespHeaders} bodyPreview={Body}", request.RequestUri, FlattenHeaders(request.Headers), string.Join("; ", response.Headers.Select(h => $"{h.Key}={string.Join(",", h.Value)}")), TruncateForLog(respBody));
+            Log.Warning("API 429 | url={Url}", request.RequestUri);
+        }
+        diagnosticLogger?.Information("Diag response | status={StatusCode} reason={ReasonPhrase} requestUrl={Url}", response.StatusCode, response.ReasonPhrase, request.RequestUri);
         response.EnsureSuccessStatusCode();
         string body = await response.Content.ReadAsStringAsync();
 
         Log.Debug(body);
+        diagnosticLogger?.Information("Diag response body (truncated) | length={Length} body={Body}", body?.Length ?? 0, TruncateForLog(body));
+        WriteRawBody(rawLabel ?? request.RequestUri?.ToString() ?? endpoint, body);
 
         return body;
     }
@@ -177,6 +212,120 @@ public class APIHelper : IAPIHelper
         return client;
     }
 
+    private static string FlattenHeaders(System.Net.Http.Headers.HttpRequestHeaders headers)
+    {
+        return string.Join("; ", headers.Select(h => $"{h.Key}={string.Join(",", h.Value)}"));
+    }
+
+    private bool DiagnosticsEnabled => downloadConfig.LoggingLevel == LoggingLevel.Verbose;
+
+    private ILogger? CurrentDiagnosticLogger => DiagnosticsEnabled ? DiagnosticLog : null;
+
+    private static string TruncateString(string? value, int maxLength = 1024)
+    {
+        if (string.IsNullOrEmpty(value))
+        {
+            return string.Empty;
+        }
+        return value.Length <= maxLength ? value : $"{value[..maxLength]}...[truncated]";
+    }
+
+    private string TruncateForLog(string? value, int? overrideMax = null)
+    {
+        int max = overrideMax.HasValue ? Math.Min(overrideMax.Value, maxLogBodyLength) : maxLogBodyLength;
+        return TruncateString(value, max);
+    }
+
+    private void DiagInfo(string messageTemplate, params object[] propertyValues)
+    {
+        if (!DiagnosticsEnabled)
+        {
+            return;
+        }
+        DiagnosticLog.Information(messageTemplate, propertyValues);
+    }
+
+    private void DiagWarning(string messageTemplate, params object[] propertyValues)
+    {
+        if (!DiagnosticsEnabled)
+        {
+            return;
+        }
+        DiagnosticLog.Warning(messageTemplate, propertyValues);
+    }
+
+    private string SanitizeForFile(string input)
+    {
+        var invalids = Path.GetInvalidFileNameChars();
+        var safe = string.Concat(input.Select(ch => invalids.Contains(ch) ? '_' : ch));
+        return string.IsNullOrWhiteSpace(safe) ? "response" : safe;
+    }
+
+    private void WriteRawBody(string? label, string? body)
+    {
+        if (!enableRawJsonBodyLogs || string.IsNullOrEmpty(body))
+        {
+            return;
+        }
+        try
+        {
+            string dir = Path.Combine("logs", "diagnostics", "raw");
+            Directory.CreateDirectory(dir);
+            string namePart = !string.IsNullOrEmpty(label) ? SanitizeForFile(label) : "response";
+            string fileName = $"{DateTime.UtcNow:yyyyMMddTHHmmssfff}-{namePart}.json";
+            string fullPath = Path.Combine(dir, fileName);
+            File.WriteAllText(fullPath, body);
+            Log.Debug("Wrote raw body log to {Path}", fullPath);
+        }
+        catch (Exception ex)
+        {
+            Log.Debug("Failed to write raw body log: {Message}", ex.Message);
+        }
+    }
+
+    private static async Task EnforceApiRateLimitAsync(ILogger? diagnosticLogger, string? context = null)
+    {
+        if (ApiRateLimitCount <= 0 || ApiRateLimitWindow.TotalSeconds <= 0)
+        {
+            return;
+        }
+
+        while (true)
+        {
+            await ApiRateLimiter.WaitAsync();
+            double waitSeconds = 0;
+            try
+            {
+                var now = DateTime.UtcNow;
+                while (ApiRequestTimes.Count > 0 && (now - ApiRequestTimes.Peek()) > ApiRateLimitWindow)
+                {
+                    ApiRequestTimes.Dequeue();
+                }
+
+                if (ApiRequestTimes.Count < ApiRateLimitCount)
+                {
+                    ApiRequestTimes.Enqueue(now);
+                    return;
+                }
+
+                waitSeconds = (ApiRateLimitWindow - (now - ApiRequestTimes.Peek())).TotalSeconds;
+            }
+            finally
+            {
+                ApiRateLimiter.Release();
+            }
+
+            if (waitSeconds < 0.1)
+            {
+                waitSeconds = 0.1;
+            }
+
+            diagnosticLogger?.Information("API rate limit sleep | waitSeconds={WaitSeconds} currentCount={CurrentCount} context={Context}", waitSeconds, ApiRequestTimes.Count, context ?? string.Empty);
+            Log.Information("API rate limit sleep | waitSeconds={WaitSeconds} currentCount={CurrentCount} context={Context}", waitSeconds, ApiRequestTimes.Count, context ?? string.Empty);
+            await Task.Delay(TimeSpan.FromSeconds(waitSeconds));
+        }
+    }
+
 
     /// <summary>
     /// this one is used during initialization only
@@ -232,6 +381,7 @@ public class APIHelper : IAPIHelper
             HttpClient client = new();
             HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
 
+            await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
             using var response = await client.SendAsync(request);
 
             if (!response.IsSuccessStatusCode)
@@ -265,6 +415,7 @@ public class APIHelper : IAPIHelper
             HttpClient client = new();
             HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
 
+            await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
             using var response = await client.SendAsync(request);
 
             response.EnsureSuccessStatusCode();
@@ -661,6 +812,7 @@ public class APIHelper : IAPIHelper
                         highlight_request.Headers.Add(keyValuePair.Key, keyValuePair.Value);
                     }
 
+                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger);
                     using var highlightResponse = await highlight_client.SendAsync(highlight_request);
                     highlightResponse.EnsureSuccessStatusCode();
                     var highlightBody = await highlightResponse.Content.ReadAsStringAsync();
@@ -1873,6 +2025,7 @@ public class APIHelper : IAPIHelper
                     {
                         looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
                     }
+                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString());
                     using (var loopresponse = await loopclient.SendAsync(looprequest))
                     {
                         loopresponse.EnsureSuccessStatusCode();
@@ -2102,6 +2255,7 @@ public class APIHelper : IAPIHelper
                     {
                         looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
                     }
+                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger);
                     using (var loopresponse = await loopclient.SendAsync(looprequest))
                     {
                         loopresponse.EnsureSuccessStatusCode();
@@ -2244,6 +2398,8 @@ public class APIHelper : IAPIHelper
     {
         Log.Debug($"Calling GetPurchasedTab - {endpoint}");
 
+        string diagContext = $"purchasedtab-{DateTime.UtcNow:yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}";
+
         try
         {
             Dictionary<long, List<Purchased.List>> userPurchases = new Dictionary<long, List<Purchased.List>>();
@@ -2258,8 +2414,18 @@ public class APIHelper : IAPIHelper
                 { "skip_users", "all" }
             };
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            string initialQueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
+            var initialHeaders = GetDynamicHeaders("/api2/v2" + endpoint, initialQueryParams);
+            DiagInfo("PurchasedTab start | ctx={Context} endpoint={Endpoint} folder={Folder} usersCount={UsersCount} limit={Limit} query={Query} headers={Headers}", diagContext, endpoint, folder, users?.Count ?? 0, post_limit, initialQueryParams, string.Join(", ", initialHeaders.Select(kvp => $"{kvp.Key}={kvp.Value}")));
+
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, diagContext);
+            DiagInfo("PurchasedTab initial response | ctx={Context} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, body?.Length ?? 0, TruncateForLog(body));
             purchased = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
+            DiagInfo("PurchasedTab parsed initial | ctx={Context} items={Items} hasMore={HasMore}", diagContext, purchased?.list?.Count ?? 0, purchased?.hasMore ?? false);
+            if (purchased == null || purchased.list == null)
+            {
+                DiagWarning("PurchasedTab parsed initial null list | ctx={Context} bodyPreview={BodyPreview}", diagContext, TruncateForLog(body));
+            }
             if (purchased != null && purchased.hasMore)
             {
                 getParams["offset"] = purchased.list.Count.ToString();
@@ -2276,11 +2442,16 @@ public class APIHelper : IAPIHelper
                     {
                         looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
                     }
+                    DiagInfo("PurchasedTab page request | ctx={Context} url={Url} offset={Offset} limit={Limit} headers={Headers}", diagContext, looprequest.RequestUri, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit, FlattenHeaders(looprequest.Headers));
+                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString());
                     using (var loopresponse = await loopclient.SendAsync(looprequest))
                     {
+                        DiagInfo("PurchasedTab page response | ctx={Context} status={Status} reason={Reason} url={Url}", diagContext, loopresponse.StatusCode, loopresponse.ReasonPhrase, looprequest.RequestUri);
                         loopresponse.EnsureSuccessStatusCode();
                         var loopbody = await loopresponse.Content.ReadAsStringAsync();
                         newPurchased = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
+                        DiagInfo("PurchasedTab page parsed | ctx={Context} offset={Offset} items={Items} hasMore={HasMore} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", newPurchased?.list?.Count ?? 0, newPurchased?.hasMore ?? false, loopbody?.Length ?? 0, TruncateForLog(loopbody));
+                        WriteRawBody($"{diagContext}-page-{getParams["offset"]}", loopbody);
                     }
                     purchased.list.AddRange(newPurchased.list);
                     if (!newPurchased.hasMore)
@@ -2318,6 +2489,17 @@ public class APIHelper : IAPIHelper
             {
                 PurchasedTabCollection purchasedTabCollection = new PurchasedTabCollection();
                 JObject userObject = await GetUserInfoById($"/users/list?x[]={user.Key}");
+                string resolvedUsername = $"Deleted User - {user.Key}";
+                if (userObject == null || userObject[user.Key.ToString()] == null || userObject[user.Key.ToString()]["username"] == null)
+                {
+                    DiagWarning("PurchasedTab user lookup missing username | ctx={Context} userId={UserId} rawUser={RawUser}", diagContext, user.Key, userObject != null ? TruncateForLog(userObject.ToString()) : "null");
+                }
+                else
+                {
+                    resolvedUsername = userObject[user.Key.ToString()]["username"].ToString();
+                    DiagInfo("PurchasedTab user lookup success | ctx={Context} userId={UserId} username={Username} rawUser={RawUser}", diagContext, user.Key, resolvedUsername, TruncateForLog(userObject.ToString()));
+                }
+                WriteRawBody($"{diagContext}-user-{user.Key}", userObject?.ToString());
                 purchasedTabCollection.UserId = user.Key;
                 purchasedTabCollection.Username = userObject is not null && !string.IsNullOrEmpty(userObject[user.Key.ToString()]["username"].ToString()) ? userObject[user.Key.ToString()]["username"].ToString() : $"Deleted User - {user.Key}";
                 string path = System.IO.Path.Combine(folder, purchasedTabCollection.Username);
@@ -2325,6 +2507,16 @@ public class APIHelper : IAPIHelper
                 {
                     foreach (Purchased.List purchase in user.Value)
                     {
+                        DiagInfo("PurchasedTab processing purchase | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} mediaNull={MediaNull} mediaCount={MediaCount} postedAt={PostedAt} createdAt={CreatedAt} fromUserId={FromUserId} authorId={AuthorId}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.media == null, purchase.media?.Count ?? 0, purchase.postedAt, purchase.createdAt, purchase.fromUser?.id, purchase.author?.id);
+                        DiagInfo("PurchasedTab purchase raw | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} rawPurchase={RawPurchase}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, TruncateForLog(JsonConvert.SerializeObject(purchase)));
+                        WriteRawBody($"{diagContext}-purchase-{purchase.id}", JsonConvert.SerializeObject(purchase));
+                        if (purchase.media == null)
+                        {
+                            string textPreview = TruncateForLog(purchase.text, 100);
+                            DiagWarning("PurchasedTab purchase media null, setting empty list | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt} textPreview={TextPreview}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.createdAt, purchase.postedAt, textPreview);
+                            Log.Warning("PurchasedTab purchase media null, setting empty list | userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt} textPreview={TextPreview}", user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.createdAt, purchase.postedAt, textPreview);
+                            purchase.media = new List<Messages.Medium>();
+                        }
                         switch (purchase.responseType)
                         {
                             case "post":
@@ -2608,6 +2800,7 @@ public class APIHelper : IAPIHelper
             request.Headers.Add("user-agent", auth.USER_AGENT);
             request.Headers.Add("Accept", "*/*");
             request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.COOKIE};");
+            await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
             using (var response = await client.SendAsync(request))
             {
                 response.EnsureSuccessStatusCode();
@@ -2653,6 +2846,7 @@ public class APIHelper : IAPIHelper
             request.Headers.Add("user-agent", auth.USER_AGENT);
             request.Headers.Add("Accept", "*/*");
             request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.COOKIE};");
+            await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
             using (var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead))
             {
                 response.EnsureSuccessStatusCode();
@@ -2709,6 +2903,7 @@ public class APIHelper : IAPIHelper
                     Content = new StringContent(json, Encoding.UTF8, "application/json")
                 };
 
+                await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
                 using var response = await client.SendAsync(request);
 
                 Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}");
@@ -2788,6 +2983,7 @@ public class APIHelper : IAPIHelper
                     Content = new StringContent(json, Encoding.UTF8, "application/json")
                 };
 
+                await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
                 using var response = await client.SendAsync(request);
 
                 if (!response.IsSuccessStatusCode)

OF DL/Program.cs

@@ -35,6 +35,32 @@ public class Program
 	private static Auth? auth = null;
 	private static LoggingLevelSwitch levelSwitch = new LoggingLevelSwitch();
 
+    private static int SafeGetInt(Akka.Configuration.Config hoconConfig, string path, int defaultValue)
+    {
+        try
+        {
+            var value = hoconConfig.GetString(path);
+            return string.IsNullOrWhiteSpace(value) ? defaultValue : hoconConfig.GetInt(path);
+        }
+        catch
+        {
+            return defaultValue;
+        }
+    }
+
+    private static bool SafeGetBool(Akka.Configuration.Config hoconConfig, string path, bool defaultValue)
+    {
+        try
+        {
+            var value = hoconConfig.GetString(path);
+            return string.IsNullOrWhiteSpace(value) ? defaultValue : hoconConfig.GetBoolean(path);
+        }
+        catch
+        {
+            return defaultValue;
+        }
+    }
+
 	private static async Task LoadAuthFromBrowser()
 	{
 		bool runningInDocker = Environment.GetEnvironmentVariable("OFDL_DOCKER") != null;
@@ -219,11 +245,15 @@ public class Program
 						hoconConfig.AppendLine($"  Timeout = {(jsonConfig.Timeout.HasValue ? jsonConfig.Timeout.Value : -1)}");
                         hoconConfig.AppendLine($"  LimitDownloadRate = {jsonConfig.LimitDownloadRate.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadLimitInMbPerSec = {jsonConfig.DownloadLimitInMbPerSec}");
+                        hoconConfig.AppendLine($"  ApiRateLimitRequests = {jsonConfig.ApiRateLimitRequests}");
+                        hoconConfig.AppendLine($"  ApiRateLimitWindowSeconds = {jsonConfig.ApiRateLimitWindowSeconds}");
                         hoconConfig.AppendLine("}");
 
                         hoconConfig.AppendLine("# Logging/Debug Settings");
                         hoconConfig.AppendLine("Logging {");
                         hoconConfig.AppendLine($"  LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\"");
+                        hoconConfig.AppendLine($"  MaxLogBodyLength = {jsonConfig.MaxLogBodyLength}");
+                        hoconConfig.AppendLine($"  EnableJsonResponseBodyRawLogs = {jsonConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
                         hoconConfig.AppendLine("}");
 
                         File.WriteAllText("config.conf", hoconConfig.ToString());
@@ -318,9 +348,13 @@ public class Program
                         Timeout = string.IsNullOrWhiteSpace(hoconConfig.GetString("Performance.Timeout")) ? -1 : hoconConfig.GetInt("Performance.Timeout"),
                         LimitDownloadRate = hoconConfig.GetBoolean("Performance.LimitDownloadRate"),
 						DownloadLimitInMbPerSec = hoconConfig.GetInt("Performance.DownloadLimitInMbPerSec"),
+                        ApiRateLimitRequests = SafeGetInt(hoconConfig, "Performance.ApiRateLimitRequests", 10),
+                        ApiRateLimitWindowSeconds = SafeGetInt(hoconConfig, "Performance.ApiRateLimitWindowSeconds", 10),
 
 						// Logging/Debug Settings
-						LoggingLevel = Enum.Parse<LoggingLevel>(hoconConfig.GetString("Logging.LoggingLevel"), true)
+						LoggingLevel = Enum.Parse<LoggingLevel>(hoconConfig.GetString("Logging.LoggingLevel"), true),
+                        MaxLogBodyLength = SafeGetInt(hoconConfig, "Logging.MaxLogBodyLength", 2000),
+                        EnableJsonResponseBodyRawLogs = SafeGetBool(hoconConfig, "Logging.EnableJsonResponseBodyRawLogs", false)
 					};
 
                     ValidateFileNameFormat(config.PaidPostFileNameFormat, "PaidPostFileNameFormat");
@@ -467,11 +501,15 @@ public class Program
                 hoconConfig.AppendLine($"  Timeout = {(jsonConfig.Timeout.HasValue ? jsonConfig.Timeout.Value : -1)}");
                 hoconConfig.AppendLine($"  LimitDownloadRate = {jsonConfig.LimitDownloadRate.ToString().ToLower()}");
                 hoconConfig.AppendLine($"  DownloadLimitInMbPerSec = {jsonConfig.DownloadLimitInMbPerSec}");
+                hoconConfig.AppendLine($"  ApiRateLimitRequests = {jsonConfig.ApiRateLimitRequests}");
+                hoconConfig.AppendLine($"  ApiRateLimitWindowSeconds = {jsonConfig.ApiRateLimitWindowSeconds}");
                 hoconConfig.AppendLine("}");
 
                 hoconConfig.AppendLine("# Logging/Debug Settings");
                 hoconConfig.AppendLine("Logging {");
                 hoconConfig.AppendLine($"  LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\"");
+                hoconConfig.AppendLine($"  MaxLogBodyLength = {jsonConfig.MaxLogBodyLength}");
+                hoconConfig.AppendLine($"  EnableJsonResponseBodyRawLogs = {jsonConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
                 hoconConfig.AppendLine("}");
 
                 File.WriteAllText("config.conf", hoconConfig.ToString());
@@ -3043,11 +3081,15 @@ public class Program
                         hoconConfig.AppendLine($"  Timeout = {(newConfig.Timeout.HasValue ? newConfig.Timeout.Value : -1)}");
                         hoconConfig.AppendLine($"  LimitDownloadRate = {newConfig.LimitDownloadRate.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadLimitInMbPerSec = {newConfig.DownloadLimitInMbPerSec}");
+                        hoconConfig.AppendLine($"  ApiRateLimitRequests = {newConfig.ApiRateLimitRequests}");
+                        hoconConfig.AppendLine($"  ApiRateLimitWindowSeconds = {newConfig.ApiRateLimitWindowSeconds}");
                         hoconConfig.AppendLine("}");
 
                         hoconConfig.AppendLine("# Logging/Debug Settings");
                         hoconConfig.AppendLine("Logging {");
                         hoconConfig.AppendLine($"  LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\"");
+                        hoconConfig.AppendLine($"  MaxLogBodyLength = {newConfig.MaxLogBodyLength}");
+                        hoconConfig.AppendLine($"  EnableJsonResponseBodyRawLogs = {newConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
                         hoconConfig.AppendLine("}");
 
                         File.WriteAllText("config.conf", hoconConfig.ToString());
@@ -3205,11 +3247,15 @@ public class Program
                         hoconConfig.AppendLine($"  Timeout = {(newConfig.Timeout.HasValue ? newConfig.Timeout.Value : -1)}");
                         hoconConfig.AppendLine($"  LimitDownloadRate = {newConfig.LimitDownloadRate.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadLimitInMbPerSec = {newConfig.DownloadLimitInMbPerSec}");
+                        hoconConfig.AppendLine($"  ApiRateLimitRequests = {newConfig.ApiRateLimitRequests}");
+                        hoconConfig.AppendLine($"  ApiRateLimitWindowSeconds = {newConfig.ApiRateLimitWindowSeconds}");
                         hoconConfig.AppendLine("}");
 
                         hoconConfig.AppendLine("# Logging/Debug Settings");
                         hoconConfig.AppendLine("Logging {");
                         hoconConfig.AppendLine($"  LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\"");
+                        hoconConfig.AppendLine($"  MaxLogBodyLength = {newConfig.MaxLogBodyLength}");
+                        hoconConfig.AppendLine($"  EnableJsonResponseBodyRawLogs = {newConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
                         hoconConfig.AppendLine("}");
 
                         File.WriteAllText("config.conf", hoconConfig.ToString());