Adding ffmpeg debug reporting config

.gitea/workflows/publish-release.yml

@@ -52,9 +52,20 @@ jobs:
           echo "➤ Creating folder for CDM"
           mkdir -p cdm/devices/chrome_1610
 
-          echo "➤ Copying ffmpeg from user folder"
+          echo "➤ Copying ffmpeg"
-          cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
+          if [ -f /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe ] && [ -f /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE ]; then
-          cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
+            cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
+            cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
+          else
+            echo "➤ Downloading ffmpeg package"
+            mkdir -p /tmp/ffmpeg
+            if [ ! -f /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build.zip ]; then
+              curl -L https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-7.1.1-essentials_build.zip -o /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build.zip
+            fi
+            unzip -o /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build.zip -d /tmp/ffmpeg
+            cp /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
+            cp /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
+          fi
 
           echo "➤ Creating release zip"
           zip ../OFDLV${{ steps.version.outputs.version }}.zip OF\ DL.exe e_sqlite3.dll rules.json config.conf cdm ffmpeg.exe LICENSE.ffmpeg
@@ -70,4 +81,4 @@ jobs:
         env:
           GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
           GITEA_REPOSITORY: ${{ gitea.repository }}
-          GITEA_SERVER_URL: ${{ gitea.server_url }}
+          GITEA_SERVER_URL: ${{ gitea.server_url }}

Dockerfile

@@ -1,6 +1,6 @@
-FROM alpine:3.20 AS build
+FROM alpine:3.23 AS build
 
-ARG VERSION
+ARG VERSION=dev
 
 RUN apk --no-cache --repository community add \
       dotnet8-sdk
@@ -21,14 +21,14 @@ RUN /src/out/OF\ DL --non-interactive || true && \
       mv /src/updated_config.conf /src/config.conf
 
 
-FROM alpine:3.20 AS final
+      FROM alpine:3.23 AS final
 
 # Install dependencies
 RUN apk --no-cache --repository community add \
       bash \
       tini \
       dotnet8-runtime \
-      ffmpeg \
+      ffmpeg7 \
       udev \
       ttf-freefont \
       chromium \

OF DL/Entities/Config.cs

@@ -57,6 +57,13 @@ namespace OF_DL.Entities
         [ToggleableConfig]
         public bool LimitDownloadRate { get; set; } = false;
         public int DownloadLimitInMbPerSec { get; set; } = 4;
+        public int ApiRateLimitRequests { get; set; } = 10;
+        public int ApiRateLimitWindowSeconds { get; set; } = 10;
+        public int MaxLogBodyLength { get; set; } = 2000;
+        public bool EnableJsonResponseBodyRawLogs { get; set; } = false;
+        [ToggleableConfig]
+        public bool EnableFfmpegReport { get; set; } = true;
+        public string FfmpegReportPath { get; set; } = Path.Combine("logs", "diagnostics", "ffmpeg");
 
         // Indicates if you want to download only on specific dates.
         [ToggleableConfig]

OF DL/Entities/IDownloadConfig.cs

@@ -51,6 +51,15 @@ namespace OF_DL.Entities
         public LoggingLevel LoggingLevel { get; set; }
 
         bool IgnoreOwnMessages { get; set; }
+
+        int ApiRateLimitRequests { get; set; }
+        int ApiRateLimitWindowSeconds { get; set; }
+
+        int MaxLogBodyLength { get; set; }
+        bool EnableJsonResponseBodyRawLogs { get; set; }
+
+        bool EnableFfmpegReport { get; set; }
+        string FfmpegReportPath { get; set; }
     }
 
 }

OF DL/Helpers/APIHelper.cs

@@ -33,6 +33,16 @@ public class APIHelper : IAPIHelper
     private static DynamicRules? cachedDynamicRules;
     private const int MaxAttempts = 30;
     private const int DelayBetweenAttempts = 3000;
+    private static int ApiRateLimitCount = 10;
+    private static TimeSpan ApiRateLimitWindow = TimeSpan.FromSeconds(10);
+    private static readonly SemaphoreSlim ApiRateLimiter = new(1, 1);
+    private static readonly Queue<DateTime> ApiRequestTimes = new();
+    private static readonly ILogger DiagnosticLog = new LoggerConfiguration()
+        .MinimumLevel.Debug()
+        .WriteTo.File("logs/purchasedtab-diagnostics.txt", rollingInterval: RollingInterval.Day, retainedFileCountLimit: 7)
+        .CreateLogger();
+    private readonly int maxLogBodyLength;
+    private readonly bool enableRawJsonBodyLogs;
 
     static APIHelper()
     {
@@ -47,6 +57,19 @@ public class APIHelper : IAPIHelper
         this.auth = auth;
         m_DBHelper = new DBHelper(downloadConfig);
         this.downloadConfig = downloadConfig;
+        maxLogBodyLength = downloadConfig.MaxLogBodyLength > 0 ? downloadConfig.MaxLogBodyLength : 2000;
+        enableRawJsonBodyLogs = downloadConfig.EnableJsonResponseBodyRawLogs;
+        if (downloadConfig.ApiRateLimitRequests > 0)
+        {
+            ApiRateLimitCount = downloadConfig.ApiRateLimitRequests;
+        }
+        if (downloadConfig.ApiRateLimitWindowSeconds > 0)
+        {
+            ApiRateLimitWindow = TimeSpan.FromSeconds(downloadConfig.ApiRateLimitWindowSeconds);
+        }
+
+        Log.Debug("DiagnosticsEnabled={DiagnosticsEnabled} MaxLogBodyLength={MaxLogBodyLength} EnableRawJson={EnableRawJson} RawLogDir={RawDir}",
+            DiagnosticsEnabled, maxLogBodyLength, enableRawJsonBodyLogs, Path.Combine("logs", "diagnostics", "raw"));
     }
 
 
@@ -118,16 +141,29 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, ILogger? diagnosticLogger = null, string? rawLabel = null)
     {
         Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        diagnosticLogger ??= CurrentDiagnosticLogger;
 
         HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
+        diagnosticLogger?.Information("Diag request | method={Method} url={Url} headers={Headers}", request.Method, request.RequestUri, FlattenHeaders(request.Headers));
+
+        await EnforceApiRateLimitAsync(diagnosticLogger, request.RequestUri?.ToString());
         using var response = await client.SendAsync(request);
+        if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
+        {
+            string respBody = await response.Content.ReadAsStringAsync();
+            diagnosticLogger?.Warning("API 429 | url={Url} headers={Headers} respHeaders={RespHeaders} bodyPreview={Body}", request.RequestUri, FlattenHeaders(request.Headers), string.Join("; ", response.Headers.Select(h => $"{h.Key}={string.Join(",", h.Value)}")), TruncateForLog(respBody));
+            Log.Warning("API 429 | url={Url}", request.RequestUri);
+        }
+        diagnosticLogger?.Information("Diag response | status={StatusCode} reason={ReasonPhrase} requestUrl={Url}", response.StatusCode, response.ReasonPhrase, request.RequestUri);
         response.EnsureSuccessStatusCode();
         string body = await response.Content.ReadAsStringAsync();
 
         Log.Debug(body);
+        diagnosticLogger?.Information("Diag response body (truncated) | length={Length} body={Body}", body?.Length ?? 0, TruncateForLog(body));
+        WriteRawBody(rawLabel ?? request.RequestUri?.ToString() ?? endpoint, body);
 
         return body;
     }
@@ -177,6 +213,120 @@ public class APIHelper : IAPIHelper
         return client;
     }
 
+    private static string FlattenHeaders(System.Net.Http.Headers.HttpRequestHeaders headers)
+    {
+        return string.Join("; ", headers.Select(h => $"{h.Key}={string.Join(",", h.Value)}"));
+    }
+
+    private bool DiagnosticsEnabled => downloadConfig.LoggingLevel == LoggingLevel.Verbose;
+
+    private ILogger? CurrentDiagnosticLogger => DiagnosticsEnabled ? DiagnosticLog : null;
+
+    private static string TruncateString(string? value, int maxLength = 1024)
+    {
+        if (string.IsNullOrEmpty(value))
+        {
+            return string.Empty;
+        }
+        return value.Length <= maxLength ? value : $"{value[..maxLength]}...[truncated]";
+    }
+
+    private string TruncateForLog(string? value, int? overrideMax = null)
+    {
+        int max = overrideMax.HasValue ? Math.Min(overrideMax.Value, maxLogBodyLength) : maxLogBodyLength;
+        return TruncateString(value, max);
+    }
+
+    private void DiagInfo(string messageTemplate, params object[] propertyValues)
+    {
+        if (!DiagnosticsEnabled)
+        {
+            return;
+        }
+        DiagnosticLog.Information(messageTemplate, propertyValues);
+    }
+
+    private void DiagWarning(string messageTemplate, params object[] propertyValues)
+    {
+        if (!DiagnosticsEnabled)
+        {
+            return;
+        }
+        DiagnosticLog.Warning(messageTemplate, propertyValues);
+    }
+
+    private string SanitizeForFile(string input)
+    {
+        var invalids = Path.GetInvalidFileNameChars();
+        var safe = string.Concat(input.Select(ch => invalids.Contains(ch) ? '_' : ch));
+        return string.IsNullOrWhiteSpace(safe) ? "response" : safe;
+    }
+
+    private void WriteRawBody(string? label, string? body)
+    {
+        if (!enableRawJsonBodyLogs || string.IsNullOrEmpty(body))
+        {
+            return;
+        }
+        try
+        {
+            string dir = Path.Combine("logs", "diagnostics", "raw");
+            Directory.CreateDirectory(dir);
+            string namePart = !string.IsNullOrEmpty(label) ? SanitizeForFile(label) : "response";
+            string fileName = $"{DateTime.UtcNow:yyyyMMddTHHmmssfff}-{namePart}.json";
+            string fullPath = Path.Combine(dir, fileName);
+            File.WriteAllText(fullPath, body);
+            Log.Debug("Wrote raw body log to {Path}", fullPath);
+        }
+        catch (Exception ex)
+        {
+            Log.Debug("Failed to write raw body log: {Message}", ex.Message);
+        }
+    }
+
+    private static async Task EnforceApiRateLimitAsync(ILogger? diagnosticLogger, string? context = null)
+    {
+        if (ApiRateLimitCount <= 0 || ApiRateLimitWindow.TotalSeconds <= 0)
+        {
+            return;
+        }
+
+        while (true)
+        {
+            await ApiRateLimiter.WaitAsync();
+            double waitSeconds = 0;
+            try
+            {
+                var now = DateTime.UtcNow;
+                while (ApiRequestTimes.Count > 0 && (now - ApiRequestTimes.Peek()) > ApiRateLimitWindow)
+                {
+                    ApiRequestTimes.Dequeue();
+                }
+
+                if (ApiRequestTimes.Count < ApiRateLimitCount)
+                {
+                    ApiRequestTimes.Enqueue(now);
+                    return;
+                }
+
+                waitSeconds = (ApiRateLimitWindow - (now - ApiRequestTimes.Peek())).TotalSeconds;
+            }
+            finally
+            {
+                ApiRateLimiter.Release();
+            }
+
+            if (waitSeconds < 0.1)
+            {
+                waitSeconds = 0.1;
+            }
+
+            diagnosticLogger?.Information("API rate limit sleep | waitSeconds={WaitSeconds} currentCount={CurrentCount} context={Context}", waitSeconds, ApiRequestTimes.Count, context ?? string.Empty);
+            Log.Information("API rate limit sleep | waitSeconds={WaitSeconds} currentCount={CurrentCount} context={Context}", waitSeconds, ApiRequestTimes.Count, context ?? string.Empty);
+            await Task.Delay(TimeSpan.FromSeconds(waitSeconds));
+        }
+    }
+
 
     /// <summary>
     /// this one is used during initialization only
@@ -232,6 +382,7 @@ public class APIHelper : IAPIHelper
             HttpClient client = new();
             HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
 
+            await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
             using var response = await client.SendAsync(request);
 
             if (!response.IsSuccessStatusCode)
@@ -241,6 +392,7 @@ public class APIHelper : IAPIHelper
 
             response.EnsureSuccessStatusCode();
             var body = await response.Content.ReadAsStringAsync();
+            WriteRawBody($"user-{endpoint.Replace("/", "_")}", body);
             user = JsonConvert.DeserializeObject<Entities.User>(body, m_JsonSerializerSettings);
             return user;
         }
@@ -258,22 +410,24 @@ public class APIHelper : IAPIHelper
         return null;
     }
 
-    public async Task<JObject> GetUserInfoById(string endpoint)
+        public async Task<JObject> GetUserInfoById(string endpoint)
-    {
-        try
         {
-            HttpClient client = new();
+            try
-            HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
+            {
+                HttpClient client = new();
+                HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
 
-            using var response = await client.SendAsync(request);
+                await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
+                using var response = await client.SendAsync(request);
 
-            response.EnsureSuccessStatusCode();
+                response.EnsureSuccessStatusCode();
-            var body = await response.Content.ReadAsStringAsync();
+                var body = await response.Content.ReadAsStringAsync();
+                WriteRawBody($"user-list-{endpoint.Replace("/", "_")}", body);
 
-            //if the content creator doesnt exist, we get a 200 response, but the content isnt usable
+                //if the content creator doesnt exist, we get a 200 response, but the content isnt usable
-            //so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
+                //so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
-            //which means we wont get loads of exceptions
+                //which means we wont get loads of exceptions
-            if (body.Equals("[]"))
+                if (body.Equals("[]"))
                 return null;
 
             JObject jObject = JObject.Parse(body);
@@ -625,10 +779,10 @@ public class APIHelper : IAPIHelper
                     {
                         Highlights newhighlights = new();
 
-                        Log.Debug("Media Highlights - " + endpoint);
+            Log.Debug("Media Highlights - " + endpoint);
 
-                        var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
-                        newhighlights = JsonConvert.DeserializeObject<Highlights>(loopbody, m_JsonSerializerSettings);
+            newhighlights = JsonConvert.DeserializeObject<Highlights>(loopbody, m_JsonSerializerSettings);
 
                         highlights.list.AddRange(newhighlights.list);
                         if (!newhighlights.hasMore)
@@ -661,9 +815,11 @@ public class APIHelper : IAPIHelper
                         highlight_request.Headers.Add(keyValuePair.Key, keyValuePair.Value);
                     }
 
+                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, highlight_request.RequestUri?.ToString());
                     using var highlightResponse = await highlight_client.SendAsync(highlight_request);
                     highlightResponse.EnsureSuccessStatusCode();
                     var highlightBody = await highlightResponse.Content.ReadAsStringAsync();
+                    WriteRawBody($"highlight-{highlight_id}", highlightBody);
                     highlightMedia = JsonConvert.DeserializeObject<HighlightMedia>(highlightBody, m_JsonSerializerSettings);
                     if (highlightMedia != null)
                     {
@@ -1852,7 +2008,7 @@ public class APIHelper : IAPIHelper
                 { "skip_users", "all" }
             };
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-initial-{username}");
             paidMessages = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
             ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.list.Count}[/]");
             ctx.Spinner(Spinner.Known.Dots);
@@ -1862,21 +2018,10 @@ public class APIHelper : IAPIHelper
                 getParams["offset"] = paidMessages.list.Count.ToString();
                 while (true)
                 {
-                    string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
                     Purchased newpaidMessages = new();
-                    Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
+                    var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-page-{getParams["offset"]}-{username}");
-                    HttpClient loopclient = GetHttpClient(config);
+                    if (!string.IsNullOrEmpty(loopbody))
-
-                    HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
-
-                    foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
                     {
-                        looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
-                    }
-                    using (var loopresponse = await loopclient.SendAsync(looprequest))
-                    {
-                        loopresponse.EnsureSuccessStatusCode();
-                        var loopbody = await loopresponse.Content.ReadAsStringAsync();
                         newpaidMessages = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
                     }
                     paidMessages.list.AddRange(newpaidMessages.list);
@@ -2084,28 +2229,18 @@ public class APIHelper : IAPIHelper
                 { "skip_users", "all" }
             };
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, "posts-paid-all-initial");
+            WriteRawBody("posts-paid-all-initial", body);
             purchased = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
             if (purchased != null && purchased.hasMore)
             {
                 getParams["offset"] = purchased.list.Count.ToString();
                 while (true)
                 {
-                    string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
                     Purchased newPurchased = new();
-                    Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
+                    var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"posts-paid-all-page-{getParams["offset"]}");
-                    HttpClient loopclient = GetHttpClient(config);
+                    if (!string.IsNullOrEmpty(loopbody))
-
-                    HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
-
-                    foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
                     {
-                        looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
-                    }
-                    using (var loopresponse = await loopclient.SendAsync(looprequest))
-                    {
-                        loopresponse.EnsureSuccessStatusCode();
-                        var loopbody = await loopresponse.Content.ReadAsStringAsync();
                         newPurchased = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
                     }
                     purchased.list.AddRange(newPurchased.list);
@@ -2244,6 +2379,8 @@ public class APIHelper : IAPIHelper
     {
         Log.Debug($"Calling GetPurchasedTab - {endpoint}");
 
+        string diagContext = $"purchasedtab-{DateTime.UtcNow:yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}";
+
         try
         {
             Dictionary<long, List<Purchased.List>> userPurchases = new Dictionary<long, List<Purchased.List>>();
@@ -2258,29 +2395,30 @@ public class APIHelper : IAPIHelper
                 { "skip_users", "all" }
             };
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            string initialQueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
+            var initialHeaders = GetDynamicHeaders("/api2/v2" + endpoint, initialQueryParams);
+            DiagInfo("PurchasedTab start | ctx={Context} endpoint={Endpoint} folder={Folder} usersCount={UsersCount} limit={Limit} query={Query} headers={Headers}", diagContext, endpoint, folder, users?.Count ?? 0, post_limit, initialQueryParams, string.Join(", ", initialHeaders.Select(kvp => $"{kvp.Key}={kvp.Value}")));
+
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, diagContext);
+            DiagInfo("PurchasedTab initial response | ctx={Context} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, body?.Length ?? 0, TruncateForLog(body));
             purchased = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
+            DiagInfo("PurchasedTab parsed initial | ctx={Context} items={Items} hasMore={HasMore}", diagContext, purchased?.list?.Count ?? 0, purchased?.hasMore ?? false);
+            if (purchased == null || purchased.list == null)
+            {
+                DiagWarning("PurchasedTab parsed initial null list | ctx={Context} bodyPreview={BodyPreview}", diagContext, TruncateForLog(body));
+            }
             if (purchased != null && purchased.hasMore)
             {
                 getParams["offset"] = purchased.list.Count.ToString();
                 while (true)
                 {
-                    string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
                     Purchased newPurchased = new();
-                    Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
+                    DiagInfo("PurchasedTab page request | ctx={Context} offset={Offset} limit={Limit}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit);
-                    HttpClient loopclient = GetHttpClient(config);
+                    var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"{diagContext}-page-{getParams["offset"]}");
-
+                    if (!string.IsNullOrEmpty(loopbody))
-                    HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
-
-                    foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
                     {
-                        looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
-                    }
-                    using (var loopresponse = await loopclient.SendAsync(looprequest))
-                    {
-                        loopresponse.EnsureSuccessStatusCode();
-                        var loopbody = await loopresponse.Content.ReadAsStringAsync();
                         newPurchased = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
+                        DiagInfo("PurchasedTab page parsed | ctx={Context} offset={Offset} items={Items} hasMore={HasMore} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", newPurchased?.list?.Count ?? 0, newPurchased?.hasMore ?? false, loopbody?.Length ?? 0, TruncateForLog(loopbody));
                     }
                     purchased.list.AddRange(newPurchased.list);
                     if (!newPurchased.hasMore)
@@ -2318,6 +2456,17 @@ public class APIHelper : IAPIHelper
             {
                 PurchasedTabCollection purchasedTabCollection = new PurchasedTabCollection();
                 JObject userObject = await GetUserInfoById($"/users/list?x[]={user.Key}");
+                string resolvedUsername = $"Deleted User - {user.Key}";
+                if (userObject == null || userObject[user.Key.ToString()] == null || userObject[user.Key.ToString()]["username"] == null)
+                {
+                    DiagWarning("PurchasedTab user lookup missing username | ctx={Context} userId={UserId} rawUser={RawUser}", diagContext, user.Key, userObject != null ? TruncateForLog(userObject.ToString()) : "null");
+                }
+                else
+                {
+                    resolvedUsername = userObject[user.Key.ToString()]["username"].ToString();
+                    DiagInfo("PurchasedTab user lookup success | ctx={Context} userId={UserId} username={Username} rawUser={RawUser}", diagContext, user.Key, resolvedUsername, TruncateForLog(userObject.ToString()));
+                }
+                WriteRawBody($"{diagContext}-user-{user.Key}", userObject?.ToString());
                 purchasedTabCollection.UserId = user.Key;
                 purchasedTabCollection.Username = userObject is not null && !string.IsNullOrEmpty(userObject[user.Key.ToString()]["username"].ToString()) ? userObject[user.Key.ToString()]["username"].ToString() : $"Deleted User - {user.Key}";
                 string path = System.IO.Path.Combine(folder, purchasedTabCollection.Username);
@@ -2325,6 +2474,16 @@ public class APIHelper : IAPIHelper
                 {
                     foreach (Purchased.List purchase in user.Value)
                     {
+                        DiagInfo("PurchasedTab processing purchase | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} mediaNull={MediaNull} mediaCount={MediaCount} postedAt={PostedAt} createdAt={CreatedAt} fromUserId={FromUserId} authorId={AuthorId}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.media == null, purchase.media?.Count ?? 0, purchase.postedAt, purchase.createdAt, purchase.fromUser?.id, purchase.author?.id);
+                        DiagInfo("PurchasedTab purchase raw | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} rawPurchase={RawPurchase}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, TruncateForLog(JsonConvert.SerializeObject(purchase)));
+                        WriteRawBody($"{diagContext}-purchase-{purchase.id}", JsonConvert.SerializeObject(purchase));
+                        if (purchase.media == null)
+                        {
+                            string textPreview = TruncateForLog(purchase.text, 100);
+                            DiagWarning("PurchasedTab purchase media null, setting empty list | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt} textPreview={TextPreview}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.createdAt, purchase.postedAt, textPreview);
+                            Log.Warning("PurchasedTab purchase media null, setting empty list | userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt} textPreview={TextPreview}", user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.createdAt, purchase.postedAt, textPreview);
+                            purchase.media = new List<Messages.Medium>();
+                        }
                         switch (purchase.responseType)
                         {
                             case "post":
@@ -2608,6 +2767,7 @@ public class APIHelper : IAPIHelper
             request.Headers.Add("user-agent", auth.USER_AGENT);
             request.Headers.Add("Accept", "*/*");
             request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.COOKIE};");
+            await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
             using (var response = await client.SendAsync(request))
             {
                 response.EnsureSuccessStatusCode();
@@ -2653,6 +2813,7 @@ public class APIHelper : IAPIHelper
             request.Headers.Add("user-agent", auth.USER_AGENT);
             request.Headers.Add("Accept", "*/*");
             request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.COOKIE};");
+            await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
             using (var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead))
             {
                 response.EnsureSuccessStatusCode();
@@ -2709,6 +2870,7 @@ public class APIHelper : IAPIHelper
                     Content = new StringContent(json, Encoding.UTF8, "application/json")
                 };
 
+                await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
                 using var response = await client.SendAsync(request);
 
                 Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}");
@@ -2788,6 +2950,7 @@ public class APIHelper : IAPIHelper
                     Content = new StringContent(json, Encoding.UTF8, "application/json")
                 };
 
+                await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
                 using var response = await client.SendAsync(request);
 
                 if (!response.IsSuccessStatusCode)

OF DL/Helpers/DownloadHelper.cs

@@ -632,8 +632,11 @@ public class DownloadHelper : IDownloadHelper
 
             // Configure ffmpeg log level and optional report file location
             bool ffmpegDebugLogging = Log.IsEnabled(Serilog.Events.LogEventLevel.Debug);
+            bool enableReport = downloadConfig.EnableFfmpegReport;
 
-            string logLevelArgs = ffmpegDebugLogging || downloadConfig.LoggingLevel is LoggingLevel.Verbose or LoggingLevel.Debug
+            bool includeReport = enableReport && (ffmpegDebugLogging || downloadConfig.LoggingLevel is LoggingLevel.Verbose or LoggingLevel.Debug);
+
+            string logLevelArgs = includeReport
                 ? "-loglevel debug -report"
                 : downloadConfig.LoggingLevel switch
                 {
@@ -644,10 +647,12 @@ public class DownloadHelper : IDownloadHelper
                     _ => string.Empty
                 };
 
-            if (logLevelArgs.Contains("-report", StringComparison.OrdinalIgnoreCase))
+            if (includeReport)
             {
-                // Direct ffmpeg report files into the same logs directory Serilog uses (relative to current working directory)
+                string reportDir = string.IsNullOrWhiteSpace(downloadConfig.FfmpegReportPath)
-                string logDir = Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, "logs"));
+                    ? Path.Combine("logs", "diagnostics", "ffmpeg")
+                    : downloadConfig.FfmpegReportPath;
+                string logDir = Path.GetFullPath(reportDir);
                 Directory.CreateDirectory(logDir);
                 string ffReportPath = Path.Combine(logDir, "ffmpeg-%p-%t.log"); // ffmpeg will replace %p/%t
                 Environment.SetEnvironmentVariable("FFREPORT", $"file={ffReportPath}:level=32");

OF DL/Program.cs

@@ -35,6 +35,32 @@ public class Program
 	private static Auth? auth = null;
 	private static LoggingLevelSwitch levelSwitch = new LoggingLevelSwitch();
 
+    private static int SafeGetInt(Akka.Configuration.Config hoconConfig, string path, int defaultValue)
+    {
+        try
+        {
+            var value = hoconConfig.GetString(path);
+            return string.IsNullOrWhiteSpace(value) ? defaultValue : hoconConfig.GetInt(path);
+        }
+        catch
+        {
+            return defaultValue;
+        }
+    }
+
+    private static bool SafeGetBool(Akka.Configuration.Config hoconConfig, string path, bool defaultValue)
+    {
+        try
+        {
+            var value = hoconConfig.GetString(path);
+            return string.IsNullOrWhiteSpace(value) ? defaultValue : hoconConfig.GetBoolean(path);
+        }
+        catch
+        {
+            return defaultValue;
+        }
+    }
+
 	private static async Task LoadAuthFromBrowser()
 	{
 		bool runningInDocker = Environment.GetEnvironmentVariable("OFDL_DOCKER") != null;
@@ -219,11 +245,15 @@ public class Program
 						hoconConfig.AppendLine($"  Timeout = {(jsonConfig.Timeout.HasValue ? jsonConfig.Timeout.Value : -1)}");
                         hoconConfig.AppendLine($"  LimitDownloadRate = {jsonConfig.LimitDownloadRate.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadLimitInMbPerSec = {jsonConfig.DownloadLimitInMbPerSec}");
+                        hoconConfig.AppendLine($"  ApiRateLimitRequests = {jsonConfig.ApiRateLimitRequests}");
+                        hoconConfig.AppendLine($"  ApiRateLimitWindowSeconds = {jsonConfig.ApiRateLimitWindowSeconds}");
                         hoconConfig.AppendLine("}");
 
                         hoconConfig.AppendLine("# Logging/Debug Settings");
                         hoconConfig.AppendLine("Logging {");
                         hoconConfig.AppendLine($"  LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\"");
+                        hoconConfig.AppendLine($"  MaxLogBodyLength = {jsonConfig.MaxLogBodyLength}");
+                        hoconConfig.AppendLine($"  EnableJsonResponseBodyRawLogs = {jsonConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
                         hoconConfig.AppendLine("}");
 
                         File.WriteAllText("config.conf", hoconConfig.ToString());
@@ -318,9 +348,13 @@ public class Program
                         Timeout = string.IsNullOrWhiteSpace(hoconConfig.GetString("Performance.Timeout")) ? -1 : hoconConfig.GetInt("Performance.Timeout"),
                         LimitDownloadRate = hoconConfig.GetBoolean("Performance.LimitDownloadRate"),
 						DownloadLimitInMbPerSec = hoconConfig.GetInt("Performance.DownloadLimitInMbPerSec"),
+                        ApiRateLimitRequests = SafeGetInt(hoconConfig, "Performance.ApiRateLimitRequests", 10),
+                        ApiRateLimitWindowSeconds = SafeGetInt(hoconConfig, "Performance.ApiRateLimitWindowSeconds", 10),
 
 						// Logging/Debug Settings
-						LoggingLevel = Enum.Parse<LoggingLevel>(hoconConfig.GetString("Logging.LoggingLevel"), true)
+						LoggingLevel = Enum.Parse<LoggingLevel>(hoconConfig.GetString("Logging.LoggingLevel"), true),
+                        MaxLogBodyLength = SafeGetInt(hoconConfig, "Logging.MaxLogBodyLength", 2000),
+                        EnableJsonResponseBodyRawLogs = SafeGetBool(hoconConfig, "Logging.EnableJsonResponseBodyRawLogs", false)
 					};
 
                     ValidateFileNameFormat(config.PaidPostFileNameFormat, "PaidPostFileNameFormat");
@@ -467,11 +501,15 @@ public class Program
                 hoconConfig.AppendLine($"  Timeout = {(jsonConfig.Timeout.HasValue ? jsonConfig.Timeout.Value : -1)}");
                 hoconConfig.AppendLine($"  LimitDownloadRate = {jsonConfig.LimitDownloadRate.ToString().ToLower()}");
                 hoconConfig.AppendLine($"  DownloadLimitInMbPerSec = {jsonConfig.DownloadLimitInMbPerSec}");
+                hoconConfig.AppendLine($"  ApiRateLimitRequests = {jsonConfig.ApiRateLimitRequests}");
+                hoconConfig.AppendLine($"  ApiRateLimitWindowSeconds = {jsonConfig.ApiRateLimitWindowSeconds}");
                 hoconConfig.AppendLine("}");
 
                 hoconConfig.AppendLine("# Logging/Debug Settings");
                 hoconConfig.AppendLine("Logging {");
                 hoconConfig.AppendLine($"  LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\"");
+                hoconConfig.AppendLine($"  MaxLogBodyLength = {jsonConfig.MaxLogBodyLength}");
+                hoconConfig.AppendLine($"  EnableJsonResponseBodyRawLogs = {jsonConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
                 hoconConfig.AppendLine("}");
 
                 File.WriteAllText("config.conf", hoconConfig.ToString());
@@ -3043,11 +3081,15 @@ public class Program
                         hoconConfig.AppendLine($"  Timeout = {(newConfig.Timeout.HasValue ? newConfig.Timeout.Value : -1)}");
                         hoconConfig.AppendLine($"  LimitDownloadRate = {newConfig.LimitDownloadRate.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadLimitInMbPerSec = {newConfig.DownloadLimitInMbPerSec}");
+                        hoconConfig.AppendLine($"  ApiRateLimitRequests = {newConfig.ApiRateLimitRequests}");
+                        hoconConfig.AppendLine($"  ApiRateLimitWindowSeconds = {newConfig.ApiRateLimitWindowSeconds}");
                         hoconConfig.AppendLine("}");
 
                         hoconConfig.AppendLine("# Logging/Debug Settings");
                         hoconConfig.AppendLine("Logging {");
                         hoconConfig.AppendLine($"  LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\"");
+                        hoconConfig.AppendLine($"  MaxLogBodyLength = {newConfig.MaxLogBodyLength}");
+                        hoconConfig.AppendLine($"  EnableJsonResponseBodyRawLogs = {newConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
                         hoconConfig.AppendLine("}");
 
                         File.WriteAllText("config.conf", hoconConfig.ToString());
@@ -3205,11 +3247,15 @@ public class Program
                         hoconConfig.AppendLine($"  Timeout = {(newConfig.Timeout.HasValue ? newConfig.Timeout.Value : -1)}");
                         hoconConfig.AppendLine($"  LimitDownloadRate = {newConfig.LimitDownloadRate.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadLimitInMbPerSec = {newConfig.DownloadLimitInMbPerSec}");
+                        hoconConfig.AppendLine($"  ApiRateLimitRequests = {newConfig.ApiRateLimitRequests}");
+                        hoconConfig.AppendLine($"  ApiRateLimitWindowSeconds = {newConfig.ApiRateLimitWindowSeconds}");
                         hoconConfig.AppendLine("}");
 
                         hoconConfig.AppendLine("# Logging/Debug Settings");
                         hoconConfig.AppendLine("Logging {");
                         hoconConfig.AppendLine($"  LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\"");
+                        hoconConfig.AppendLine($"  MaxLogBodyLength = {newConfig.MaxLogBodyLength}");
+                        hoconConfig.AppendLine($"  EnableJsonResponseBodyRawLogs = {newConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
                         hoconConfig.AppendLine("}");
 
                         File.WriteAllText("config.conf", hoconConfig.ToString());