Ensure all response bodies logged and refactoring pagination loops

OF DL/Helpers/APIHelper.cs

@@ -144,6 +144,7 @@ public class APIHelper : IAPIHelper
     private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, ILogger? diagnosticLogger = null, string? rawLabel = null)
     {
         Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        diagnosticLogger ??= CurrentDiagnosticLogger;
 
         HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
         diagnosticLogger?.Information("Diag request | method={Method} url={Url} headers={Headers}", request.Method, request.RequestUri, FlattenHeaders(request.Headers));
@@ -391,6 +392,7 @@ public class APIHelper : IAPIHelper
 
             response.EnsureSuccessStatusCode();
             var body = await response.Content.ReadAsStringAsync();
+            WriteRawBody($"user-{endpoint.Replace("/", "_")}", body);
             user = JsonConvert.DeserializeObject<Entities.User>(body, m_JsonSerializerSettings);
             return user;
         }
@@ -408,23 +410,24 @@ public class APIHelper : IAPIHelper
         return null;
     }
 
-    public async Task<JObject> GetUserInfoById(string endpoint)
+        public async Task<JObject> GetUserInfoById(string endpoint)
-    {
-        try
         {
-            HttpClient client = new();
+            try
-            HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
+            {
+                HttpClient client = new();
+                HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
 
-            await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
+                await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
-            using var response = await client.SendAsync(request);
+                using var response = await client.SendAsync(request);
 
-            response.EnsureSuccessStatusCode();
+                response.EnsureSuccessStatusCode();
-            var body = await response.Content.ReadAsStringAsync();
+                var body = await response.Content.ReadAsStringAsync();
+                WriteRawBody($"user-list-{endpoint.Replace("/", "_")}", body);
 
-            //if the content creator doesnt exist, we get a 200 response, but the content isnt usable
+                //if the content creator doesnt exist, we get a 200 response, but the content isnt usable
-            //so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
+                //so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
-            //which means we wont get loads of exceptions
+                //which means we wont get loads of exceptions
-            if (body.Equals("[]"))
+                if (body.Equals("[]"))
                 return null;
 
             JObject jObject = JObject.Parse(body);
@@ -776,10 +779,10 @@ public class APIHelper : IAPIHelper
                     {
                         Highlights newhighlights = new();
 
-                        Log.Debug("Media Highlights - " + endpoint);
+            Log.Debug("Media Highlights - " + endpoint);
 
-                        var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
-                        newhighlights = JsonConvert.DeserializeObject<Highlights>(loopbody, m_JsonSerializerSettings);
+            newhighlights = JsonConvert.DeserializeObject<Highlights>(loopbody, m_JsonSerializerSettings);
 
                         highlights.list.AddRange(newhighlights.list);
                         if (!newhighlights.hasMore)
@@ -812,10 +815,11 @@ public class APIHelper : IAPIHelper
                         highlight_request.Headers.Add(keyValuePair.Key, keyValuePair.Value);
                     }
 
-                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger);
+                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, highlight_request.RequestUri?.ToString());
                     using var highlightResponse = await highlight_client.SendAsync(highlight_request);
                     highlightResponse.EnsureSuccessStatusCode();
                     var highlightBody = await highlightResponse.Content.ReadAsStringAsync();
+                    WriteRawBody($"highlight-{highlight_id}", highlightBody);
                     highlightMedia = JsonConvert.DeserializeObject<HighlightMedia>(highlightBody, m_JsonSerializerSettings);
                     if (highlightMedia != null)
                     {
@@ -2004,7 +2008,7 @@ public class APIHelper : IAPIHelper
                 { "skip_users", "all" }
             };
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-initial-{username}");
             paidMessages = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
             ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.list.Count}[/]");
             ctx.Spinner(Spinner.Known.Dots);
@@ -2014,22 +2018,10 @@ public class APIHelper : IAPIHelper
                 getParams["offset"] = paidMessages.list.Count.ToString();
                 while (true)
                 {
-                    string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
                     Purchased newpaidMessages = new();
-                    Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
+                    var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-page-{getParams["offset"]}-{username}");
-                    HttpClient loopclient = GetHttpClient(config);
+                    if (!string.IsNullOrEmpty(loopbody))
-
-                    HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
-
-                    foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
                     {
-                        looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
-                    }
-                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString());
-                    using (var loopresponse = await loopclient.SendAsync(looprequest))
-                    {
-                        loopresponse.EnsureSuccessStatusCode();
-                        var loopbody = await loopresponse.Content.ReadAsStringAsync();
                         newpaidMessages = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
                     }
                     paidMessages.list.AddRange(newpaidMessages.list);
@@ -2237,29 +2229,18 @@ public class APIHelper : IAPIHelper
                 { "skip_users", "all" }
             };
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, "posts-paid-all-initial");
+            WriteRawBody("posts-paid-all-initial", body);
             purchased = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
             if (purchased != null && purchased.hasMore)
             {
                 getParams["offset"] = purchased.list.Count.ToString();
                 while (true)
                 {
-                    string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
                     Purchased newPurchased = new();
-                    Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
+                    var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"posts-paid-all-page-{getParams["offset"]}");
-                    HttpClient loopclient = GetHttpClient(config);
+                    if (!string.IsNullOrEmpty(loopbody))
-
-                    HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
-
-                    foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
                     {
-                        looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
-                    }
-                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger);
-                    using (var loopresponse = await loopclient.SendAsync(looprequest))
-                    {
-                        loopresponse.EnsureSuccessStatusCode();
-                        var loopbody = await loopresponse.Content.ReadAsStringAsync();
                         newPurchased = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
                     }
                     purchased.list.AddRange(newPurchased.list);
@@ -2431,27 +2412,13 @@ public class APIHelper : IAPIHelper
                 getParams["offset"] = purchased.list.Count.ToString();
                 while (true)
                 {
-                    string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
                     Purchased newPurchased = new();
-                    Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
+                    DiagInfo("PurchasedTab page request | ctx={Context} offset={Offset} limit={Limit}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit);
-                    HttpClient loopclient = GetHttpClient(config);
+                    var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"{diagContext}-page-{getParams["offset"]}");
-
+                    if (!string.IsNullOrEmpty(loopbody))
-                    HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
-
-                    foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
                     {
-                        looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
-                    }
-                    DiagInfo("PurchasedTab page request | ctx={Context} url={Url} offset={Offset} limit={Limit} headers={Headers}", diagContext, looprequest.RequestUri, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit, FlattenHeaders(looprequest.Headers));
-                    await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString());
-                    using (var loopresponse = await loopclient.SendAsync(looprequest))
-                    {
-                        DiagInfo("PurchasedTab page response | ctx={Context} status={Status} reason={Reason} url={Url}", diagContext, loopresponse.StatusCode, loopresponse.ReasonPhrase, looprequest.RequestUri);
-                        loopresponse.EnsureSuccessStatusCode();
-                        var loopbody = await loopresponse.Content.ReadAsStringAsync();
                         newPurchased = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
                         DiagInfo("PurchasedTab page parsed | ctx={Context} offset={Offset} items={Items} hasMore={HasMore} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", newPurchased?.list?.Count ?? 0, newPurchased?.hasMore ?? false, loopbody?.Length ?? 0, TruncateForLog(loopbody));
-                        WriteRawBody($"{diagContext}-page-{getParams["offset"]}", loopbody);
                     }
                     purchased.list.AddRange(newPurchased.list);
                     if (!newPurchased.hasMore)