Debug logging in BuildHeaderAndExecuteRequests

It always increments the offset with the limit before making the next request.

OF DL/Helpers/APIHelper.cs

@@ -14,6 +14,7 @@ using OF_DL.Enumerations;
 using OF_DL.Enumurations;
 using Serilog;
 using Spectre.Console;
+using System.Diagnostics;
 using System.Globalization;
 using System.Security.Cryptography;
 using System.Text;
@@ -26,6 +27,9 @@ namespace OF_DL.Helpers;
 
 public class APIHelper : IAPIHelper
 {
+    private const int MAX_RETRIES = 10;
+    private const int DELAY_BEFORE_RETRY = 1000;
+
     private static readonly JsonSerializerSettings m_JsonSerializerSettings;
     private readonly IDBHelper m_DBHelper;
     private readonly IDownloadConfig downloadConfig;
@@ -119,18 +123,41 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null, int retryCount = 0)
     {
-        Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", retryCount + 1);
 
-        HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
+        try
-        using var response = await client.SendAsync(request);
+        {
-        response.EnsureSuccessStatusCode();
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
-        string body = await response.Content.ReadAsStringAsync();
 
-        Log.Debug(body);
+            Debug.WriteLine($"Executing {request.Method.Method.ToUpper()} request: {request.RequestUri}\r\n\t{GetParamsString(getParams)}");
 
-        return body;
+            using var response = await client.SendAsync(request);
+
+            if (Debugger.IsAttached && !response.IsSuccessStatusCode)
+                Debugger.Break();
+
+            response.EnsureSuccessStatusCode();
+            string body = await response.Content.ReadAsStringAsync();
+
+            Log.Debug(body);
+
+            return body;
+        }
+        catch (HttpRequestException ex)
+        {
+            if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && retryCount < MAX_RETRIES)
+            {
+                await Task.Delay(DELAY_BEFORE_RETRY);
+                return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, method, ++retryCount);
+            }
+
+            throw;
+        }
+
+        static string GetParamsString(Dictionary<string, string> getParams)
+            => string.Join(" | ", getParams.Select(kv => $"{kv.Key}={kv.Value}"));
     }
 
 
@@ -299,13 +326,16 @@ public class APIHelper : IAPIHelper
     }
 
 
-    public async Task<Dictionary<string, int>?> GetAllSubscriptions(Dictionary<string, string> getParams, string endpoint, bool includeRestricted, IDownloadConfig config)
+    public async Task<Dictionary<string, int>?> GetAllSubscriptions(Dictionary<string, string> getParams, int limit, string endpoint, bool includeRestricted, IDownloadConfig config)
     {
         try
         {
             Dictionary<string, int> users = new();
             Subscriptions subscriptions = new();
 
+            int offset = 0;
+            getParams["offset"] = offset.ToString();
+
             Log.Debug("Calling GetAllSubscrptions");
 
             string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
@@ -313,7 +343,8 @@ public class APIHelper : IAPIHelper
             subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
             if (subscriptions != null && subscriptions.hasMore)
             {
-                getParams["offset"] = subscriptions.list.Count.ToString();
+                offset += limit;
+                getParams["offset"] = offset.ToString();
 
                 while (true)
                 {
@@ -334,7 +365,9 @@ public class APIHelper : IAPIHelper
                     {
                         break;
                     }
-                    getParams["offset"] = subscriptions.list.Count.ToString();
+
+                    offset += limit;
+                    getParams["offset"] = offset.ToString();
                 }
             }
 
@@ -368,46 +401,51 @@ public class APIHelper : IAPIHelper
 
     public async Task<Dictionary<string, int>?> GetActiveSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config)
     {
+        int limit = 50;
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
+            { "limit", $"{limit}" },
-            { "limit", "50" },
             { "type", "active" },
             { "format", "infinite"}
         };
 
-        return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
+        Log.Debug("Calling GetActiveSubscriptions");
+        AnsiConsole.Markup($"[red]Getting Active Subscriptions (Include Restricted: {includeRestricted})\n[/]");
+
+        return await GetAllSubscriptions(getParams, limit, endpoint, includeRestricted, config);
     }
 
 
     public async Task<Dictionary<string, int>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config)
     {
+        int limit = 50;
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
+            { "limit", $"{limit}" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
 
         Log.Debug("Calling GetExpiredSubscriptions");
+        AnsiConsole.Markup($"[red]Getting Expired Subscriptions (Include Restricted: {includeRestricted})\n[/]");
 
-        return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
+        return await GetAllSubscriptions(getParams, limit, endpoint, includeRestricted, config);
     }
 
     public async Task<Dictionary<string, int>?> GetBlockedUsers(string endpoint, IDownloadConfig config)
     {
+        int limit = 50;
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
+            { "limit", $"{limit}" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
 
         Log.Debug("Calling GetBlockedUsers");
+        AnsiConsole.Markup($"[red]Getting Blocked Users\n[/]");
 
-        return await GetAllSubscriptions(getParams, endpoint, true, config);
+        return await GetAllSubscriptions(getParams, limit, endpoint, true, config);
     }
 
     public async Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config)
@@ -1172,7 +1210,7 @@ public class APIHelper : IAPIHelper
                                     }
                                     break;
                                 case VideoResolution._240:
-                                    if(medium.videoSources != null)
+                                    if (medium.videoSources != null)
                                     {
                                         if (!string.IsNullOrEmpty(medium.videoSources._240))
                                         {
@@ -1199,7 +1237,7 @@ public class APIHelper : IAPIHelper
                                         }
                                     }
                                     break;
-                                
+
                             }
                         }
                         else if (medium.canView && medium.files != null && medium.files.drm != null)

OF DL/Program.cs

@@ -893,10 +893,6 @@ public class Program
     {
         const string OUTPUT_FILE = "blocked-users.json";
 
-        Log.Debug($"Calling GetBlockedUsers");
-
-        AnsiConsole.Markup($"[red]Getting Blocked Users\n[/]");
-
         Dictionary<string, int>? blockedUsers = await m_ApiHelper.GetBlockedUsers("/users/blocked", Config);
 
         if (blockedUsers is null || blockedUsers.Count == 0)
@@ -922,15 +918,10 @@ public class Program
 			DateTime startTime = DateTime.Now;
             Dictionary<string, int> users = new();
 
-            Task<Dictionary<string, int>?> taskActive = m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
+            Dictionary<string, int> subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
-            Task<Dictionary<string, int>?> taskExpired = Config!.IncludeExpiredSubscriptions
+            Dictionary<string, int> subsExpired = Config!.IncludeExpiredSubscriptions
-                ? m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config)
+                ? await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? []
-                : Task.FromResult<Dictionary<string, int>?>([]);
+                : [] ;
-
-            await Task.WhenAll(taskActive, taskExpired);
-
-            Dictionary<string, int> subsActive = await taskActive ?? [];
-            Dictionary<string, int> subsExpired = await taskExpired ?? [];
 
             Log.Debug("Subscriptions: ");
             foreach (KeyValuePair<string, int> activeSub in subsActive)