Debug logging in BuildHeaderAndExecuteRequests

It always increments the offset with the limit before making the next request.

OF DL/Helpers/APIHelper.cs

@@ -14,6 +14,7 @@ using OF_DL.Enumerations;
 using OF_DL.Enumurations;
 using Serilog;
 using Spectre.Console;
+using System.Diagnostics;
 using System.Globalization;
 using System.Security.Cryptography;
 using System.Text;
@@ -26,6 +27,9 @@ namespace OF_DL.Helpers;
 
 public class APIHelper : IAPIHelper
 {
+    private const int MAX_RETRIES = 10;
+    private const int DELAY_BEFORE_RETRY = 1000;
+
     private static readonly JsonSerializerSettings m_JsonSerializerSettings;
     private readonly IDBHelper m_DBHelper;
     private readonly IDownloadConfig downloadConfig;
@@ -119,18 +123,41 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null, int retryCount = 0)
     {
-        Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", retryCount + 1);
 
-        HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
+        try
-        using var response = await client.SendAsync(request);
+        {
-        response.EnsureSuccessStatusCode();
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
-        string body = await response.Content.ReadAsStringAsync();
 
-        Log.Debug(body);
+            Debug.WriteLine($"Executing {request.Method.Method.ToUpper()} request: {request.RequestUri}\r\n\t{GetParamsString(getParams)}");
 
-        return body;
+            using var response = await client.SendAsync(request);
+
+            if (Debugger.IsAttached && !response.IsSuccessStatusCode)
+                Debugger.Break();
+
+            response.EnsureSuccessStatusCode();
+            string body = await response.Content.ReadAsStringAsync();
+
+            Log.Debug(body);
+
+            return body;
+        }
+        catch (HttpRequestException ex)
+        {
+            if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && retryCount < MAX_RETRIES)
+            {
+                await Task.Delay(DELAY_BEFORE_RETRY);
+                return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, method, ++retryCount);
+            }
+
+            throw;
+        }
+
+        static string GetParamsString(Dictionary<string, string> getParams)
+            => string.Join(" | ", getParams.Select(kv => $"{kv.Key}={kv.Value}"));
     }
 
 
@@ -304,50 +331,44 @@ public class APIHelper : IAPIHelper
         try
         {
             Dictionary<string, int> users = new();
-            Subscriptions subscriptions = new();
+
+            int limit = 25;
+            int offset = 0;
+
+            getParams["limit"] = limit.ToString();
+            getParams["offset"] = offset.ToString();
 
             Log.Debug("Calling GetAllSubscrptions");
 
-            string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
+            while (true)
-
-            subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
-            if (subscriptions != null && subscriptions.hasMore)
             {
-                getParams["offset"] = subscriptions.list.Count.ToString();
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
 
-                while (true)
+                if (string.IsNullOrWhiteSpace(body))
+                    break;
+
+                Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
+
+                if (subscriptions?.list is null)
+                    break;
+
+                foreach (Subscriptions.List item in subscriptions.list)
                 {
-                    Subscriptions newSubscriptions = new();
+                    if (users.ContainsKey(item.username))
-                    string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
+                        continue;
 
-                    if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
+                    bool isRestricted = item.isRestricted ?? false;
-                    {
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
-                        newSubscriptions = JsonConvert.DeserializeObject<Subscriptions>(loopbody, m_JsonSerializerSettings);
-                    }
-                    else
-                    {
-                        break;
-                    }
 
-                    subscriptions.list.AddRange(newSubscriptions.list);
+                    if (!isRestricted || isRestrictedButAllowed)
-                    if (!newSubscriptions.hasMore)
+                        users.Add(item.username, item.id);
-                    {
-                        break;
-                    }
-                    getParams["offset"] = subscriptions.list.Count.ToString();
                 }
-            }
 
-            foreach (Subscriptions.List subscription in subscriptions.list)
+                if (!subscriptions.hasMore)
-            {
+                    break;
-                if (users.ContainsKey(subscription.username))
-                    continue;
 
-                bool isRestricted = subscription.isRestricted ?? false;
+                offset += limit;
-                bool isRestrictedButAllowed = isRestricted && includeRestricted;
+                getParams["offset"] = offset.ToString();
-
-                if (!isRestricted || isRestrictedButAllowed)
-                    users.Add(subscription.username, subscription.id);
             }
 
             return users;
@@ -370,12 +391,13 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "active" },
             { "format", "infinite"}
         };
 
+        Log.Debug("Calling GetActiveSubscriptions");
+        AnsiConsole.Markup($"[red]Getting Active Subscriptions (Include Restricted: {includeRestricted})\n[/]");
+
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
 
@@ -384,13 +406,12 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
 
         Log.Debug("Calling GetExpiredSubscriptions");
+        AnsiConsole.Markup($"[red]Getting Expired Subscriptions (Include Restricted: {includeRestricted})\n[/]");
 
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
@@ -399,13 +420,12 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
 
         Log.Debug("Calling GetBlockedUsers");
+        AnsiConsole.Markup($"[red]Getting Blocked Users\n[/]");
 
         return await GetAllSubscriptions(getParams, endpoint, true, config);
     }
@@ -1172,7 +1192,7 @@ public class APIHelper : IAPIHelper
                                     }
                                     break;
                                 case VideoResolution._240:
-                                    if(medium.videoSources != null)
+                                    if (medium.videoSources != null)
                                     {
                                         if (!string.IsNullOrEmpty(medium.videoSources._240))
                                         {

OF DL/Program.cs

@@ -893,10 +893,6 @@ public class Program
     {
         const string OUTPUT_FILE = "blocked-users.json";
 
-        Log.Debug($"Calling GetBlockedUsers");
-
-        AnsiConsole.Markup($"[red]Getting Blocked Users\n[/]");
-
         Dictionary<string, int>? blockedUsers = await m_ApiHelper.GetBlockedUsers("/users/blocked", Config);
 
         if (blockedUsers is null || blockedUsers.Count == 0)
@@ -922,15 +918,10 @@ public class Program
 			DateTime startTime = DateTime.Now;
             Dictionary<string, int> users = new();
 
-            Task<Dictionary<string, int>?> taskActive = m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
+            Dictionary<string, int> subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
-            Task<Dictionary<string, int>?> taskExpired = Config!.IncludeExpiredSubscriptions
+            Dictionary<string, int> subsExpired = Config!.IncludeExpiredSubscriptions
-                ? m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config)
+                ? await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? []
-                : Task.FromResult<Dictionary<string, int>?>([]);
+                : [] ;
-
-            await Task.WhenAll(taskActive, taskExpired);
-
-            Dictionary<string, int> subsActive = await taskActive ?? [];
-            Dictionary<string, int> subsExpired = await taskExpired ?? [];
 
             Log.Debug("Subscriptions: ");
             foreach (KeyValuePair<string, int> activeSub in subsActive)