Updated subscription lookup to match OF website.

It always increments the offset with the limit before making the next request.

OF DL/Helpers/APIHelper.cs

@@ -26,6 +26,9 @@ namespace OF_DL.Helpers;
 
 public class APIHelper : IAPIHelper
 {
+    private const int MAX_RETRIES = 10;
+    private const int DELAY_BEFORE_RETRY = 1000;
+
     private static readonly JsonSerializerSettings m_JsonSerializerSettings;
     private readonly IDBHelper m_DBHelper;
     private readonly IDownloadConfig downloadConfig;
@@ -119,18 +122,31 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null, int retryCount = 0)
     {
-        Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", retryCount + 1);
 
-        HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
+        try
-        using var response = await client.SendAsync(request);
+        {
-        response.EnsureSuccessStatusCode();
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
-        string body = await response.Content.ReadAsStringAsync();
+            using var response = await client.SendAsync(request);
+            response.EnsureSuccessStatusCode();
+            string body = await response.Content.ReadAsStringAsync();
 
-        Log.Debug(body);
+            Log.Debug(body);
 
-        return body;
+            return body;
+        }
+        catch (HttpRequestException ex)
+        {
+            if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && retryCount < MAX_RETRIES)
+            {
+                await Task.Delay(DELAY_BEFORE_RETRY);
+                return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, method, ++retryCount);
+            }
+
+            throw;
+        }
     }
 
 
@@ -304,50 +320,44 @@ public class APIHelper : IAPIHelper
         try
         {
             Dictionary<string, int> users = new();
-            Subscriptions subscriptions = new();
+
+            int limit = 25;
+            int offset = 0;
+
+            getParams["limit"] = limit.ToString();
+            getParams["offset"] = offset.ToString();
 
             Log.Debug("Calling GetAllSubscrptions");
 
-            string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
+            while (true)
-
-            subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
-            if (subscriptions != null && subscriptions.hasMore)
             {
-                getParams["offset"] = subscriptions.list.Count.ToString();
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
 
-                while (true)
+                if (string.IsNullOrWhiteSpace(body))
+                    break;
+
+                Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
+
+                if (subscriptions?.list is null)
+                    break;
+
+                foreach (Subscriptions.List item in subscriptions.list)
                 {
-                    Subscriptions newSubscriptions = new();
+                    if (users.ContainsKey(item.username))
-                    string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
+                        continue;
 
-                    if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
+                    bool isRestricted = item.isRestricted ?? false;
-                    {
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
-                        newSubscriptions = JsonConvert.DeserializeObject<Subscriptions>(loopbody, m_JsonSerializerSettings);
-                    }
-                    else
-                    {
-                        break;
-                    }
 
-                    subscriptions.list.AddRange(newSubscriptions.list);
+                    if (!isRestricted || isRestrictedButAllowed)
-                    if (!newSubscriptions.hasMore)
+                        users.Add(item.username, item.id);
-                    {
-                        break;
-                    }
-                    getParams["offset"] = subscriptions.list.Count.ToString();
                 }
-            }
 
-            foreach (Subscriptions.List subscription in subscriptions.list)
+                if (!subscriptions.hasMore)
-            {
+                    break;
-                if (users.ContainsKey(subscription.username))
-                    continue;
 
-                bool isRestricted = subscription.isRestricted ?? false;
+                offset += limit;
-                bool isRestrictedButAllowed = isRestricted && includeRestricted;
+                getParams["offset"] = offset.ToString();
-
-                if (!isRestricted || isRestrictedButAllowed)
-                    users.Add(subscription.username, subscription.id);
             }
 
             return users;
@@ -370,12 +380,13 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "active" },
             { "format", "infinite"}
         };
 
+        Log.Debug("Calling GetActiveSubscriptions");
+        AnsiConsole.Markup($"[red]Getting Active Subscriptions (Include Restricted: {includeRestricted})\n[/]");
+
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
 
@@ -384,13 +395,12 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
 
         Log.Debug("Calling GetExpiredSubscriptions");
+        AnsiConsole.Markup($"[red]Getting Expired Subscriptions (Include Restricted: {includeRestricted})\n[/]");
 
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
@@ -399,13 +409,12 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
 
         Log.Debug("Calling GetBlockedUsers");
+        AnsiConsole.Markup($"[red]Getting Blocked Users\n[/]");
 
         return await GetAllSubscriptions(getParams, endpoint, true, config);
     }
@@ -1172,7 +1181,7 @@ public class APIHelper : IAPIHelper
                                     }
                                     break;
                                 case VideoResolution._240:
-                                    if(medium.videoSources != null)
+                                    if (medium.videoSources != null)
                                     {
                                         if (!string.IsNullOrEmpty(medium.videoSources._240))
                                         {

OF DL/Program.cs

@@ -893,10 +893,6 @@ public class Program
     {
         const string OUTPUT_FILE = "blocked-users.json";
 
-        Log.Debug($"Calling GetBlockedUsers");
-
-        AnsiConsole.Markup($"[red]Getting Blocked Users\n[/]");
-
         Dictionary<string, int>? blockedUsers = await m_ApiHelper.GetBlockedUsers("/users/blocked", Config);
 
         if (blockedUsers is null || blockedUsers.Count == 0)