WIP

It increments the offset with the limit before making the next request, until 'hasMore' is false. Also tweaked loop code, to be a bit more clean.

OF DL/Helpers/APIHelper.cs

@@ -13,6 +13,7 @@ using OF_DL.Enumerations;
 using OF_DL.Enumurations;
 using Serilog;
 using Spectre.Console;
+using System.Diagnostics;
 using System.Globalization;
 using System.Security.Cryptography;
 using System.Text;
@@ -116,12 +117,21 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, int attemptNumber = 1)
     {
-        Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        const int MAX_ATTEMPTS = 10;
+        const int DELAY_BEFORE_RETRY = 1000;
 
+        Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", attemptNumber);
+
+        try
+        {
             HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
             using var response = await client.SendAsync(request);
+
+            if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
+                Debugger.Break();
+
             response.EnsureSuccessStatusCode();
             string body = await response.Content.ReadAsStringAsync();
 
@@ -129,6 +139,17 @@ public class APIHelper : IAPIHelper
 
             return body;
         }
+        catch (HttpRequestException ex)
+        {
+            if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && attemptNumber <= MAX_ATTEMPTS)
+            {
+                await Task.Delay(DELAY_BEFORE_RETRY);
+                return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, ++attemptNumber);
+            }
+
+            throw;
+        }
+    }
 
 
     private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint)
@@ -298,47 +319,44 @@ public class APIHelper : IAPIHelper
         try
         {
             Dictionary<string, int> users = new();
-            Subscriptions subscriptions = new();
+
+            int limit = 25;
+            int offset = 0;
+
+            getParams["limit"] = limit.ToString();
+            getParams["offset"] = offset.ToString();
 
             Log.Debug("Calling GetAllSubscrptions");
 
-            string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
-
-            subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
-            if (subscriptions != null && subscriptions.hasMore)
-            {
-                getParams["offset"] = subscriptions.list.Count.ToString();
-
             while (true)
             {
-                    Subscriptions newSubscriptions = new();
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
-                    string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
 
-                    if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
+                if (string.IsNullOrWhiteSpace(body))
-                    {
-                        newSubscriptions = JsonConvert.DeserializeObject<Subscriptions>(loopbody, m_JsonSerializerSettings);
-                    }
-                    else
-                    {
                     break;
-                    }
 
-                    subscriptions.list.AddRange(newSubscriptions.list);
+                Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
-                    if (!newSubscriptions.hasMore)
+
-                    {
+                if (subscriptions?.list is null)
                     break;
-                    }
+
-                    getParams["offset"] = subscriptions.list.Count.ToString();
+                foreach (Subscriptions.List item in subscriptions.list)
-                }
+                {
+                    if (users.ContainsKey(item.username))
+                        continue;
+
+                    bool isRestricted = item.isRestricted ?? false;
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
+
+                    if (!isRestricted || isRestrictedButAllowed)
+                        users.Add(item.username, item.id);
                 }
 
-            foreach (Subscriptions.List subscription in subscriptions.list)
+                if (!subscriptions.hasMore)
-            {
+                    break;
-                if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted))
+
-                    && !users.ContainsKey(subscription.username))
+                offset += limit;
-                {
+                getParams["offset"] = offset.ToString();
-                    users.Add(subscription.username, subscription.id);
-                }
             }
 
             return users;
@@ -361,8 +379,6 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "active" },
             { "format", "infinite"}
         };
@@ -376,8 +392,6 @@ public class APIHelper : IAPIHelper
 
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
@@ -460,12 +474,16 @@ public class APIHelper : IAPIHelper
 
         try
         {
+            int limit = 50;
             int offset = 0;
+
             Dictionary<string, string> getParams = new()
             {
+                { "limit", limit.ToString() },
                 { "offset", offset.ToString() },
-                { "limit", "50" }
+                { "format", "infinite"}
             };
+
             List<string> users = new();
 
             while (true)
@@ -476,6 +494,8 @@ public class APIHelper : IAPIHelper
                     break;
                 }
 
+                UserList? userList = JsonConvert.DeserializeObject<UserList>(body, m_JsonSerializerSettings);
+
                 List<UsersList>? usersList = JsonConvert.DeserializeObject<List<UsersList>>(body);
 
                 if (usersList == null || usersList.Count <= 0)
@@ -497,6 +517,7 @@ public class APIHelper : IAPIHelper
                 getParams["offset"] = Convert.ToString(offset);
 
             }
+
             return users;
         }
         catch (Exception ex)

OF DL/Program.cs

@@ -844,6 +844,8 @@ public class Program
 		{
 			DateTime startTime = DateTime.Now;
 			Dictionary<string, int> users = new();
+
+            AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
             Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
 
 			Log.Debug("Subscriptions: ");
@@ -860,6 +862,7 @@ public class Program
 			{
 				Log.Debug("Inactive Subscriptions: ");
 
+                AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
                 Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
 				foreach (KeyValuePair<string, int> expiredSub in expiredSubs)
 				{
@@ -1024,9 +1027,11 @@ public class Program
 				Log.Debug($"Download path: {p}");
 
 				List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
+                int userNum = 1;
+                int userCount = purchasedTabCollections.Count;
                 foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
 				{
-					AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]");
+					AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]");
 					string path = "";
 					if (!string.IsNullOrEmpty(Config.DownloadPath))
 					{
@@ -1136,6 +1141,8 @@ public class Program
 			else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
 			{
                 //Iterate over each user in the list of users
+                int userNum = 1;
+                int userCount = hasSelectedUsersKVP.Value.Count;
                 foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
 				{
 					int paidPostCount = 0;
@@ -1146,7 +1153,7 @@ public class Program
 					int highlightsCount = 0;
 					int messagesCount = 0;
 					int paidMessagesCount = 0;
-					AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]");
+					AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]");
 
 					Log.Debug($"Scraping Data for {user.Key}");