WIP

It increments the offset with the limit before making the next request, until 'hasMore' is false. Also tweaked loop code, to be a bit more clean.

OF DL/Helpers/APIHelper.cs

@@ -13,6 +13,7 @@ using OF_DL.Enumerations;
 using OF_DL.Enumurations;
 using Serilog;
 using Spectre.Console;
+using System.Diagnostics;
 using System.Globalization;
 using System.Security.Cryptography;
 using System.Text;
@@ -116,18 +117,38 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, int attemptNumber = 1)
     {
-        Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        const int MAX_ATTEMPTS = 10;
+        const int DELAY_BEFORE_RETRY = 1000;
 
-        HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
+        Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", attemptNumber);
-        using var response = await client.SendAsync(request);
-        response.EnsureSuccessStatusCode();
-        string body = await response.Content.ReadAsStringAsync();
 
-        Log.Debug(body);
+        try
+        {
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
+            using var response = await client.SendAsync(request);
 
-        return body;
+            if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
+                Debugger.Break();
+
+            response.EnsureSuccessStatusCode();
+            string body = await response.Content.ReadAsStringAsync();
+
+            Log.Debug(body);
+
+            return body;
+        }
+        catch (HttpRequestException ex)
+        {
+            if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && attemptNumber <= MAX_ATTEMPTS)
+            {
+                await Task.Delay(DELAY_BEFORE_RETRY);
+                return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, ++attemptNumber);
+            }
+
+            throw;
+        }
     }
 
 
@@ -298,47 +319,44 @@ public class APIHelper : IAPIHelper
         try
         {
             Dictionary<string, int> users = new();
-            Subscriptions subscriptions = new();
+
+            int limit = 25;
+            int offset = 0;
+
+            getParams["limit"] = limit.ToString();
+            getParams["offset"] = offset.ToString();
 
             Log.Debug("Calling GetAllSubscrptions");
 
-            string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            while (true)
-
-            subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
-            if (subscriptions != null && subscriptions.hasMore)
             {
-                getParams["offset"] = subscriptions.list.Count.ToString();
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
 
-                while (true)
+                if (string.IsNullOrWhiteSpace(body))
+                    break;
+
+                Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
+
+                if (subscriptions?.list is null)
+                    break;
+
+                foreach (Subscriptions.List item in subscriptions.list)
                 {
-                    Subscriptions newSubscriptions = new();
+                    if (users.ContainsKey(item.username))
-                    string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+                        continue;
 
-                    if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
+                    bool isRestricted = item.isRestricted ?? false;
-                    {
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
-                        newSubscriptions = JsonConvert.DeserializeObject<Subscriptions>(loopbody, m_JsonSerializerSettings);
-                    }
-                    else
-                    {
-                        break;
-                    }
 
-                    subscriptions.list.AddRange(newSubscriptions.list);
+                    if (!isRestricted || isRestrictedButAllowed)
-                    if (!newSubscriptions.hasMore)
+                        users.Add(item.username, item.id);
-                    {
-                        break;
-                    }
-                    getParams["offset"] = subscriptions.list.Count.ToString();
                 }
-            }
 
-            foreach (Subscriptions.List subscription in subscriptions.list)
+                if (!subscriptions.hasMore)
-            {
+                    break;
-                if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted))
+
-                    && !users.ContainsKey(subscription.username))
+                offset += limit;
-                {
+                getParams["offset"] = offset.ToString();
-                    users.Add(subscription.username, subscription.id);
-                }
             }
 
             return users;
@@ -361,8 +379,6 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "active" },
             { "format", "infinite"}
         };
@@ -376,8 +392,6 @@ public class APIHelper : IAPIHelper
 
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
@@ -460,12 +474,16 @@ public class APIHelper : IAPIHelper
 
         try
         {
+            int limit = 50;
             int offset = 0;
+
             Dictionary<string, string> getParams = new()
             {
+                { "limit", limit.ToString() },
                 { "offset", offset.ToString() },
-                { "limit", "50" }
+                { "format", "infinite"}
             };
+
             List<string> users = new();
 
             while (true)
@@ -476,6 +494,8 @@ public class APIHelper : IAPIHelper
                     break;
                 }
 
+                UserList? userList = JsonConvert.DeserializeObject<UserList>(body, m_JsonSerializerSettings);
+
                 List<UsersList>? usersList = JsonConvert.DeserializeObject<List<UsersList>>(body);
 
                 if (usersList == null || usersList.Count <= 0)
@@ -497,6 +517,7 @@ public class APIHelper : IAPIHelper
                 getParams["offset"] = Convert.ToString(offset);
 
             }
+
             return users;
         }
         catch (Exception ex)
@@ -1150,7 +1171,7 @@ public class APIHelper : IAPIHelper
                                     }
                                     break;
                                 case VideoResolution._240:
-                                    if(medium.videoSources != null)
+                                    if (medium.videoSources != null)
                                     {
                                         if (!string.IsNullOrEmpty(medium.videoSources._240))
                                         {
@@ -1177,7 +1198,7 @@ public class APIHelper : IAPIHelper
                                         }
                                     }
                                     break;
-                                
+
                             }
                         }
                         else if (medium.canView && medium.files != null && medium.files.drm != null)
@@ -2134,11 +2155,11 @@ public class APIHelper : IAPIHelper
                         {
                             JObject user = await GetUserInfoById($"/users/list?x[]={purchase.fromUser.id}");
 
-                            if(user is null)
+                            if (user is null)
                             {
                                 if (!config.BypassContentForCreatorsWhoNoLongerExist)
                                 {
-                                    if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
+                                    if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
                                     {
                                         purchasedTabUsers.Add($"Deleted User - {purchase.fromUser.id}", purchase.fromUser.id);
                                     }
@@ -2188,7 +2209,7 @@ public class APIHelper : IAPIHelper
                             {
                                 if (!config.BypassContentForCreatorsWhoNoLongerExist)
                                 {
-                                    if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
+                                    if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
                                     {
                                         purchasedTabUsers.Add($"Deleted User - {purchase.author.id}", purchase.author.id);
                                     }

OF DL/Program.cs

@@ -844,7 +844,9 @@ public class Program
 		{
 			DateTime startTime = DateTime.Now;
 			Dictionary<string, int> users = new();
-			Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
+
+            AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
+            Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
 
 			Log.Debug("Subscriptions: ");
 
@@ -860,7 +862,8 @@ public class Program
 			{
 				Log.Debug("Inactive Subscriptions: ");
 
-				Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
+                AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
+                Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
 				foreach (KeyValuePair<string, int> expiredSub in expiredSubs)
 				{
 					if (!users.ContainsKey(expiredSub.Key))
@@ -1024,9 +1027,11 @@ public class Program
 				Log.Debug($"Download path: {p}");
 
 				List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
-				foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
+                int userNum = 1;
+                int userCount = purchasedTabCollections.Count;
+                foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
 				{
-					AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]");
+					AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]");
 					string path = "";
 					if (!string.IsNullOrEmpty(Config.DownloadPath))
 					{
@@ -1135,8 +1140,10 @@ public class Program
 			}
 			else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
 			{
-				//Iterate over each user in the list of users
+                //Iterate over each user in the list of users
-				foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
+                int userNum = 1;
+                int userCount = hasSelectedUsersKVP.Value.Count;
+                foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
 				{
 					int paidPostCount = 0;
 					int postCount = 0;
@@ -1146,7 +1153,7 @@ public class Program
 					int highlightsCount = 0;
 					int messagesCount = 0;
 					int paidMessagesCount = 0;
-					AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]");
+					AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]");
 
 					Log.Debug($"Scraping Data for {user.Key}");