From a4af7f27da8721f5db849e2ec59297bf8775285d Mon Sep 17 00:00:00 2001 From: Casper Sparre Date: Sat, 8 Mar 2025 15:14:27 +0100 Subject: [PATCH] Updated subscription lookup to match OF website. --- OF DL/Helpers/APIHelper.cs | 103 ++++++++++++++++++++----------------- OF DL/Program.cs | 37 +++++++------ 2 files changed, 76 insertions(+), 64 deletions(-) diff --git a/OF DL/Helpers/APIHelper.cs b/OF DL/Helpers/APIHelper.cs index 8034d45..21d1d6d 100644 --- a/OF DL/Helpers/APIHelper.cs +++ b/OF DL/Helpers/APIHelper.cs @@ -25,6 +25,9 @@ namespace OF_DL.Helpers; public class APIHelper : IAPIHelper { + private const int MAX_RETRIES = 10; + private const int DELAY_BEFORE_RETRY = 1000; + private static readonly JsonSerializerSettings m_JsonSerializerSettings; private readonly IDBHelper m_DBHelper; private readonly IDownloadConfig downloadConfig; @@ -118,18 +121,31 @@ public class APIHelper : IAPIHelper } - private async Task BuildHeaderAndExecuteRequests(Dictionary getParams, string endpoint, HttpClient client) + private async Task BuildHeaderAndExecuteRequests(Dictionary getParams, string endpoint, HttpClient client, HttpMethod? method = null, int retryCount = 0) { - Log.Debug("Calling BuildHeaderAndExecuteRequests"); + Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", retryCount + 1); - HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint); - using var response = await client.SendAsync(request); - response.EnsureSuccessStatusCode(); - string body = await response.Content.ReadAsStringAsync(); + try + { + HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method); + using var response = await client.SendAsync(request); + response.EnsureSuccessStatusCode(); + string body = await response.Content.ReadAsStringAsync(); - Log.Debug(body); + Log.Debug(body); - return body; + return body; + } + catch (HttpRequestException ex) + { + if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && retryCount < MAX_RETRIES) + { + await Task.Delay(DELAY_BEFORE_RETRY); + return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, method, ++retryCount); + } + + throw; + } } @@ -298,47 +314,44 @@ public class APIHelper : IAPIHelper try { Dictionary users = new(); - Subscriptions subscriptions = new(); + + int limit = 25; + int offset = 0; + + getParams["limit"] = limit.ToString(); + getParams["offset"] = offset.ToString(); Log.Debug("Calling GetAllSubscrptions"); - string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient); - - subscriptions = JsonConvert.DeserializeObject(body); - if (subscriptions != null && subscriptions.hasMore) + while (true) { - getParams["offset"] = subscriptions.list.Count.ToString(); + string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient); - while (true) + if (string.IsNullOrWhiteSpace(body)) + break; + + Subscriptions? subscriptions = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); + + if (subscriptions?.list is null) + break; + + foreach (Subscriptions.List item in subscriptions.list) { - Subscriptions newSubscriptions = new(); - string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient); + if (users.ContainsKey(item.username)) + continue; - if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]")) - { - newSubscriptions = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); - } - else - { - break; - } + bool isRestricted = item.isRestricted ?? false; + bool isRestrictedButAllowed = isRestricted && includeRestricted; - subscriptions.list.AddRange(newSubscriptions.list); - if (!newSubscriptions.hasMore) - { - break; - } - getParams["offset"] = subscriptions.list.Count.ToString(); + if (!isRestricted || isRestrictedButAllowed) + users.Add(item.username, item.id); } - } - foreach (Subscriptions.List subscription in subscriptions.list) - { - if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted)) - && !users.ContainsKey(subscription.username)) - { - users.Add(subscription.username, subscription.id); - } + if (!subscriptions.hasMore) + break; + + offset += limit; + getParams["offset"] = offset.ToString(); } return users; @@ -361,12 +374,12 @@ public class APIHelper : IAPIHelper { Dictionary getParams = new() { - { "offset", "0" }, - { "limit", "50" }, { "type", "active" }, { "format", "infinite"} }; + Log.Debug("Calling GetActiveSubscriptions"); + return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config); } @@ -375,8 +388,6 @@ public class APIHelper : IAPIHelper { Dictionary getParams = new() { - { "offset", "0" }, - { "limit", "50" }, { "type", "expired" }, { "format", "infinite"} }; @@ -390,8 +401,6 @@ public class APIHelper : IAPIHelper { Dictionary getParams = new() { - { "offset", "0" }, - { "limit", "50" }, { "type", "expired" }, { "format", "infinite"} }; @@ -1163,7 +1172,7 @@ public class APIHelper : IAPIHelper } break; case VideoResolution._240: - if(medium.videoSources != null) + if (medium.videoSources != null) { if (!string.IsNullOrEmpty(medium.videoSources._240)) { @@ -1190,7 +1199,7 @@ public class APIHelper : IAPIHelper } } break; - + } } else if (medium.canView && medium.files != null && medium.files.drm != null) diff --git a/OF DL/Program.cs b/OF DL/Program.cs index 788b6b0..92d8190 100644 --- a/OF DL/Program.cs +++ b/OF DL/Program.cs @@ -892,19 +892,15 @@ public class Program { const string OUTPUT_FILE = "blocked-users.json"; - Log.Debug($"Calling GetBlockedUsers"); - - AnsiConsole.Markup($"[red]Getting Blocked Users\n[/]"); - Dictionary? blockedUsers = await m_ApiHelper.GetBlockedUsers("/users/blocked", Config); if (blockedUsers is null || blockedUsers.Count == 0) { - AnsiConsole.Markup($"[red]No Blocked Users found.\n[/]"); + AnsiConsole.Markup($"[green]No Blocked Users found.\n[/]"); } else { - AnsiConsole.Markup($"[red]Found {blockedUsers.Count} Blocked Users, saving to '{OUTPUT_FILE}'\n[/]"); + AnsiConsole.Markup($"[green]Found {blockedUsers.Count} Blocked Users, saving to '{OUTPUT_FILE}'\n[/]"); string json = JsonConvert.SerializeObject(blockedUsers, Formatting.Indented); await File.WriteAllTextAsync(OUTPUT_FILE, json); } @@ -919,27 +915,31 @@ public class Program do { DateTime startTime = DateTime.Now; - Dictionary users = new(); - Dictionary activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config); + Dictionary users = new(); - Log.Debug("Subscriptions: "); + AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); + Dictionary subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? []; - foreach (KeyValuePair activeSub in activeSubs) - { + Log.Debug("Subscriptions: "); + foreach (KeyValuePair activeSub in subsActive) + { if (!users.ContainsKey(activeSub.Key)) { users.Add(activeSub.Key, activeSub.Value); Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}"); } } + if (Config!.IncludeExpiredSubscriptions) { Log.Debug("Inactive Subscriptions: "); - Dictionary expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config); - foreach (KeyValuePair expiredSub in expiredSubs) - { - if (!users.ContainsKey(expiredSub.Key)) + AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); + Dictionary subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? []; + + foreach (KeyValuePair expiredSub in subsExpired) + { + if (!users.ContainsKey(expiredSub.Key)) { users.Add(expiredSub.Key, expiredSub.Value); Log.Debug($"Name: {expiredSub.Key} ID: {expiredSub.Value}"); @@ -962,9 +962,12 @@ public class Program var ignoredUsernames = await m_ApiHelper.GetListUsers($"/lists/{ignoredUsersListId}/users", Config) ?? []; users = users.Where(x => !ignoredUsernames.Contains(x.Key)).ToDictionary(x => x.Key, x => x.Value); } - } + } - await dBHelper.CreateUsersDB(users); + if (users.Count <= 0) + throw new InvalidOperationException("No users found!"); + + await dBHelper.CreateUsersDB(users); KeyValuePair> hasSelectedUsersKVP; if(Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab) {