Debug logging in BuildHeaderAndExecuteRequests

OF DL/Entities/Chats/ChatCollection.cs

@@ -0,0 +1,7 @@
+namespace OF_DL.Entities.Chats
+{
+    public class ChatCollection
+    {
+        public Dictionary<int, Chats.Chat> Chats { get; set; } = [];
+    }
+}

OF DL/Entities/Chats/Chats.cs

@@ -0,0 +1,20 @@
+namespace OF_DL.Entities.Chats
+{
+    public class Chats
+    {
+        public List<Chat> list { get; set; }
+        public bool hasMore { get; set; }
+        public int nextOffset { get; set; }
+
+        public class Chat
+        {
+            public User withUser { get; set; }
+            public int unreadMessagesCount { get; set; }
+        }
+
+        public class User
+        {
+            public int id { get; set; }
+        }
+    }
+}

OF DL/Entities/Config.cs

@@ -106,6 +106,8 @@ namespace OF_DL.Entities
 
         public string[] NonInteractiveSpecificUsers { get; set; } = [];
         public string[] NonInteractiveSpecificLists { get; set; } = [];
+
+        public bool OutputBlockedUsers { get; set; }
     }
 
     public class CreatorConfig : IFileNameFormatConfig

OF DL/Helpers/APIHelper.cs

@@ -2,6 +2,7 @@ using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using OF_DL.Entities;
 using OF_DL.Entities.Archived;
+using OF_DL.Entities.Chats;
 using OF_DL.Entities.Highlights;
 using OF_DL.Entities.Lists;
 using OF_DL.Entities.Messages;
@@ -13,6 +14,7 @@ using OF_DL.Enumerations;
 using OF_DL.Enumurations;
 using Serilog;
 using Spectre.Console;
+using System.Diagnostics;
 using System.Globalization;
 using System.Security.Cryptography;
 using System.Text;
@@ -25,6 +27,9 @@ namespace OF_DL.Helpers;
 
 public class APIHelper : IAPIHelper
 {
+    private const int MAX_RETRIES = 10;
+    private const int DELAY_BEFORE_RETRY = 1000;
+
     private static readonly JsonSerializerSettings m_JsonSerializerSettings;
     private readonly IDBHelper m_DBHelper;
     private readonly IDownloadConfig downloadConfig;
@@ -118,30 +123,58 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null, int retryCount = 0)
     {
-        Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", retryCount + 1);
 
-        HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
-        using var response = await client.SendAsync(request);
-        response.EnsureSuccessStatusCode();
-        string body = await response.Content.ReadAsStringAsync();
+        try
+        {
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
 
-        Log.Debug(body);
+            Debug.WriteLine($"Executing {request.Method.Method.ToUpper()} request: {request.RequestUri}\r\n\t{GetParamsString(getParams)}");
 
-        return body;
+            using var response = await client.SendAsync(request);
+
+            if (Debugger.IsAttached && !response.IsSuccessStatusCode)
+                Debugger.Break();
+
+            response.EnsureSuccessStatusCode();
+            string body = await response.Content.ReadAsStringAsync();
+
+            Log.Debug(body);
+
+            return body;
+        }
+        catch (HttpRequestException ex)
+        {
+            if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && retryCount < MAX_RETRIES)
+            {
+                await Task.Delay(DELAY_BEFORE_RETRY);
+                return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, method, ++retryCount);
+            }
+
+            throw;
+        }
+
+        static string GetParamsString(Dictionary<string, string> getParams)
+            => string.Join(" | ", getParams.Select(kv => $"{kv.Key}={kv.Value}"));
     }
 
 
-    private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint)
+    private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint, HttpMethod? method = null)
     {
         Log.Debug("Calling BuildHttpRequestMessage");
 
-        string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
+        method ??= HttpMethod.Get;
+
+        string queryParams = "";
+
+        if (getParams.Any())
+            queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
 
         Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
 
-        HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{queryParams}");
+        HttpRequestMessage request = new(method, $"{Constants.API_URL}{endpoint}{queryParams}");
 
         Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}");
 
@@ -298,47 +331,44 @@ public class APIHelper : IAPIHelper
         try
         {
             Dictionary<string, int> users = new();
-            Subscriptions subscriptions = new();
+
+            int limit = 25;
+            int offset = 0;
+
+            getParams["limit"] = limit.ToString();
+            getParams["offset"] = offset.ToString();
 
             Log.Debug("Calling GetAllSubscrptions");
 
-            string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
-
-            subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
-            if (subscriptions != null && subscriptions.hasMore)
+            while (true)
             {
-                getParams["offset"] = subscriptions.list.Count.ToString();
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
 
-                while (true)
+                if (string.IsNullOrWhiteSpace(body))
+                    break;
+
+                Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
+
+                if (subscriptions?.list is null)
+                    break;
+
+                foreach (Subscriptions.List item in subscriptions.list)
                 {
-                    Subscriptions newSubscriptions = new();
-                    string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
+                    if (users.ContainsKey(item.username))
+                        continue;
 
-                    if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
-                    {
-                        newSubscriptions = JsonConvert.DeserializeObject<Subscriptions>(loopbody, m_JsonSerializerSettings);
-                    }
-                    else
-                    {
-                        break;
-                    }
+                    bool isRestricted = item.isRestricted ?? false;
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
 
-                    subscriptions.list.AddRange(newSubscriptions.list);
-                    if (!newSubscriptions.hasMore)
-                    {
-                        break;
-                    }
-                    getParams["offset"] = subscriptions.list.Count.ToString();
+                    if (!isRestricted || isRestrictedButAllowed)
+                        users.Add(item.username, item.id);
                 }
-            }
 
-            foreach (Subscriptions.List subscription in subscriptions.list)
-            {
-                if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted))
-                    && !users.ContainsKey(subscription.username))
-                {
-                    users.Add(subscription.username, subscription.id);
-                }
+                if (!subscriptions.hasMore)
+                    break;
+
+                offset += limit;
+                getParams["offset"] = offset.ToString();
             }
 
             return users;
@@ -361,23 +391,20 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "active" },
             { "format", "infinite"}
         };
 
+        Log.Debug("Calling GetActiveSubscriptions");
+
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
 
 
     public async Task<Dictionary<string, int>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config)
     {
-
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
@@ -387,6 +414,18 @@ public class APIHelper : IAPIHelper
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
 
+    public async Task<Dictionary<string, int>?> GetBlockedUsers(string endpoint, IDownloadConfig config)
+    {
+        Dictionary<string, string> getParams = new()
+        {
+            { "type", "expired" },
+            { "format", "infinite"}
+        };
+
+        Log.Debug("Calling GetBlockedUsers");
+
+        return await GetAllSubscriptions(getParams, endpoint, true, config);
+    }
 
     public async Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config)
     {
@@ -1150,7 +1189,7 @@ public class APIHelper : IAPIHelper
                                     }
                                     break;
                                 case VideoResolution._240:
-                                    if(medium.videoSources != null)
+                                    if (medium.videoSources != null)
                                     {
                                         if (!string.IsNullOrEmpty(medium.videoSources._240))
                                         {
@@ -1177,7 +1216,7 @@ public class APIHelper : IAPIHelper
                                         }
                                     }
                                     break;
-                                
+
                             }
                         }
                         else if (medium.canView && medium.files != null && medium.files.drm != null)
@@ -2134,11 +2173,11 @@ public class APIHelper : IAPIHelper
                         {
                             JObject user = await GetUserInfoById($"/users/list?x[]={purchase.fromUser.id}");
 
-                            if(user is null)
+                            if (user is null)
                             {
                                 if (!config.BypassContentForCreatorsWhoNoLongerExist)
                                 {
-                                    if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
+                                    if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
                                     {
                                         purchasedTabUsers.Add($"Deleted User - {purchase.fromUser.id}", purchase.fromUser.id);
                                     }
@@ -2188,7 +2227,7 @@ public class APIHelper : IAPIHelper
                             {
                                 if (!config.BypassContentForCreatorsWhoNoLongerExist)
                                 {
-                                    if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
+                                    if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
                                     {
                                         purchasedTabUsers.Add($"Deleted User - {purchase.author.id}", purchase.author.id);
                                     }
@@ -2585,6 +2624,94 @@ public class APIHelper : IAPIHelper
         return null;
     }
 
+    public async Task<ChatCollection> GetChats(string endpoint, IDownloadConfig config, bool onlyUnread)
+    {
+        Log.Debug($"Calling GetChats - {endpoint}");
+
+        try
+        {
+            Chats chats = new();
+            ChatCollection collection = new();
+
+            int limit = 60;
+            Dictionary<string, string> getParams = new()
+            {
+                { "limit", $"{limit}" },
+                { "offset", "0" },
+                { "skip_users", "all" },
+                { "order", "recent" }
+            };
+
+            if (onlyUnread)
+                getParams["filter"] = "unread";
+
+            string body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            chats = JsonConvert.DeserializeObject<Chats>(body, m_JsonSerializerSettings);
+
+            if (chats.hasMore)
+            {
+                getParams["offset"] = $"{chats.nextOffset}";
+
+                while (true)
+                {
+                    string loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+                    Chats newChats = JsonConvert.DeserializeObject<Chats>(loopbody, m_JsonSerializerSettings);
+
+                    chats.list.AddRange(newChats.list);
+
+                    if (!newChats.hasMore)
+                        break;
+
+                    getParams["offset"] = $"{newChats.nextOffset}";
+                }
+            }
+
+            foreach (Chats.Chat chat in chats.list)
+                collection.Chats.Add(chat.withUser.id, chat);
+
+            return collection;
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            if (ex.InnerException != null)
+            {
+                Console.WriteLine("\nInner Exception:");
+                Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+                Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+            }
+        }
+
+        return null;
+    }
+
+    public async Task MarkAsUnread(string endpoint, IDownloadConfig config)
+    {
+        Log.Debug($"Calling MarkAsUnread - {endpoint}");
+
+        try
+        {
+            var result = new { success = false };
+
+            string body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(config), HttpMethod.Delete);
+            result = JsonConvert.DeserializeAnonymousType(body, result);
+
+            if (result?.success != true)
+                Console.WriteLine($"Failed to mark chat as unread! Endpoint: {endpoint}");
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            if (ex.InnerException != null)
+            {
+                Console.WriteLine("\nInner Exception:");
+                Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+                Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+            }
+        }
+    }
 
     public async Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp)
     {

OF DL/Program.cs

@@ -22,6 +22,7 @@ using Akka.Configuration;
 using System.Text;
 using static Akka.Actor.ProviderSelection;
 using System.Diagnostics;
+using OF_DL.Entities.Chats;
 
 namespace OF_DL;
 
@@ -513,6 +514,14 @@ public class Program
                     }
                 }
 
+                const string OUTPUT_BLOCKED_USERS_ARG = "--output-blocked";
+
+                if (args.Any(a => OUTPUT_BLOCKED_USERS_ARG.Equals(a, StringComparison.OrdinalIgnoreCase)))
+                {
+                    config.NonInteractiveMode = true;
+                    config.OutputBlockedUsers = true;
+                }
+
                 Log.Debug("Additional arguments:");
 				foreach (string argument in args)
 				{
@@ -848,6 +857,12 @@ public class Program
 
             try
             {
+                if (config.OutputBlockedUsers)
+                {
+                    await DownloadBlockedUsers(apiHelper, config);
+                    return;
+                }
+
                 await DownloadAllData(apiHelper, auth, config);
             }
             finally
@@ -874,8 +889,25 @@ public class Program
 		}
 	}
 
+    private static async Task DownloadBlockedUsers(APIHelper m_ApiHelper, Entities.Config Config)
+    {
+        const string OUTPUT_FILE = "blocked-users.json";
 
-	private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config)
+        Dictionary<string, int>? blockedUsers = await m_ApiHelper.GetBlockedUsers("/users/blocked", Config);
+
+        if (blockedUsers is null || blockedUsers.Count == 0)
+        {
+            AnsiConsole.Markup($"[green]No Blocked Users found.\n[/]");
+        }
+        else
+        {
+            AnsiConsole.Markup($"[green]Found {blockedUsers.Count} Blocked Users, saving to '{OUTPUT_FILE}'\n[/]");
+            string json = JsonConvert.SerializeObject(blockedUsers, Formatting.Indented);
+            await File.WriteAllTextAsync(OUTPUT_FILE, json);
+        }
+    }
+
+    private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config)
 	{
 		DBHelper dBHelper = new DBHelper(Config);
 
@@ -884,27 +916,31 @@ public class Program
 		do
 		{
 			DateTime startTime = DateTime.Now;
-			Dictionary<string, int> users = new();
-			Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
+            Dictionary<string, int> users = new();
 
-			Log.Debug("Subscriptions: ");
+            AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
+            Dictionary<string, int> subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
 
-			foreach (KeyValuePair<string, int> activeSub in activeSubs)
-			{
+            Log.Debug("Subscriptions: ");
+            foreach (KeyValuePair<string, int> activeSub in subsActive)
+            {
 				if (!users.ContainsKey(activeSub.Key))
 				{
 					users.Add(activeSub.Key, activeSub.Value);
 					Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}");
 				}
 			}
+
 			if (Config!.IncludeExpiredSubscriptions)
 			{
 				Log.Debug("Inactive Subscriptions: ");
 
-				Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
-				foreach (KeyValuePair<string, int> expiredSub in expiredSubs)
-				{
-					if (!users.ContainsKey(expiredSub.Key))
+                AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
+                Dictionary<string, int> subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
+
+                foreach (KeyValuePair<string, int> expiredSub in subsExpired)
+                {
+                    if (!users.ContainsKey(expiredSub.Key))
 					{
 						users.Add(expiredSub.Key, expiredSub.Value);
 						Log.Debug($"Name: {expiredSub.Key} ID: {expiredSub.Value}");
@@ -927,9 +963,12 @@ public class Program
 					var ignoredUsernames = await m_ApiHelper.GetListUsers($"/lists/{ignoredUsersListId}/users", Config) ?? [];
 					users = users.Where(x => !ignoredUsernames.Contains(x.Key)).ToDictionary(x => x.Key, x => x.Value);
 				}
-			}
+            }
 
-			await dBHelper.CreateUsersDB(users);
+            if (users.Count <= 0)
+                throw new InvalidOperationException("No users found!");
+
+            await dBHelper.CreateUsersDB(users);
 			KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP;
 			if(Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab)
 			{
@@ -1497,6 +1536,9 @@ public class Program
 	{
 		Log.Debug($"Calling DownloadMessages - {user.Key}");
 
+        AnsiConsole.Markup($"[grey]Getting Unread Chats\n[/]");
+        HashSet<int> unreadChats = await GetUsersWithUnreadChats(downloadContext.ApiHelper, downloadContext.DownloadConfig);
+
         MessageCollection messages = new MessageCollection();
 
         await AnsiConsole.Status()
@@ -1504,7 +1546,14 @@ public class Program
         {
             messages = await downloadContext.ApiHelper.GetMessages($"/chats/{user.Value}/messages", path, downloadContext.DownloadConfig!, ctx);
         });
-		int oldMessagesCount = 0;
+
+        if (unreadChats.Contains(user.Value))
+        {
+            AnsiConsole.Markup($"[grey]Restoring unread state\n[/]");
+            await downloadContext.ApiHelper.MarkAsUnread($"/chats/{user.Value}/mark-as-read", downloadContext.DownloadConfig);
+        }
+
+        int oldMessagesCount = 0;
 		int newMessagesCount = 0;
 		if (messages != null && messages.Messages.Count > 0)
 		{
@@ -3253,7 +3302,18 @@ public class Program
 		}
 	}
 
-	static bool ValidateFilePath(string path)
+    private static async Task<HashSet<int>> GetUsersWithUnreadChats(APIHelper apiHelper, IDownloadConfig currentConfig)
+    {
+        ChatCollection chats = await apiHelper.GetChats($"/chats", currentConfig, onlyUnread: true);
+
+        var unreadChats = chats.Chats
+            .Where(c => c.Value.unreadMessagesCount > 0)
+            .ToList();
+
+        return [.. unreadChats.Select(c => c.Key)];
+    }
+
+    static bool ValidateFilePath(string path)
 	{
 		char[] invalidChars = System.IO.Path.GetInvalidPathChars();
 		char[] foundInvalidChars = path.Where(c => invalidChars.Contains(c)).ToArray();

Publish_OF-DL.bat

@@ -1,7 +1,33 @@
 @ECHO OFF
 
 ECHO.
-dotnet publish ".\OF DL\OF DL.csproj" -o ".\Publish"
+
+ECHO ==============================
+ECHO == Cleaning Output ===========
+ECHO ==============================
+dotnet clean ".\OF DL\OF DL.csproj" -v minimal
+DEL /Q /F ".\Publish"
 
+ECHO.
+
+ECHO ==============================
+ECHO == Publishing OF-DL ==========
+ECHO ==============================
+dotnet publish ".\OF DL\OF DL.csproj" -o ".\Publish" -c Debug
+
+ECHO.
+
+ECHO ==============================
+ECHO == Copy to network drive? ====
+ECHO ==============================
+CHOICE /C yn /m "Copy published files to network drive? "
+
+IF %ERRORLEVEL%==1 (GOTO Copy) ELSE (GOTO Exit)
+
+:Copy
+xcopy .\Publish\* p:\_Utils\OF_DL /I /Y /Q /EXCLUDE:.\excludes.txt
+
+:Exit
+ECHO.
 ECHO.
 PAUSE

excludes.txt

@@ -0,0 +1,2 @@
+excludes.txt
+rules.json