Added safe-guard against paid content returned for other models

Tweaked order to fully replace users before updating DB.

.editorconfig

@@ -5,5 +5,12 @@ root = true
 charset = utf-8
 indent_style = space
 indent_size = 4
+tab_width = 4
+end_of_line = crlf
 insert_final_newline = true
 trim_trailing_whitespace = true
+
+[*.{sln,csproj,xml,json,config}]
+indent_size = 2
+indent_style = space
+tab_width = 2

.gitignore

@@ -370,4 +370,6 @@ FodyWeavers.xsd
 !.gitea-actions/**/node_modules/
 
 # venv
-venv/
+venv/
+
+Publish/

OF DL/Entities/Chats/ChatCollection.cs

@@ -0,0 +1,7 @@
+namespace OF_DL.Entities.Chats
+{
+    public class ChatCollection
+    {
+        public Dictionary<int, Chats.Chat> Chats { get; set; } = [];
+    }
+}

OF DL/Entities/Chats/Chats.cs

@@ -0,0 +1,20 @@
+namespace OF_DL.Entities.Chats
+{
+    public class Chats
+    {
+        public List<Chat> list { get; set; }
+        public bool hasMore { get; set; }
+        public int nextOffset { get; set; }
+
+        public class Chat
+        {
+            public User withUser { get; set; }
+            public int unreadMessagesCount { get; set; }
+        }
+
+        public class User
+        {
+            public int id { get; set; }
+        }
+    }
+}

OF DL/Entities/Config.cs

@@ -103,6 +103,15 @@ namespace OF_DL.Entities
 
         [JsonConverter(typeof(StringEnumConverter))]
         public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
+
+        // When enabled, post/message text is stored as-is without XML stripping.
+        [ToggleableConfig]
+        public bool DisableTextSanitization { get; set; } = false;
+
+        public string[] NonInteractiveSpecificUsers { get; set; } = [];
+        public string[] NonInteractiveSpecificLists { get; set; } = [];
+
+        public bool OutputBlockedUsers { get; set; }
     }
 
     public class CreatorConfig : IFileNameFormatConfig

OF DL/Entities/Subscriptions.cs

@@ -98,6 +98,7 @@ namespace OF_DL.Entities
             public object id { get; set; }
             public int? userId { get; set; }
             public int? subscriberId { get; set; }
+            public long? earningId { get; set; }
             public DateTime? date { get; set; }
             public int? duration { get; set; }
             public DateTime? startDate { get; set; }

OF DL/Helpers/APIHelper.cs

@@ -2,6 +2,7 @@ using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using OF_DL.Entities;
 using OF_DL.Entities.Archived;
+using OF_DL.Entities.Chats;
 using OF_DL.Entities.Highlights;
 using OF_DL.Entities.Lists;
 using OF_DL.Entities.Messages;
@@ -13,6 +14,7 @@ using OF_DL.Enumerations;
 using OF_DL.Enumurations;
 using Serilog;
 using Spectre.Console;
+using System.Diagnostics;
 using System.Globalization;
 using System.Security.Cryptography;
 using System.Text;
@@ -25,10 +27,15 @@ namespace OF_DL.Helpers;
 
 public class APIHelper : IAPIHelper
 {
+    private const int MAX_RETRIES = 10;
+    private const int DELAY_BEFORE_RETRY = 1000;
+
     private static readonly JsonSerializerSettings m_JsonSerializerSettings;
     private readonly IDBHelper m_DBHelper;
     private readonly IDownloadConfig downloadConfig;
     private readonly Auth auth;
+    private HttpClient httpClient = new();
+
     private static DateTime? cachedDynamicRulesExpiration;
     private static DynamicRules? cachedDynamicRules;
 
@@ -116,30 +123,58 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null, int retryCount = 0)
     {
-        Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", retryCount + 1);
 
-        HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
+        try
-        using var response = await client.SendAsync(request);
+        {
-        response.EnsureSuccessStatusCode();
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
-        string body = await response.Content.ReadAsStringAsync();
 
-        Log.Debug(body);
+            Debug.WriteLine($"Executing {request.Method.Method.ToUpper()} request: {request.RequestUri}\r\n\t{GetParamsString(getParams)}");
 
-        return body;
+            using var response = await client.SendAsync(request);
+
+            if (Debugger.IsAttached && !response.IsSuccessStatusCode)
+                Debugger.Break();
+
+            response.EnsureSuccessStatusCode();
+            string body = await response.Content.ReadAsStringAsync();
+
+            Log.Debug(body);
+
+            return body;
+        }
+        catch (HttpRequestException ex)
+        {
+            if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && retryCount < MAX_RETRIES)
+            {
+                await Task.Delay(DELAY_BEFORE_RETRY);
+                return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, method, ++retryCount);
+            }
+
+            throw;
+        }
+
+        static string GetParamsString(Dictionary<string, string> getParams)
+            => string.Join(" | ", getParams.Select(kv => $"{kv.Key}={kv.Value}"));
     }
 
 
-    private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint)
+    private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint, HttpMethod? method = null)
     {
         Log.Debug("Calling BuildHttpRequestMessage");
 
-        string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
+        method ??= HttpMethod.Get;
+
+        string queryParams = "";
+
+        if (getParams.Any())
+            queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
 
         Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
 
-        HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{queryParams}");
+        HttpRequestMessage request = new(method, $"{Constants.API_URL}{endpoint}{queryParams}");
 
         Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}");
 
@@ -164,18 +199,16 @@ public class APIHelper : IAPIHelper
         return input.All(char.IsDigit);
     }
 
-
+    private HttpClient GetHttpClient(IDownloadConfig? config = null)
-    private static HttpClient GetHttpClient(IDownloadConfig? config = null)
     {
-        var client = new HttpClient();
+        httpClient ??= new HttpClient();
         if (config?.Timeout != null && config.Timeout > 0)
         {
-            client.Timeout = TimeSpan.FromSeconds(config.Timeout.Value);
+            httpClient.Timeout = TimeSpan.FromSeconds(config.Timeout.Value);
         }
-        return client;
+        return httpClient;
     }
 
-
     /// <summary>
     /// this one is used during initialization only
     /// if the config option is not available then no modificatiotns will be done on the getParams
@@ -298,47 +331,44 @@ public class APIHelper : IAPIHelper
         try
         {
             Dictionary<string, int> users = new();
-            Subscriptions subscriptions = new();
+
+            int limit = 25;
+            int offset = 0;
+
+            getParams["limit"] = limit.ToString();
+            getParams["offset"] = offset.ToString();
 
             Log.Debug("Calling GetAllSubscrptions");
 
-            string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            while (true)
-
-            subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
-            if (subscriptions != null && subscriptions.hasMore)
             {
-                getParams["offset"] = subscriptions.list.Count.ToString();
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
 
-                while (true)
+                if (string.IsNullOrWhiteSpace(body))
+                    break;
+
+                Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
+
+                if (subscriptions?.list is null)
+                    break;
+
+                foreach (Subscriptions.List item in subscriptions.list)
                 {
-                    Subscriptions newSubscriptions = new();
+                    if (users.ContainsKey(item.username))
-                    string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+                        continue;
 
-                    if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
+                    bool isRestricted = item.isRestricted ?? false;
-                    {
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
-                        newSubscriptions = JsonConvert.DeserializeObject<Subscriptions>(loopbody, m_JsonSerializerSettings);
-                    }
-                    else
-                    {
-                        break;
-                    }
 
-                    subscriptions.list.AddRange(newSubscriptions.list);
+                    if (!isRestricted || isRestrictedButAllowed)
-                    if (!newSubscriptions.hasMore)
+                        users.Add(item.username, item.id);
-                    {
-                        break;
-                    }
-                    getParams["offset"] = subscriptions.list.Count.ToString();
                 }
-            }
 
-            foreach (Subscriptions.List subscription in subscriptions.list)
+                if (!subscriptions.hasMore)
-            {
+                    break;
-                if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted))
+
-                    && !users.ContainsKey(subscription.username))
+                offset += limit;
-                {
+                getParams["offset"] = offset.ToString();
-                    users.Add(subscription.username, subscription.id);
-                }
             }
 
             return users;
@@ -361,23 +391,20 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "active" },
             { "format", "infinite"}
         };
 
+        Log.Debug("Calling GetActiveSubscriptions");
+
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
 
 
     public async Task<Dictionary<string, int>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config)
     {
-
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
@@ -387,6 +414,86 @@ public class APIHelper : IAPIHelper
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
 
+    public async Task<Dictionary<string, int>?> GetUsersWithProgress(string typeDisplay, string endpoint, StatusContext ctx, string? typeParam, bool offsetByCount)
+    {
+        int limit = 50;
+        int offset = 0;
+        bool includeRestricted = true;
+
+        Dictionary<string, string> getParams = new()
+        {
+            ["format"] = "infinite",
+            ["limit"] = limit.ToString(),
+            ["offset"] = offset.ToString()
+        };
+
+        if (!string.IsNullOrWhiteSpace(typeParam))
+            getParams["type"] = typeParam;
+
+        try
+        {
+            Dictionary<string, int> users = [];
+
+            Log.Debug("Calling GetUsersWithProgress");
+
+            bool isLastLoop = false;
+            while (true)
+            {
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
+
+                if (string.IsNullOrWhiteSpace(body))
+                    break;
+
+                Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
+
+                if (subscriptions?.list is null)
+                    break;
+
+                foreach (Subscriptions.List item in subscriptions.list)
+                {
+                    if (users.ContainsKey(item.username))
+                        continue;
+
+                    bool isRestricted = item.isRestricted ?? false;
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
+
+                    if (!isRestricted || isRestrictedButAllowed)
+                        users.Add(item.username, item.id);
+                }
+
+                ctx.Status($"[red]Getting {typeDisplay} Users\n[/] [red]Found {users.Count}[/]");
+                ctx.Spinner(Spinner.Known.Dots);
+                ctx.SpinnerStyle(Style.Parse("blue"));
+
+                if (isLastLoop)
+                    break;
+
+                if (!subscriptions.hasMore || subscriptions.list.Count == 0)
+                    isLastLoop = true;
+
+                offset += offsetByCount
+                    ? subscriptions.list.Count
+                    : limit;
+
+                getParams["offset"] = offset.ToString();
+            }
+
+            return users;
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            if (ex.InnerException != null)
+            {
+                Console.WriteLine("\nInner Exception:");
+                Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+                Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+            }
+        }
+
+        return null;
+    }
 
     public async Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config)
     {
@@ -405,7 +512,7 @@ public class APIHelper : IAPIHelper
             Dictionary<string, int> lists = new();
             while (true)
             {
-                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
 
                 if (body == null)
                 {
@@ -464,13 +571,13 @@ public class APIHelper : IAPIHelper
             Dictionary<string, string> getParams = new()
             {
                 { "offset", offset.ToString() },
-                { "limit", "50" }
+                { "limit", "50" },
             };
             List<string> users = new();
 
             while (true)
             {
-                var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+                var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
                 if (body == null)
                 {
                     break;
@@ -514,6 +621,66 @@ public class APIHelper : IAPIHelper
     }
 
 
+    public async Task<Dictionary<string, int>?> GetUsersFromList(string endpoint, bool includeRestricted, IDownloadConfig config)
+    {
+        var model = new { list = new[] { new { id = int.MaxValue, username = string.Empty, isRestricted = false, isBlocked = false } }, nextOffset = 0, hasMore = false };
+
+        Log.Debug($"Calling GetUsersFromList - {endpoint}");
+
+        int limit = 50;
+        int offset = 0;
+
+        Dictionary<string, string> getParams = new()
+        {
+            { "offset", offset.ToString() },
+            { "limit", limit.ToString() },
+            { "format", "infinite" }
+        };
+
+        try
+        {
+            Dictionary<string, int> users = [];
+
+            while (true)
+            {
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+
+                if (string.IsNullOrWhiteSpace(body))
+                    break;
+
+                var data = JsonConvert.DeserializeAnonymousType(body, model, m_JsonSerializerSettings);
+
+                if (data is null)
+                    break;
+
+                foreach (var item in data.list)
+                {
+                    if (users.ContainsKey(item.username))
+                        continue;
+
+                    bool isRestricted = item.isRestricted;
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
+
+                    if (!isRestricted || isRestrictedButAllowed)
+                        users.Add(item.username, item.id);
+                }
+
+                if (!data.hasMore)
+                    break;
+
+
+                offset += data.nextOffset;
+                getParams["offset"] = offset.ToString();
+            }
+
+            return users;
+        }
+        catch (Exception ex)
+        {
+            throw;
+        }
+    }
+
     public async Task<Dictionary<long, string>> GetMedia(MediaType mediatype,
                                                          string endpoint,
                                                          string? username,
@@ -540,7 +707,8 @@ public class APIHelper : IAPIHelper
                     getParams = new Dictionary<string, string>
                     {
                         { "limit", post_limit.ToString() },
-                        { "order", "publish_date_desc" }
+                        { "order", "publish_date_desc" },
+                        { "skip_users", "all" }
                     };
                     break;
 
@@ -548,12 +716,13 @@ public class APIHelper : IAPIHelper
                     getParams = new Dictionary<string, string>
                     {
                         { "limit", limit.ToString() },
-                        { "offset", offset.ToString() }
+                        { "offset", offset.ToString() },
+                        { "skip_users", "all" }
                     };
                     break;
             }
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
 
 
             if (mediatype == MediaType.Stories)
@@ -726,7 +895,7 @@ public class APIHelper : IAPIHelper
     }
 
 
-    public async Task<PaidPostCollection> GetPaidPosts(string endpoint, string folder, string username, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx)
+    public async Task<PaidPostCollection> GetPaidPosts(string endpoint, string folder, string username, int userId, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx)
     {
         Log.Debug($"Calling GetPaidPosts - {username}");
 
@@ -735,12 +904,13 @@ public class APIHelper : IAPIHelper
             Purchased paidPosts = new();
             PaidPostCollection paidPostCollection = new();
             int post_limit = 50;
+            int offset = 0;
             Dictionary<string, string> getParams = new()
             {
                 { "limit", post_limit.ToString() },
-                { "order", "publish_date_desc" },
+                { "skip_users", "all" },
                 { "format", "infinite" },
-                { "user_id", username }
+                { "author", username },
             };
 
             var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
@@ -750,9 +920,10 @@ public class APIHelper : IAPIHelper
             ctx.SpinnerStyle(Style.Parse("blue"));
             if (paidPosts != null && paidPosts.hasMore)
             {
-                getParams["offset"] = paidPosts.list.Count.ToString();
                 while (true)
                 {
+                    offset += post_limit;
+                    getParams["offset"] = offset.ToString();
 
                     Purchased newPaidPosts = new();
 
@@ -767,7 +938,6 @@ public class APIHelper : IAPIHelper
                     {
                         break;
                     }
-                    getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit);
                 }
 
             }
@@ -776,6 +946,9 @@ public class APIHelper : IAPIHelper
             {
                 if (purchase.responseType == "post" && purchase.media != null && purchase.media.Count > 0)
                 {
+                    if (purchase.fromUser.id != userId)
+                        continue; // Ensures only posts from current model are included
+
                     List<long> previewids = new();
                     if (purchase.previews != null)
                     {
@@ -906,7 +1079,8 @@ public class APIHelper : IAPIHelper
             {
                 { "limit", post_limit.ToString() },
                 { "order", "publish_date_desc" },
-                { "format", "infinite" }
+                { "format", "infinite" },
+                { "skip_users", "all" }
             };
 
             Enumerations.DownloadDateSelection downloadDateSelection = Enumerations.DownloadDateSelection.before;
@@ -932,7 +1106,7 @@ public class APIHelper : IAPIHelper
                 ref getParams,
                 downloadAsOf);
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
             posts = JsonConvert.DeserializeObject<Post>(body, m_JsonSerializerSettings);
             ctx.Status($"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.list.Count}[/]");
             ctx.Spinner(Spinner.Known.Dots);
@@ -1090,7 +1264,7 @@ public class APIHelper : IAPIHelper
                 { "skip_users", "all" }
             };
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
             singlePost = JsonConvert.DeserializeObject<SinglePost>(body, m_JsonSerializerSettings);
 
             if (singlePost != null)
@@ -1150,7 +1324,7 @@ public class APIHelper : IAPIHelper
                                     }
                                     break;
                                 case VideoResolution._240:
-                                    if(medium.videoSources != null)
+                                    if (medium.videoSources != null)
                                     {
                                         if (!string.IsNullOrEmpty(medium.videoSources._240))
                                         {
@@ -1177,7 +1351,7 @@ public class APIHelper : IAPIHelper
                                         }
                                     }
                                     break;
-                                
+
                             }
                         }
                         else if (medium.canView && medium.files != null && medium.files.drm != null)
@@ -1237,7 +1411,8 @@ public class APIHelper : IAPIHelper
             {
                 { "limit", post_limit.ToString() },
                 { "order", "publish_date_desc" },
-                { "format", "infinite" }
+                { "format", "infinite" },
+                { "skip_users", "all" }
             };
 
             Enumerations.DownloadDateSelection downloadDateSelection = Enumerations.DownloadDateSelection.before;
@@ -1251,7 +1426,7 @@ public class APIHelper : IAPIHelper
                 ref getParams,
                 config.CustomDate);
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
             streams = JsonConvert.DeserializeObject<Streams>(body, m_JsonSerializerSettings);
             ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.list.Count}[/]");
             ctx.Spinner(Spinner.Known.Dots);
@@ -1524,7 +1699,8 @@ public class APIHelper : IAPIHelper
             Dictionary<string, string> getParams = new()
             {
                 { "limit", post_limit.ToString() },
-                { "order", "desc" }
+                { "order", "desc" },
+                { "skip_users", "all" },
             };
 
             var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
@@ -1534,9 +1710,10 @@ public class APIHelper : IAPIHelper
             ctx.SpinnerStyle(Style.Parse("blue"));
             if (messages.hasMore)
             {
-                getParams["id"] = messages.list[^1].id.ToString();
                 while (true)
                 {
+                    getParams["id"] = messages.list[^1].id.ToString();
+
                     Messages newmessages = new();
 
                     var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
@@ -1550,7 +1727,6 @@ public class APIHelper : IAPIHelper
                     {
                         break;
                     }
-                    getParams["id"] = newmessages.list[newmessages.list.Count - 1].id.ToString();
                 }
             }
 
@@ -1826,7 +2002,7 @@ public class APIHelper : IAPIHelper
     }
 
 
-    public async Task<PaidMessageCollection> GetPaidMessages(string endpoint, string folder, string username, IDownloadConfig config, StatusContext ctx)
+    public async Task<PaidMessageCollection> GetPaidMessages(string endpoint, string folder, string username, int userId, IDownloadConfig config, StatusContext ctx)
     {
         Log.Debug($"Calling GetPaidMessages - {username}");
 
@@ -1835,12 +2011,14 @@ public class APIHelper : IAPIHelper
             Purchased paidMessages = new();
             PaidMessageCollection paidMessageCollection = new();
             int post_limit = 50;
+            int offset = 0;
             Dictionary<string, string> getParams = new()
             {
                 { "limit", post_limit.ToString() },
-                { "order", "publish_date_desc" },
+                { "skip_users", "all" },
                 { "format", "infinite" },
-                { "user_id", username }
+                { "offset", offset.ToString() },
+                { "author", username },
             };
 
             var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
@@ -1850,9 +2028,11 @@ public class APIHelper : IAPIHelper
             ctx.SpinnerStyle(Style.Parse("blue"));
             if (paidMessages != null && paidMessages.hasMore)
             {
-                getParams["offset"] = paidMessages.list.Count.ToString();
                 while (true)
                 {
+                    offset += post_limit;
+                    getParams["offset"] = offset.ToString();
+
                     string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
                     Purchased newpaidMessages = new();
                     Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
@@ -1864,12 +2044,14 @@ public class APIHelper : IAPIHelper
                     {
                         looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
                     }
+
                     using (var loopresponse = await loopclient.SendAsync(looprequest))
                     {
                         loopresponse.EnsureSuccessStatusCode();
                         var loopbody = await loopresponse.Content.ReadAsStringAsync();
                         newpaidMessages = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
                     }
+
                     paidMessages.list.AddRange(newpaidMessages.list);
                     ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.list.Count}[/]");
                     ctx.Spinner(Spinner.Known.Dots);
@@ -1878,16 +2060,21 @@ public class APIHelper : IAPIHelper
                     {
                         break;
                     }
-                    getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit);
                 }
             }
 
             if (paidMessages.list != null && paidMessages.list.Count > 0)
             {
+                int ownUserId = Convert.ToInt32(auth.USER_ID);
+                int[] validUserIds = [ownUserId, userId];
+
                 foreach (Purchased.List purchase in paidMessages.list.Where(p => p.responseType == "message").OrderByDescending(p => p.postedAt ?? p.createdAt))
                 {
-                    if (!config.IgnoreOwnMessages || purchase.fromUser.id != Convert.ToInt32(auth.USER_ID))
+                    if (!config.IgnoreOwnMessages || purchase.fromUser.id != ownUserId)
                     {
+                        if (!validUserIds.Contains(purchase.fromUser.id))
+                            continue; // Ensures only messages from current model (or self) are included
+
                         if (purchase.postedAt != null)
                         {
                             await m_DBHelper.AddMessage(folder, purchase.id, purchase.text != null ? purchase.text : string.Empty, purchase.price != null ? purchase.price : "0", true, false, purchase.postedAt.Value, purchase.fromUser.id);
@@ -2134,11 +2321,11 @@ public class APIHelper : IAPIHelper
                         {
                             JObject user = await GetUserInfoById($"/users/list?x[]={purchase.fromUser.id}");
 
-                            if(user is null)
+                            if (user is null)
                             {
                                 if (!config.BypassContentForCreatorsWhoNoLongerExist)
                                 {
-                                    if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
+                                    if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
                                     {
                                         purchasedTabUsers.Add($"Deleted User - {purchase.fromUser.id}", purchase.fromUser.id);
                                     }
@@ -2188,7 +2375,7 @@ public class APIHelper : IAPIHelper
                             {
                                 if (!config.BypassContentForCreatorsWhoNoLongerExist)
                                 {
-                                    if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
+                                    if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
                                     {
                                         purchasedTabUsers.Add($"Deleted User - {purchase.author.id}", purchase.author.id);
                                     }
@@ -2585,6 +2772,94 @@ public class APIHelper : IAPIHelper
         return null;
     }
 
+    public async Task<ChatCollection> GetChats(string endpoint, IDownloadConfig config, bool onlyUnread)
+    {
+        Log.Debug($"Calling GetChats - {endpoint}");
+
+        try
+        {
+            Chats chats = new();
+            ChatCollection collection = new();
+
+            int limit = 60;
+            Dictionary<string, string> getParams = new()
+            {
+                { "limit", $"{limit}" },
+                { "offset", "0" },
+                { "skip_users", "all" },
+                { "order", "recent" }
+            };
+
+            if (onlyUnread)
+                getParams["filter"] = "unread";
+
+            string body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            chats = JsonConvert.DeserializeObject<Chats>(body, m_JsonSerializerSettings);
+
+            if (chats.hasMore)
+            {
+                getParams["offset"] = $"{chats.nextOffset}";
+
+                while (true)
+                {
+                    string loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+                    Chats newChats = JsonConvert.DeserializeObject<Chats>(loopbody, m_JsonSerializerSettings);
+
+                    chats.list.AddRange(newChats.list);
+
+                    if (!newChats.hasMore)
+                        break;
+
+                    getParams["offset"] = $"{newChats.nextOffset}";
+                }
+            }
+
+            foreach (Chats.Chat chat in chats.list)
+                collection.Chats.Add(chat.withUser.id, chat);
+
+            return collection;
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            if (ex.InnerException != null)
+            {
+                Console.WriteLine("\nInner Exception:");
+                Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+                Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+            }
+        }
+
+        return null;
+    }
+
+    public async Task MarkAsUnread(string endpoint, IDownloadConfig config)
+    {
+        Log.Debug($"Calling MarkAsUnread - {endpoint}");
+
+        try
+        {
+            var result = new { success = false };
+
+            string body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(config), HttpMethod.Delete);
+            result = JsonConvert.DeserializeAnonymousType(body, result);
+
+            if (result?.success != true)
+                Console.WriteLine($"Failed to mark chat as unread! Endpoint: {endpoint}");
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            if (ex.InnerException != null)
+            {
+                Console.WriteLine("\nInner Exception:");
+                Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+                Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+            }
+        }
+    }
 
     public async Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp)
     {
@@ -2703,7 +2978,7 @@ public class APIHelper : IAPIHelper
 
                 using var response = await client.SendAsync(request);
 
-                Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}");
+                Log.Debug($"CDRM Project Response (Attempt {attempt}): {await response.Content.ReadAsStringAsync()}");
 
                 response.EnsureSuccessStatusCode();
                 var body = await response.Content.ReadAsStringAsync();
@@ -2805,11 +3080,11 @@ public class APIHelper : IAPIHelper
 
         try
         {
-            var resp1 = PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 });
+            var resp1 = await PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 });
             var certDataB64 = Convert.ToBase64String(resp1);
             var cdm = new CDMApi();
             var challenge = cdm.GetChallenge(pssh, certDataB64, false, false);
-            var resp2 = PostData(licenceURL, drmHeaders, challenge);
+            var resp2 = await PostData(licenceURL, drmHeaders, challenge);
             var licenseB64 = Convert.ToBase64String(resp2);
             Log.Debug($"resp1: {resp1}");
             Log.Debug($"certDataB64: {certDataB64}");

OF DL/Helpers/Constants.cs

@@ -3,4 +3,7 @@ namespace OF_DL.Helpers;
 public static class Constants
 {
     public const string API_URL = "https://onlyfans.com/api2/v2";
+
+    public const int WIDEVINE_RETRY_DELAY = 10;
+    public const int WIDEVINE_MAX_RETRIES = 3;
 }

OF DL/Helpers/DBHelper.cs

@@ -1,18 +1,14 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using OF_DL.Enumurations;
-using System.IO;
 using Microsoft.Data.Sqlite;
-using Serilog;
 using OF_DL.Entities;
+using Serilog;
+using System.Text;
 
 namespace OF_DL.Helpers
 {
     public class DBHelper : IDBHelper
     {
+        private static readonly Dictionary<string, SqliteConnection> _connections = [];
+
         private readonly IDownloadConfig downloadConfig;
 
         public DBHelper(IDownloadConfig downloadConfig)
@@ -32,9 +28,7 @@ namespace OF_DL.Helpers
                 string dbFilePath = $"{folder}/Metadata/user_data.db";
 
                 // connect to the new database file
-                using SqliteConnection connection = new($"Data Source={dbFilePath}");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={dbFilePath}");
-                // open the connection
-                connection.Open();
 
                 // create the 'medias' table
                 using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection))
@@ -139,11 +133,9 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
                 Log.Debug("Database data source: " + connection.DataSource);
 
-                connection.Open();
-
                 using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection))
                 {
                     await cmd.ExecuteNonQueryAsync();
@@ -194,9 +186,7 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
-
-                connection.Open();
 
                 using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection))
                 {
@@ -247,8 +237,8 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-                connection.Open();
+
                 await EnsureCreatedAtColumnExists(connection, "messages");
                 using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection);
                 cmd.Parameters.AddWithValue("@post_id", post_id);
@@ -286,8 +276,8 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-                connection.Open();
+
                 await EnsureCreatedAtColumnExists(connection, "posts");
                 using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection);
                 cmd.Parameters.AddWithValue("@post_id", post_id);
@@ -324,8 +314,8 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-                connection.Open();
+
                 await EnsureCreatedAtColumnExists(connection, "stories");
                 using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection);
                 cmd.Parameters.AddWithValue("@post_id", post_id);
@@ -362,8 +352,8 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-                connection.Open();
+
                 await EnsureCreatedAtColumnExists(connection, "medias");
                 StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id");
                 if (downloadConfig.DownloadDuplicatedMedia)
@@ -400,22 +390,21 @@ namespace OF_DL.Helpers
         {
             try
             {
-                bool downloaded = false;
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
 
-                using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
+                StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id");
+                if (downloadConfig.DownloadDuplicatedMedia)
                 {
-                    StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id");
+                    sql.Append(" and api_type=@api_type");
-                    if(downloadConfig.DownloadDuplicatedMedia)
-                    {
-                        sql.Append(" and api_type=@api_type");
-                    }
-
-                    connection.Open();
-                    using SqliteCommand cmd = new (sql.ToString(), connection);
-                    cmd.Parameters.AddWithValue("@media_id", media_id);
-                    cmd.Parameters.AddWithValue("@api_type", api_type);
-                    downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
                 }
+
+                connection.Open();
+                using SqliteCommand cmd = new(sql.ToString(), connection);
+                cmd.Parameters.AddWithValue("@media_id", media_id);
+                cmd.Parameters.AddWithValue("@api_type", api_type);
+
+                bool downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
+
                 return downloaded;
             }
             catch (Exception ex)
@@ -435,8 +424,7 @@ namespace OF_DL.Helpers
 
         public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at)
         {
-            using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+            SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-            connection.Open();
 
             // Construct the update command
             StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id");
@@ -463,25 +451,21 @@ namespace OF_DL.Helpers
 
         public async Task<long> GetStoredFileSize(string folder, long media_id, string api_type)
         {
-            long size;
+            SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-            using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
+
-            {
+            using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection);
-                connection.Open();
+            cmd.Parameters.AddWithValue("@media_id", media_id);
-                using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection);
+            cmd.Parameters.AddWithValue("@api_type", api_type);
-                cmd.Parameters.AddWithValue("@media_id", media_id);
+
-                cmd.Parameters.AddWithValue("@api_type", api_type);
+            long size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
-                size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
-            }
             return size;
         }
 
         public async Task<DateTime?> GetMostRecentPostDate(string folder)
         {
-            DateTime? mostRecentDate = null;
+            SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-            using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
+
-            {
+            using SqliteCommand cmd = new(@"
-                connection.Open();
-                using SqliteCommand cmd = new(@"
                     SELECT
 	                    MIN(created_at) AS created_at
                     FROM  (
@@ -497,13 +481,14 @@ namespace OF_DL.Helpers
 				                    ON P.post_id = m.post_id
 		                    WHERE m.downloaded = 0 
 	                      )", connection);
-                var scalarValue = await cmd.ExecuteScalarAsync();
+
-                if(scalarValue != null && scalarValue != DBNull.Value)
+            var scalarValue = await cmd.ExecuteScalarAsync();
-                {
+            if (scalarValue != null && scalarValue != DBNull.Value)
-                    mostRecentDate = Convert.ToDateTime(scalarValue);
+            {
-                }
+                return Convert.ToDateTime(scalarValue);
             }
-            return mostRecentDate;
+
+            return null;
         }
 
         private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName)
@@ -527,5 +512,35 @@ namespace OF_DL.Helpers
                 await alterCmd.ExecuteNonQueryAsync();
             }
         }
+
+        public static void CloseAllConnections()
+        {
+            foreach (SqliteConnection cn in _connections.Values)
+            {
+                cn?.Close();
+                cn?.Dispose();
+            }
+
+            _connections.Clear();
+        }
+
+        private static async Task<SqliteConnection> GetAndOpenConnectionAsync(string connectionString, int numberOfRetries = 2)
+        {
+            try
+            {
+                SqliteConnection connection = new(connectionString);
+                connection.Open();
+
+                return connection;
+            }
+            catch (Exception)
+            {
+                if (--numberOfRetries <= 0)
+                    throw;
+
+                await Task.Delay(300);
+                return await GetAndOpenConnectionAsync(connectionString, numberOfRetries);
+            }
+        }
     }
 }

OF DL/Helpers/DownloadHelper.cs

@@ -855,7 +855,7 @@ public class DownloadHelper : IDownloadHelper
                 memoryStream.Seek(0, SeekOrigin.Begin);
 
                 MD5 md5 = MD5.Create();
-                byte[] hash = md5.ComputeHash(memoryStream);
+                byte[] hash = await md5.ComputeHashAsync(memoryStream);
                 memoryStream.Seek(0, SeekOrigin.Begin);
                 if (!avatarMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant()))
                 {
@@ -898,7 +898,7 @@ public class DownloadHelper : IDownloadHelper
                 memoryStream.Seek(0, SeekOrigin.Begin);
 
                 MD5 md5 = MD5.Create();
-                byte[] hash = md5.ComputeHash(memoryStream);
+                byte[] hash = await md5.ComputeHashAsync(memoryStream);
                 memoryStream.Seek(0, SeekOrigin.Begin);
                 if (!headerMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant()))
                 {

OF DL/Helpers/Interfaces/IAPIHelper.cs

@@ -18,14 +18,15 @@ namespace OF_DL.Helpers
         Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp);
         Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config);
         Task<List<string>> GetListUsers(string endpoint, IDownloadConfig config);
+        Task<Dictionary<string, int>?> GetUsersFromList(string endpoint, bool includeRestricted, IDownloadConfig config);
         Task<Dictionary<long, string>> GetMedia(MediaType mediatype, string endpoint, string? username, string folder, IDownloadConfig config, List<long> paid_post_ids);
-        Task<PaidPostCollection> GetPaidPosts(string endpoint, string folder, string username, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx);
+        Task<PaidPostCollection> GetPaidPosts(string endpoint, string folder, string username, int userId, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx);
         Task<PostCollection> GetPosts(string endpoint, string folder, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx);
         Task<SinglePostCollection> GetPost(string endpoint, string folder, IDownloadConfig config);
         Task<StreamsCollection> GetStreams(string endpoint, string folder, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx);
         Task<ArchivedCollection> GetArchived(string endpoint, string folder, IDownloadConfig config, StatusContext ctx);
         Task<MessageCollection> GetMessages(string endpoint, string folder, IDownloadConfig config, StatusContext ctx);
-        Task<PaidMessageCollection> GetPaidMessages(string endpoint, string folder, string username, IDownloadConfig config, StatusContext ctx);
+        Task<PaidMessageCollection> GetPaidMessages(string endpoint, string folder, string username, int userId, IDownloadConfig config, StatusContext ctx);
         Task<Dictionary<string, int>> GetPurchasedTabUsers(string endpoint, IDownloadConfig config, Dictionary<string, int> users);
         Task<List<PurchasedTabCollection>> GetPurchasedTab(string endpoint, string folder, IDownloadConfig config, Dictionary<string, int> users);
         Task<User> GetUserInfo(string endpoint);

OF DL/HttpUtil.cs

@@ -1,4 +1,5 @@
-using System;
+using OF_DL.Helpers;
+using System;
 using System.Collections.Generic;
 using System.Net.Http;
 using System.Text;
@@ -13,46 +14,66 @@ namespace WidevineClient
             //Proxy = null
         });
 
-        public static byte[] PostData(string URL, Dictionary<string, string> headers, string postData)
+        public static async Task<byte[]> PostData(string URL, Dictionary<string, string> headers, string postData)
         {
             var mediaType = postData.StartsWith("{") ? "application/json" : "application/x-www-form-urlencoded";
-            StringContent content = new StringContent(postData, Encoding.UTF8, mediaType);
+            var response = await PerformOperation(async () =>
-            //ByteArrayContent content = new ByteArrayContent(postData);
+            {
+                StringContent content = new StringContent(postData, Encoding.UTF8, mediaType);
+                //ByteArrayContent content = new ByteArrayContent(postData);
 
-            HttpResponseMessage response = Post(URL, headers, content);
+                return await Post(URL, headers, content);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
 
-        public static byte[] PostData(string URL, Dictionary<string, string> headers, byte[] postData)
+        public static async Task<byte[]> PostData(string URL, Dictionary<string, string> headers, byte[] postData)
         {
-            ByteArrayContent content = new ByteArrayContent(postData);
+            var response = await PerformOperation(async () =>
+            {
+                ByteArrayContent content = new ByteArrayContent(postData);
 
-            HttpResponseMessage response = Post(URL, headers, content);
+                return await Post(URL, headers, content);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
 
-        public static byte[] PostData(string URL, Dictionary<string, string> headers, Dictionary<string, string> postData)
+        public static async Task<byte[]> PostData(string URL, Dictionary<string, string> headers, Dictionary<string, string> postData)
         {
-            FormUrlEncodedContent content = new FormUrlEncodedContent(postData);
+            var response = await PerformOperation(async () =>
+            {
+                FormUrlEncodedContent content = new FormUrlEncodedContent(postData);
 
-            HttpResponseMessage response = Post(URL, headers, content);
+                return await Post(URL, headers, content);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
 
-        public static string GetWebSource(string URL, Dictionary<string, string> headers = null)
+        public static async Task<string> GetWebSource(string URL, Dictionary<string, string> headers = null)
         {
-            HttpResponseMessage response = Get(URL, headers);
+            var response = await PerformOperation(async () =>
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            {
+                return await Get(URL, headers);
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return Encoding.UTF8.GetString(bytes);
         }
 
-        public static byte[] GetBinary(string URL, Dictionary<string, string> headers = null)
+        public static async Task<byte[]> GetBinary(string URL, Dictionary<string, string> headers = null)
         {
-            HttpResponseMessage response = Get(URL, headers);
+            var response = await PerformOperation(async () =>
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            {
+                return await Get(URL, headers);
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
         public static string GetString(byte[] bytes)
@@ -60,7 +81,7 @@ namespace WidevineClient
             return Encoding.UTF8.GetString(bytes);
         }
 
-        static HttpResponseMessage Get(string URL, Dictionary<string, string> headers = null)
+        private static async Task<HttpResponseMessage> Get(string URL, Dictionary<string, string> headers = null)
         {
             HttpRequestMessage request = new HttpRequestMessage()
             {
@@ -72,10 +93,10 @@ namespace WidevineClient
                 foreach (KeyValuePair<string, string> header in headers)
                     request.Headers.TryAddWithoutValidation(header.Key, header.Value);
 
-            return Send(request);
+            return await Send(request);
         }
 
-        static HttpResponseMessage Post(string URL, Dictionary<string, string> headers, HttpContent content)
+        private static async Task<HttpResponseMessage> Post(string URL, Dictionary<string, string> headers, HttpContent content)
         {
             HttpRequestMessage request = new HttpRequestMessage()
             {
@@ -88,12 +109,41 @@ namespace WidevineClient
                 foreach (KeyValuePair<string, string> header in headers)
                     request.Headers.TryAddWithoutValidation(header.Key, header.Value);
 
-            return Send(request);
+            return await Send(request);
         }
 
-        static HttpResponseMessage Send(HttpRequestMessage request)
+        private static async Task<HttpResponseMessage> Send(HttpRequestMessage request)
         {
-            return Client.SendAsync(request).Result;
+            return await Client.SendAsync(request);
+        }
+
+        private static async Task<HttpResponseMessage> PerformOperation(Func<Task<HttpResponseMessage>> operation)
+        {
+            var response = await operation();
+
+            var retryCount = 0;
+
+            while (retryCount < Constants.WIDEVINE_MAX_RETRIES && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
+            {
+                //
+                // We've hit a rate limit, so we should wait before retrying.
+                //
+                var retryAfterSeconds = Constants.WIDEVINE_RETRY_DELAY * (retryCount + 1); // Default retry time. Increases with each retry.
+                if (response.Headers.RetryAfter != null && response.Headers.RetryAfter.Delta.HasValue)
+                {
+                    if (response.Headers.RetryAfter.Delta.Value.TotalSeconds > 0)
+                        retryAfterSeconds = (int)response.Headers.RetryAfter.Delta.Value.TotalSeconds + 1; // Add 1 second to ensure we wait a bit longer than the suggested time
+                }
+
+                await Task.Delay(retryAfterSeconds * 1000); // Peform the delay
+
+                response = await operation();
+                retryCount++;
+            }
+
+            response.EnsureSuccessStatusCode(); // Throw an exception if the response is not successful
+
+            return response;
         }
     }
 }

OF DL/OF DL.csproj

@@ -7,8 +7,15 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <ApplicationIcon>Icon\download.ico</ApplicationIcon>
+    <LangVersion>12</LangVersion>
+    <PublishSingleFile>true</PublishSingleFile>
+    <SelfContained>true</SelfContained>
   </PropertyGroup>
 
+  <PropertyGroup>
+    <NoWarn>CS0168;CS0219;CS0472;CS1998;CS8073;CS8600;CS8602;CS8603;CS8604;CS8605;CS8613;CS8618;CS8622;CS8625;CS8629;SYSLIB0021;AsyncFixer01;AsyncFixer02</NoWarn>
+  </PropertyGroup>
+  
   <ItemGroup>
     <Content Include="Icon\download.ico" />
   </ItemGroup>
@@ -37,12 +44,15 @@
   <ItemGroup>
     <None Update="auth.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <ExcludeFromSingleFile>true</ExcludeFromSingleFile>
     </None>
     <None Update="config.conf">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <ExcludeFromSingleFile>true</ExcludeFromSingleFile>
     </None>
     <None Update="rules.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <ExcludeFromSingleFile>true</ExcludeFromSingleFile>
     </None>
   </ItemGroup>
 

OF DL/Program.cs

@@ -1,7 +1,9 @@
+using Akka.Configuration;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using OF_DL.Entities;
 using OF_DL.Entities.Archived;
+using OF_DL.Entities.Chats;
 using OF_DL.Entities.Messages;
 using OF_DL.Entities.Post;
 using OF_DL.Entities.Purchased;
@@ -13,14 +15,12 @@ using Serilog;
 using Serilog.Core;
 using Serilog.Events;
 using Spectre.Console;
-using System.IO;
+using System.Diagnostics;
 using System.Reflection;
 using System.Runtime.InteropServices;
+using System.Text;
 using System.Text.RegularExpressions;
 using static OF_DL.Entities.Messages.Messages;
-using Akka.Configuration;
-using System.Text;
-using static Akka.Actor.ProviderSelection;
 
 namespace OF_DL;
 
@@ -44,15 +44,15 @@ public class Program
 			AuthHelper authHelper = new();
 			Task setupBrowserTask = authHelper.SetupBrowser(runningInDocker);
 
-			Task.Delay(1000).Wait();
+            await Task.Delay(1000);
 			if (!setupBrowserTask.IsCompleted)
 			{
 				AnsiConsole.MarkupLine($"[yellow]Downloading dependencies. Please wait ...[/]");
 			}
-			setupBrowserTask.Wait();
+            await setupBrowserTask;
 
 			Task<Auth?> getAuthTask = authHelper.GetAuthFromBrowser();
-			Task.Delay(5000).Wait();
+            await Task.Delay(5000);
 			if (!getAuthTask.IsCompleted)
 			{
 				if (runningInDocker)
@@ -117,13 +117,15 @@ public class Program
 
 			AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
 
+            ExitIfOtherProcess();
+
             //Remove config.json and convert to config.conf
             if (File.Exists("config.json"))
             {
                 AnsiConsole.Markup("[green]config.json located successfully!\n[/]");
                 try
                 {
-                    string jsonText = File.ReadAllText("config.json");
+                    string jsonText = await File.ReadAllTextAsync("config.json");
                     var jsonConfig = JsonConvert.DeserializeObject<Entities.Config>(jsonText);
 
                     if (jsonConfig != null)
@@ -160,6 +162,7 @@ public class Program
                         hoconConfig.AppendLine($"  DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
                         hoconConfig.AppendLine($"  CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
                         hoconConfig.AppendLine($"  ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = false");
                         hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
                         hoconConfig.AppendLine("}");
 
@@ -219,7 +222,7 @@ public class Program
                         hoconConfig.AppendLine($"  LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\"");
                         hoconConfig.AppendLine("}");
 
-                        File.WriteAllText("config.conf", hoconConfig.ToString());
+                        await File.WriteAllTextAsync("config.conf", hoconConfig.ToString());
                         File.Delete("config.json");
                         AnsiConsole.Markup("[green]config.conf created successfully from config.json!\n[/]");
                     }
@@ -245,9 +248,9 @@ public class Program
 				AnsiConsole.Markup("[green]config.conf located successfully!\n[/]");
 				try
 				{
-					string hoconText = File.ReadAllText("config.conf");
+					string hoconText = await File.ReadAllTextAsync("config.conf");
 
-					var hoconConfig = ConfigurationFactory.ParseString(hoconText);
+                    var hoconConfig = ConfigurationFactory.ParseString(hoconText);
 
 					config = new Entities.Config
 					{
@@ -279,7 +282,9 @@ public class Program
 						DownloadOnlySpecificDates = hoconConfig.GetBoolean("Download.DownloadOnlySpecificDates"),
 						DownloadDateSelection = Enum.Parse<DownloadDateSelection>(hoconConfig.GetString("Download.DownloadDateSelection"), true),
 						CustomDate = !string.IsNullOrWhiteSpace(hoconConfig.GetString("Download.CustomDate")) ? DateTime.Parse(hoconConfig.GetString("Download.CustomDate")) : null,
-						ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
+                        ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
+                        // Optional flag; default to false when missing
+                        DisableTextSanitization = bool.TryParse(hoconConfig.GetString("Download.DisableTextSanitization", "false"), out var dts) ? dts : false,
                         DownloadVideoResolution = ParseVideoResolution(hoconConfig.GetString("Download.DownloadVideoResolution", "source")),
 
                         // File Settings
@@ -344,7 +349,9 @@ public class Program
                         }
                     }
 
-					levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel;      //set the logging level based on config
+                    levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel;      //set the logging level based on config
+                    // Apply text sanitization preference globally
+                    OF_DL.Utils.XmlUtils.Passthrough = config.DisableTextSanitization;
 					Log.Debug("Configuration:");
 					string configString = JsonConvert.SerializeObject(config, Formatting.Indented);
 					Log.Debug(configString);
@@ -399,9 +406,11 @@ public class Program
                 hoconConfig.AppendLine($"  DownloadOnlySpecificDates = {jsonConfig.DownloadOnlySpecificDates.ToString().ToLower()}");
                 hoconConfig.AppendLine($"  DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
                 hoconConfig.AppendLine($"  CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
-                hoconConfig.AppendLine($"  ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
-                hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
+                        // New option defaults to false when converting legacy json
-                hoconConfig.AppendLine("}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = false");
+                        hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
+                        hoconConfig.AppendLine("}");
 
                 hoconConfig.AppendLine("# File Settings");
                 hoconConfig.AppendLine("File {");
@@ -473,15 +482,52 @@ public class Program
 
 			if (args is not null && args.Length > 0)
 			{
-				const string NON_INTERACTIVE_ARG = "--non-interactive";
+                const string NON_INTERACTIVE_ARG = "--non-interactive";
+                const string SPECIFIC_LISTS_ARG = "--specific-lists";
+                const string SPECIFIC_USERS_ARG = "--specific-users";
 
-				if (args.Any(a => NON_INTERACTIVE_ARG.Equals(NON_INTERACTIVE_ARG, StringComparison.OrdinalIgnoreCase)))
+                if (args.Any(a => NON_INTERACTIVE_ARG.Equals(NON_INTERACTIVE_ARG, StringComparison.OrdinalIgnoreCase)))
-				{
+                {
-					cliNonInteractive = true;
+                    AnsiConsole.Markup($"[grey]Non-Interactive Mode enabled through command-line argument![/]\n");
-					Log.Debug("NonInteractiveMode set via command line");
-				}
 
-				Log.Debug("Additional arguments:");
+                    config.NonInteractiveMode = true;
+
+                    int indexOfSpecificListsArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_LISTS_ARG, StringComparison.OrdinalIgnoreCase));
+                    int indexOfSpecificUsersArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_USERS_ARG, StringComparison.OrdinalIgnoreCase));
+                    char[] separator = [','];
+
+                    if (indexOfSpecificListsArg >= 0)
+                    {
+                        int indexOfListValues = indexOfSpecificListsArg + 1;
+
+                        string[] strListValues = args.ElementAtOrDefault(indexOfListValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
+                        if (strListValues.Length > 0)
+                        {
+                            config.NonInteractiveSpecificLists = strListValues;
+                            config.NonInteractiveModeListName = string.Empty;
+                        }
+                    }
+
+                    if (indexOfSpecificUsersArg >= 0)
+                    {
+                        int indexOfUserValues = indexOfSpecificUsersArg + 1;
+                        string[] strUserValues = args.ElementAtOrDefault(indexOfUserValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
+                        if (strUserValues.Length > 0)
+                        {
+                            config.NonInteractiveSpecificUsers = strUserValues;
+                        }
+                    }
+                }
+
+                const string OUTPUT_BLOCKED_USERS_ARG = "--output-blocked";
+
+                if (args.Any(a => OUTPUT_BLOCKED_USERS_ARG.Equals(a, StringComparison.OrdinalIgnoreCase)))
+                {
+                    config.NonInteractiveMode = true;
+                    config.OutputBlockedUsers = true;
+                }
+
+                Log.Debug("Additional arguments:");
 				foreach (string argument in args)
 				{
 					Log.Debug(argument);
@@ -646,9 +692,10 @@ public class Program
 				AnsiConsole.Markup("[green]rules.json located successfully!\n[/]");
 				try
 				{
-					JsonConvert.DeserializeObject<DynamicRules>(File.ReadAllText("rules.json"));
+                    string rulesJson = await File.ReadAllTextAsync("rules.json");
+                    DynamicRules? dynamicRules = JsonConvert.DeserializeObject<DynamicRules>(rulesJson);
 					Log.Debug($"Rules.json: ");
-					Log.Debug(JsonConvert.SerializeObject(File.ReadAllText("rules.json"), Formatting.Indented));
+					Log.Debug(JsonConvert.SerializeObject(dynamicRules, Formatting.Indented));
 				}
 				catch (Exception e)
 				{
@@ -812,7 +859,21 @@ public class Program
 			}
 
 			AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]");
-			await DownloadAllData(apiHelper, auth, config);
+
+            try
+            {
+                if (config.OutputBlockedUsers)
+                {
+                    await DownloadBlockedOrExpiredUsers(apiHelper, config);
+                    return;
+                }
+
+                await DownloadAllData(apiHelper, auth, config);
+            }
+            finally
+            {
+                DBHelper.CloseAllConnections();
+            }
 		}
 		catch (Exception ex)
 		{
@@ -833,8 +894,41 @@ public class Program
 		}
 	}
 
+    private static async Task DownloadBlockedOrExpiredUsers(APIHelper m_ApiHelper, Entities.Config Config)
+    {
+        const string OUTPUT_FILE_BLOCKED = "blocked-users.json";
+        const string OUTPUT_FILE_EXPIRED = "expired-users.json";
 
-	private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config)
+        await GetUsers("Blocked", "/users/blocked", OUTPUT_FILE_BLOCKED);
+        await GetUsers("Expired", "/subscriptions/subscribes", OUTPUT_FILE_EXPIRED, typeParam: "expired", offsetByCount: false);
+
+        async Task GetUsers(string typeDisplay, string uri, string outputFile, string? typeParam = null, bool offsetByCount = true)
+        {
+            Dictionary<string, int>? users = null;
+
+            await AnsiConsole
+                .Status()
+                .StartAsync($"[red]Getting {typeDisplay} Users[/]", async ctx =>
+                {
+                    users = await m_ApiHelper.GetUsersWithProgress(typeDisplay, uri, ctx, typeParam, offsetByCount);
+                });
+
+            Console.WriteLine();
+
+            if (users is null || users.Count == 0)
+            {
+                AnsiConsole.Markup($"[green]No {typeDisplay} Users found.\n[/]");
+            }
+            else
+            {
+                AnsiConsole.Markup($"[green]Found {users.Count} {typeDisplay} Users, saving to '{outputFile}'\n[/]");
+                string json = JsonConvert.SerializeObject(users, Formatting.Indented);
+                await File.WriteAllTextAsync(outputFile, json);
+            }
+        }
+    }
+
+    private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config)
 	{
 		DBHelper dBHelper = new DBHelper(Config);
 
@@ -843,27 +937,31 @@ public class Program
 		do
 		{
 			DateTime startTime = DateTime.Now;
-			Dictionary<string, int> users = new();
+            Dictionary<string, int> users = new();
-			Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
 
-			Log.Debug("Subscriptions: ");
+            AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
+            Dictionary<string, int> subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
 
-			foreach (KeyValuePair<string, int> activeSub in activeSubs)
+            Log.Debug("Subscriptions: ");
-			{
+            foreach (KeyValuePair<string, int> activeSub in subsActive)
+            {
 				if (!users.ContainsKey(activeSub.Key))
 				{
 					users.Add(activeSub.Key, activeSub.Value);
 					Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}");
 				}
 			}
+
 			if (Config!.IncludeExpiredSubscriptions)
 			{
 				Log.Debug("Inactive Subscriptions: ");
 
-				Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
+                AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
-				foreach (KeyValuePair<string, int> expiredSub in expiredSubs)
+                Dictionary<string, int> subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
-				{
+
-					if (!users.ContainsKey(expiredSub.Key))
+                foreach (KeyValuePair<string, int> expiredSub in subsExpired)
+                {
+                    if (!users.ContainsKey(expiredSub.Key))
 					{
 						users.Add(expiredSub.Key, expiredSub.Value);
 						Log.Debug($"Name: {expiredSub.Key} ID: {expiredSub.Value}");
@@ -886,26 +984,56 @@ public class Program
 					var ignoredUsernames = await m_ApiHelper.GetListUsers($"/lists/{ignoredUsersListId}/users", Config) ?? [];
 					users = users.Where(x => !ignoredUsernames.Contains(x.Key)).ToDictionary(x => x.Key, x => x.Value);
 				}
-			}
+            }
 
-			await dBHelper.CreateUsersDB(users);
+            KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP = new(false, []);
-			KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP;
+            if (Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab)
-			if(Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab)
+            {
-			{
+                hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, new Dictionary<string, int> { { "PurchasedTab", 0 } });
-				hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, new Dictionary<string, int> { { "PurchasedTab", 0 } });
+            }
-			}
+            else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificLists is not null && Config.NonInteractiveSpecificLists.Length > 0)
-			else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName))
+            {
-			{
+                Dictionary<string, int> usersFromLists = new(StringComparer.OrdinalIgnoreCase);
-				hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
+
-			}
+                foreach (string listName in Config.NonInteractiveSpecificLists)
-			else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName))
+                {
-			{
+                    if (!lists.TryGetValue(listName, out int listId))
-				var listId = lists[Config.NonInteractiveModeListName];
+                        continue;
-				var listUsernames = await m_ApiHelper.GetListUsers($"/lists/{listId}/users", Config) ?? [];
+
-				var selectedUsers = users.Where(x => listUsernames.Contains(x.Key)).Distinct().ToDictionary(x => x.Key, x => x.Value);
+                    AnsiConsole.Markup($"[green]Getting Users from list '{listName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
-				hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, selectedUsers);
+                    Dictionary<string, int> list = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
-			}
+
-			else
+                    foreach ((string username, int id) in list)
+                        usersFromLists.TryAdd(username, id);
+                }
+
+                users = usersFromLists;
+                hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
+            }
+            else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificUsers is not null && Config.NonInteractiveSpecificUsers.Length > 0)
+            {
+                HashSet<string> usernames = [.. Config.NonInteractiveSpecificUsers];
+                users = users.Where(u => usernames.Contains(u.Key)).ToDictionary(u => u.Key, u => u.Value);
+                hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
+            }
+            else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName))
+            {
+                hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
+            }
+            else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName))
+            {
+                var listId = lists[Config.NonInteractiveModeListName];
+                AnsiConsole.Markup($"[green]Getting Users from list '{Config.NonInteractiveModeListName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
+                users = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
+                hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
+            }
+
+            if (users.Count <= 0)
+                throw new InvalidOperationException("No users found!");
+
+            await dBHelper.CreateUsersDB(users);
+            			
+			if (hasSelectedUsersKVP.Key == false)
 			{
 				var userSelectionResult = await HandleUserSelection(m_ApiHelper, Config, users, lists);
 
@@ -1024,9 +1152,11 @@ public class Program
 				Log.Debug($"Download path: {p}");
 
 				List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
-				foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
+                int userNum = 1;
+                int userCount = purchasedTabCollections.Count;
+                foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
 				{
-					AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]");
+					AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]");
 					string path = "";
 					if (!string.IsNullOrEmpty(Config.DownloadPath))
 					{
@@ -1135,8 +1265,10 @@ public class Program
 			}
 			else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
 			{
-				//Iterate over each user in the list of users
+                //Iterate over each user in the list of users
-				foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
+                int userNum = 1;
+                int userCount = hasSelectedUsersKVP.Value.Count;
+                foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
 				{
 					int paidPostCount = 0;
 					int postCount = 0;
@@ -1146,7 +1278,7 @@ public class Program
 					int highlightsCount = 0;
 					int messagesCount = 0;
 					int paidMessagesCount = 0;
-					AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]");
+					AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]");
 
 					Log.Debug($"Scraping Data for {user.Key}");
 
@@ -1231,15 +1363,15 @@ public class Program
 
 					AnsiConsole.Markup("\n");
 					AnsiConsole.Write(new BreakdownChart()
-					.FullSize()
+					    .FullSize()
-					.AddItem("Paid Posts", paidPostCount, Color.Red)
+					    .AddItem("Paid Posts", paidPostCount, Color.Red)
-					.AddItem("Posts", postCount, Color.Blue)
+					    .AddItem("Posts", postCount, Color.Blue)
-					.AddItem("Archived", archivedCount, Color.Green)
+					    .AddItem("Archived", archivedCount, Color.Green)
-					.AddItem("Streams", streamsCount, Color.Purple)
+					    .AddItem("Streams", streamsCount, Color.Purple)
-					.AddItem("Stories", storiesCount, Color.Yellow)
+					    .AddItem("Stories", storiesCount, Color.Yellow)
-					.AddItem("Highlights", highlightsCount, Color.Orange1)
+					    .AddItem("Highlights", highlightsCount, Color.Orange1)
-					.AddItem("Messages", messagesCount, Color.LightGreen)
+					    .AddItem("Messages", messagesCount, Color.LightGreen)
-					.AddItem("Paid Messages", paidMessagesCount, Color.Aqua));
+					    .AddItem("Paid Messages", paidMessagesCount, Color.Aqua));
 					AnsiConsole.Markup("\n");
 				}
 				DateTime endTime = DateTime.Now;
@@ -1303,7 +1435,7 @@ public class Program
         await AnsiConsole.Status()
         .StartAsync("[red]Getting Paid Messages[/]", async ctx =>
         {
-            paidMessageCollection = await downloadContext.ApiHelper.GetPaidMessages("/posts/paid", path, user.Key, downloadContext.DownloadConfig!, ctx);
+            paidMessageCollection = await downloadContext.ApiHelper.GetPaidMessages("/posts/paid/chat", path, user.Key, user.Value, downloadContext.DownloadConfig!, ctx);
         });
 		int oldPaidMessagesCount = 0;
 		int newPaidMessagesCount = 0;
@@ -1431,6 +1563,9 @@ public class Program
 	{
 		Log.Debug($"Calling DownloadMessages - {user.Key}");
 
+        AnsiConsole.Markup($"[grey]Getting Unread Chats\n[/]");
+        HashSet<int> unreadChats = await GetUsersWithUnreadChats(downloadContext.ApiHelper, downloadContext.DownloadConfig);
+
         MessageCollection messages = new MessageCollection();
 
         await AnsiConsole.Status()
@@ -1438,7 +1573,14 @@ public class Program
         {
             messages = await downloadContext.ApiHelper.GetMessages($"/chats/{user.Value}/messages", path, downloadContext.DownloadConfig!, ctx);
         });
-		int oldMessagesCount = 0;
+
+        if (unreadChats.Contains(user.Value))
+        {
+            AnsiConsole.Markup($"[grey]Restoring unread state\n[/]");
+            await downloadContext.ApiHelper.MarkAsUnread($"/chats/{user.Value}/mark-as-read", downloadContext.DownloadConfig);
+        }
+
+        int oldMessagesCount = 0;
 		int newMessagesCount = 0;
 		if (messages != null && messages.Messages.Count > 0)
 		{
@@ -1956,7 +2098,7 @@ public class Program
         await AnsiConsole.Status()
         .StartAsync("[red]Getting Paid Posts[/]", async ctx =>
         {
-            purchasedPosts = await downloadContext.ApiHelper.GetPaidPosts("/posts/paid", path, user.Key, downloadContext.DownloadConfig!, paid_post_ids, ctx);
+            purchasedPosts = await downloadContext.ApiHelper.GetPaidPosts("/posts/paid/post", path, user.Key, user.Value, downloadContext.DownloadConfig!, paid_post_ids, ctx);
         });
 
 		int oldPaidPostCount = 0;
@@ -2904,6 +3046,7 @@ public class Program
                         hoconConfig.AppendLine($"  DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
                         hoconConfig.AppendLine($"  CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
                         hoconConfig.AppendLine($"  ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
                         hoconConfig.AppendLine("}");
 
@@ -2963,7 +3106,7 @@ public class Program
                         hoconConfig.AppendLine($"  LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\"");
                         hoconConfig.AppendLine("}");
 
-                        File.WriteAllText("config.conf", hoconConfig.ToString());
+                        await File.WriteAllTextAsync("config.conf", hoconConfig.ToString());
 
                         string newConfigString = JsonConvert.SerializeObject(newConfig, Formatting.Indented);
 
@@ -3063,6 +3206,7 @@ public class Program
                         hoconConfig.AppendLine($"  DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
                         hoconConfig.AppendLine($"  CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
                         hoconConfig.AppendLine($"  ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
                         hoconConfig.AppendLine("}");
 
@@ -3187,7 +3331,18 @@ public class Program
 		}
 	}
 
-	static bool ValidateFilePath(string path)
+    private static async Task<HashSet<int>> GetUsersWithUnreadChats(APIHelper apiHelper, IDownloadConfig currentConfig)
+    {
+        ChatCollection chats = await apiHelper.GetChats($"/chats", currentConfig, onlyUnread: true);
+
+        var unreadChats = chats.Chats
+            .Where(c => c.Value.unreadMessagesCount > 0)
+            .ToList();
+
+        return [.. unreadChats.Select(c => c.Key)];
+    }
+
+    static bool ValidateFilePath(string path)
 	{
 		char[] invalidChars = System.IO.Path.GetInvalidPathChars();
 		char[] foundInvalidChars = path.Where(c => invalidChars.Contains(c)).ToArray();
@@ -3290,4 +3445,24 @@ public class Program
 
         return Enum.Parse<VideoResolution>("_" + value, ignoreCase: true);
     }
+
+    static void ExitIfOtherProcess()
+    {
+        Assembly entryAssembly = Assembly.GetEntryAssembly();
+        AssemblyName entryAssemblyName = entryAssembly?.GetName();
+
+        if (entryAssemblyName?.Name is null)
+            return;
+
+        Process thisProcess = Process.GetCurrentProcess();
+        Process[] otherProcesses = [.. Process.GetProcessesByName(entryAssemblyName.Name).Where(p => p.Id != thisProcess.Id)];
+
+        if (otherProcesses.Length <= 0)
+            return;
+
+        AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]");
+        Log.Warning("Other OF DL process detected, exiting..");
+
+        Environment.Exit(0);
+    }
 }

OF DL/Utils/XmlUtils.cs

@@ -9,8 +9,16 @@ namespace OF_DL.Utils
 {
     internal static class XmlUtils
     {
+        // When true, return original text without parsing/stripping.
+        public static bool Passthrough { get; set; } = false;
+
         public static string EvaluateInnerText(string xmlValue)
         {
+            if (Passthrough)
+            {
+                return xmlValue ?? string.Empty;
+            }
+
             try
             {
                 var parsedText = XElement.Parse($"<root>{xmlValue}</root>");

Publish_OF-DL.bat

@@ -0,0 +1,33 @@
+@ECHO OFF
+
+ECHO.
+
+ECHO ==============================
+ECHO == Cleaning Output ===========
+ECHO ==============================
+dotnet clean ".\OF DL\OF DL.csproj" -v minimal
+DEL /Q /F ".\Publish"
+
+ECHO.
+
+ECHO ==============================
+ECHO == Publishing OF-DL ==========
+ECHO ==============================
+dotnet publish ".\OF DL\OF DL.csproj" -o ".\Publish" -c Debug
+
+ECHO.
+
+ECHO ==============================
+ECHO == Copy to network drive? ====
+ECHO ==============================
+CHOICE /C yn /m "Copy published files to network drive? "
+
+IF %ERRORLEVEL%==1 (GOTO Copy) ELSE (GOTO Exit)
+
+:Copy
+xcopy .\Publish\* p:\_Utils\OF_DL /I /Y /Q /EXCLUDE:.\excludes.txt
+
+:Exit
+ECHO.
+ECHO.
+PAUSE

excludes.txt

@@ -0,0 +1,2 @@
+excludes.txt
+rules.json