Updated non-interactive list user lookup.

Tweaked order to fully replace users before updating DB.

.editorconfig

@@ -5,5 +5,12 @@ root = true
 charset = utf-8
 indent_style = space
 indent_size = 4
+tab_width = 4
+end_of_line = crlf
 insert_final_newline = true
 trim_trailing_whitespace = true
+
+[*.{sln,csproj,xml,json,config}]
+indent_size = 2
+indent_style = space
+tab_width = 2

.gitignore

@@ -371,3 +371,5 @@ FodyWeavers.xsd
 
 # venv
 venv/
+
+Publish/

OF DL/Entities/Chats/ChatCollection.cs

@@ -0,0 +1,7 @@
+namespace OF_DL.Entities.Chats
+{
+    public class ChatCollection
+    {
+        public Dictionary<int, Chats.Chat> Chats { get; set; } = [];
+    }
+}

OF DL/Entities/Chats/Chats.cs

@@ -0,0 +1,20 @@
+namespace OF_DL.Entities.Chats
+{
+    public class Chats
+    {
+        public List<Chat> list { get; set; }
+        public bool hasMore { get; set; }
+        public int nextOffset { get; set; }
+
+        public class Chat
+        {
+            public User withUser { get; set; }
+            public int unreadMessagesCount { get; set; }
+        }
+
+        public class User
+        {
+            public int id { get; set; }
+        }
+    }
+}

OF DL/Entities/Config.cs

@@ -103,6 +103,11 @@ namespace OF_DL.Entities
 
         [JsonConverter(typeof(StringEnumConverter))]
         public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
+
+        public string[] NonInteractiveSpecificUsers { get; set; } = [];
+        public string[] NonInteractiveSpecificLists { get; set; } = [];
+
+        public bool OutputBlockedUsers { get; set; }
     }
 
     public class CreatorConfig : IFileNameFormatConfig

OF DL/Entities/Subscriptions.cs

@@ -98,6 +98,7 @@ namespace OF_DL.Entities
             public object id { get; set; }
             public int? userId { get; set; }
             public int? subscriberId { get; set; }
+            public long? earningId { get; set; }
             public DateTime? date { get; set; }
             public int? duration { get; set; }
             public DateTime? startDate { get; set; }

OF DL/Helpers/APIHelper.cs

@@ -2,6 +2,7 @@ using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using OF_DL.Entities;
 using OF_DL.Entities.Archived;
+using OF_DL.Entities.Chats;
 using OF_DL.Entities.Highlights;
 using OF_DL.Entities.Lists;
 using OF_DL.Entities.Messages;
@@ -13,6 +14,7 @@ using OF_DL.Enumerations;
 using OF_DL.Enumurations;
 using Serilog;
 using Spectre.Console;
+using System.Diagnostics;
 using System.Globalization;
 using System.Security.Cryptography;
 using System.Text;
@@ -25,10 +27,15 @@ namespace OF_DL.Helpers;
 
 public class APIHelper : IAPIHelper
 {
+    private const int MAX_RETRIES = 10;
+    private const int DELAY_BEFORE_RETRY = 1000;
+
     private static readonly JsonSerializerSettings m_JsonSerializerSettings;
     private readonly IDBHelper m_DBHelper;
     private readonly IDownloadConfig downloadConfig;
     private readonly Auth auth;
+    private HttpClient httpClient = new();
+
     private static DateTime? cachedDynamicRulesExpiration;
     private static DynamicRules? cachedDynamicRules;
 
@@ -116,12 +123,21 @@ public class APIHelper : IAPIHelper
     }
 
 
-    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
+    private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null, int retryCount = 0)
     {
-        Log.Debug("Calling BuildHeaderAndExecuteRequests");
+        Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", retryCount + 1);
+
+        try
+        {
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
+
+            Debug.WriteLine($"Executing {request.Method.Method.ToUpper()} request: {request.RequestUri}\r\n\t{GetParamsString(getParams)}");
 
-        HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
             using var response = await client.SendAsync(request);
+
+            if (Debugger.IsAttached && !response.IsSuccessStatusCode)
+                Debugger.Break();
+
             response.EnsureSuccessStatusCode();
             string body = await response.Content.ReadAsStringAsync();
 
@@ -129,17 +145,36 @@ public class APIHelper : IAPIHelper
 
             return body;
         }
+        catch (HttpRequestException ex)
+        {
+            if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && retryCount < MAX_RETRIES)
+            {
+                await Task.Delay(DELAY_BEFORE_RETRY);
+                return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, method, ++retryCount);
+            }
+
+            throw;
+        }
+
+        static string GetParamsString(Dictionary<string, string> getParams)
+            => string.Join(" | ", getParams.Select(kv => $"{kv.Key}={kv.Value}"));
+    }
 
 
-    private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint)
+    private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint, HttpMethod? method = null)
     {
         Log.Debug("Calling BuildHttpRequestMessage");
 
-        string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
+        method ??= HttpMethod.Get;
+
+        string queryParams = "";
+
+        if (getParams.Any())
+            queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
 
         Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
 
-        HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{queryParams}");
+        HttpRequestMessage request = new(method, $"{Constants.API_URL}{endpoint}{queryParams}");
 
         Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}");
 
@@ -164,18 +199,16 @@ public class APIHelper : IAPIHelper
         return input.All(char.IsDigit);
     }
 
-
+    private HttpClient GetHttpClient(IDownloadConfig? config = null)
-    private static HttpClient GetHttpClient(IDownloadConfig? config = null)
     {
-        var client = new HttpClient();
+        httpClient ??= new HttpClient();
         if (config?.Timeout != null && config.Timeout > 0)
         {
-            client.Timeout = TimeSpan.FromSeconds(config.Timeout.Value);
+            httpClient.Timeout = TimeSpan.FromSeconds(config.Timeout.Value);
         }
-        return client;
+        return httpClient;
     }
 
-
     /// <summary>
     /// this one is used during initialization only
     /// if the config option is not available then no modificatiotns will be done on the getParams
@@ -298,47 +331,44 @@ public class APIHelper : IAPIHelper
         try
         {
             Dictionary<string, int> users = new();
-            Subscriptions subscriptions = new();
+
+            int limit = 25;
+            int offset = 0;
+
+            getParams["limit"] = limit.ToString();
+            getParams["offset"] = offset.ToString();
 
             Log.Debug("Calling GetAllSubscrptions");
 
-            string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
-
-            subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
-            if (subscriptions != null && subscriptions.hasMore)
-            {
-                getParams["offset"] = subscriptions.list.Count.ToString();
-
             while (true)
             {
-                    Subscriptions newSubscriptions = new();
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
-                    string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
 
-                    if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
+                if (string.IsNullOrWhiteSpace(body))
-                    {
-                        newSubscriptions = JsonConvert.DeserializeObject<Subscriptions>(loopbody, m_JsonSerializerSettings);
-                    }
-                    else
-                    {
                     break;
-                    }
 
-                    subscriptions.list.AddRange(newSubscriptions.list);
+                Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
-                    if (!newSubscriptions.hasMore)
+
-                    {
+                if (subscriptions?.list is null)
                     break;
-                    }
+
-                    getParams["offset"] = subscriptions.list.Count.ToString();
+                foreach (Subscriptions.List item in subscriptions.list)
-                }
+                {
+                    if (users.ContainsKey(item.username))
+                        continue;
+
+                    bool isRestricted = item.isRestricted ?? false;
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
+
+                    if (!isRestricted || isRestrictedButAllowed)
+                        users.Add(item.username, item.id);
                 }
 
-            foreach (Subscriptions.List subscription in subscriptions.list)
+                if (!subscriptions.hasMore)
-            {
+                    break;
-                if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted))
+
-                    && !users.ContainsKey(subscription.username))
+                offset += limit;
-                {
+                getParams["offset"] = offset.ToString();
-                    users.Add(subscription.username, subscription.id);
-                }
             }
 
             return users;
@@ -361,23 +391,20 @@ public class APIHelper : IAPIHelper
     {
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "active" },
             { "format", "infinite"}
         };
 
+        Log.Debug("Calling GetActiveSubscriptions");
+
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
 
 
     public async Task<Dictionary<string, int>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config)
     {
-
         Dictionary<string, string> getParams = new()
         {
-            { "offset", "0" },
-            { "limit", "50" },
             { "type", "expired" },
             { "format", "infinite"}
         };
@@ -387,6 +414,18 @@ public class APIHelper : IAPIHelper
         return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
     }
 
+    public async Task<Dictionary<string, int>?> GetBlockedUsers(string endpoint, IDownloadConfig config)
+    {
+        Dictionary<string, string> getParams = new()
+        {
+            { "type", "expired" },
+            { "format", "infinite"}
+        };
+
+        Log.Debug("Calling GetBlockedUsers");
+
+        return await GetAllSubscriptions(getParams, endpoint, true, config);
+    }
 
     public async Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config)
     {
@@ -405,7 +444,7 @@ public class APIHelper : IAPIHelper
             Dictionary<string, int> lists = new();
             while (true)
             {
-                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
 
                 if (body == null)
                 {
@@ -470,7 +509,7 @@ public class APIHelper : IAPIHelper
 
             while (true)
             {
-                var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+                var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
                 if (body == null)
                 {
                     break;
@@ -514,6 +553,66 @@ public class APIHelper : IAPIHelper
     }
 
 
+    public async Task<Dictionary<string, int>?> GetUsersFromList(string endpoint, bool includeRestricted, IDownloadConfig config)
+    {
+        var model = new { list = new[] { new { id = int.MaxValue, username = string.Empty, isRestricted = false, isBlocked = false } }, nextOffset = 0, hasMore = false };
+
+        Log.Debug($"Calling GetUsersFromList - {endpoint}");
+
+        int limit = 50;
+        int offset = 0;
+
+        Dictionary<string, string> getParams = new()
+        {
+            { "offset", offset.ToString() },
+            { "limit", limit.ToString() },
+            { "format", "infinite" }
+        };
+
+        try
+        {
+            Dictionary<string, int> users = [];
+
+            while (true)
+            {
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+
+                if (string.IsNullOrWhiteSpace(body))
+                    break;
+
+                var data = JsonConvert.DeserializeAnonymousType(body, model, m_JsonSerializerSettings);
+
+                if (data is null)
+                    break;
+
+                foreach (var item in data.list)
+                {
+                    if (users.ContainsKey(item.username))
+                        continue;
+
+                    bool isRestricted = item.isRestricted;
+                    bool isRestrictedButAllowed = isRestricted && includeRestricted;
+
+                    if (!isRestricted || isRestrictedButAllowed)
+                        users.Add(item.username, item.id);
+                }
+
+                if (!data.hasMore)
+                    break;
+
+
+                offset += data.nextOffset;
+                getParams["offset"] = offset.ToString();
+            }
+
+            return users;
+        }
+        catch (Exception ex)
+        {
+            throw;
+        }
+    }
+
     public async Task<Dictionary<long, string>> GetMedia(MediaType mediatype,
                                                          string endpoint,
                                                          string? username,
@@ -553,7 +652,7 @@ public class APIHelper : IAPIHelper
                     break;
             }
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
 
 
             if (mediatype == MediaType.Stories)
@@ -932,7 +1031,7 @@ public class APIHelper : IAPIHelper
                 ref getParams,
                 downloadAsOf);
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
             posts = JsonConvert.DeserializeObject<Post>(body, m_JsonSerializerSettings);
             ctx.Status($"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.list.Count}[/]");
             ctx.Spinner(Spinner.Known.Dots);
@@ -1090,7 +1189,7 @@ public class APIHelper : IAPIHelper
                 { "skip_users", "all" }
             };
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
             singlePost = JsonConvert.DeserializeObject<SinglePost>(body, m_JsonSerializerSettings);
 
             if (singlePost != null)
@@ -1150,7 +1249,7 @@ public class APIHelper : IAPIHelper
                                     }
                                     break;
                                 case VideoResolution._240:
-                                    if(medium.videoSources != null)
+                                    if (medium.videoSources != null)
                                     {
                                         if (!string.IsNullOrEmpty(medium.videoSources._240))
                                         {
@@ -1251,7 +1350,7 @@ public class APIHelper : IAPIHelper
                 ref getParams,
                 config.CustomDate);
 
-            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+            var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
             streams = JsonConvert.DeserializeObject<Streams>(body, m_JsonSerializerSettings);
             ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.list.Count}[/]");
             ctx.Spinner(Spinner.Known.Dots);
@@ -2134,11 +2233,11 @@ public class APIHelper : IAPIHelper
                         {
                             JObject user = await GetUserInfoById($"/users/list?x[]={purchase.fromUser.id}");
 
-                            if(user is null)
+                            if (user is null)
                             {
                                 if (!config.BypassContentForCreatorsWhoNoLongerExist)
                                 {
-                                    if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
+                                    if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
                                     {
                                         purchasedTabUsers.Add($"Deleted User - {purchase.fromUser.id}", purchase.fromUser.id);
                                     }
@@ -2188,7 +2287,7 @@ public class APIHelper : IAPIHelper
                             {
                                 if (!config.BypassContentForCreatorsWhoNoLongerExist)
                                 {
-                                    if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
+                                    if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
                                     {
                                         purchasedTabUsers.Add($"Deleted User - {purchase.author.id}", purchase.author.id);
                                     }
@@ -2585,6 +2684,94 @@ public class APIHelper : IAPIHelper
         return null;
     }
 
+    public async Task<ChatCollection> GetChats(string endpoint, IDownloadConfig config, bool onlyUnread)
+    {
+        Log.Debug($"Calling GetChats - {endpoint}");
+
+        try
+        {
+            Chats chats = new();
+            ChatCollection collection = new();
+
+            int limit = 60;
+            Dictionary<string, string> getParams = new()
+            {
+                { "limit", $"{limit}" },
+                { "offset", "0" },
+                { "skip_users", "all" },
+                { "order", "recent" }
+            };
+
+            if (onlyUnread)
+                getParams["filter"] = "unread";
+
+            string body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+            chats = JsonConvert.DeserializeObject<Chats>(body, m_JsonSerializerSettings);
+
+            if (chats.hasMore)
+            {
+                getParams["offset"] = $"{chats.nextOffset}";
+
+                while (true)
+                {
+                    string loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
+                    Chats newChats = JsonConvert.DeserializeObject<Chats>(loopbody, m_JsonSerializerSettings);
+
+                    chats.list.AddRange(newChats.list);
+
+                    if (!newChats.hasMore)
+                        break;
+
+                    getParams["offset"] = $"{newChats.nextOffset}";
+                }
+            }
+
+            foreach (Chats.Chat chat in chats.list)
+                collection.Chats.Add(chat.withUser.id, chat);
+
+            return collection;
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            if (ex.InnerException != null)
+            {
+                Console.WriteLine("\nInner Exception:");
+                Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+                Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+            }
+        }
+
+        return null;
+    }
+
+    public async Task MarkAsUnread(string endpoint, IDownloadConfig config)
+    {
+        Log.Debug($"Calling MarkAsUnread - {endpoint}");
+
+        try
+        {
+            var result = new { success = false };
+
+            string body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(config), HttpMethod.Delete);
+            result = JsonConvert.DeserializeAnonymousType(body, result);
+
+            if (result?.success != true)
+                Console.WriteLine($"Failed to mark chat as unread! Endpoint: {endpoint}");
+        }
+        catch (Exception ex)
+        {
+            Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
+            if (ex.InnerException != null)
+            {
+                Console.WriteLine("\nInner Exception:");
+                Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+                Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
+            }
+        }
+    }
 
     public async Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp)
     {
@@ -2703,7 +2890,7 @@ public class APIHelper : IAPIHelper
 
                 using var response = await client.SendAsync(request);
 
-                Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}");
+                Log.Debug($"CDRM Project Response (Attempt {attempt}): {await response.Content.ReadAsStringAsync()}");
 
                 response.EnsureSuccessStatusCode();
                 var body = await response.Content.ReadAsStringAsync();

OF DL/Helpers/DBHelper.cs

@@ -1,18 +1,14 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-using OF_DL.Enumurations;
-using System.IO;
 using Microsoft.Data.Sqlite;
-using Serilog;
 using OF_DL.Entities;
+using Serilog;
+using System.Text;
 
 namespace OF_DL.Helpers
 {
     public class DBHelper : IDBHelper
     {
+        private static readonly Dictionary<string, SqliteConnection> _connections = [];
+
         private readonly IDownloadConfig downloadConfig;
 
         public DBHelper(IDownloadConfig downloadConfig)
@@ -32,9 +28,7 @@ namespace OF_DL.Helpers
                 string dbFilePath = $"{folder}/Metadata/user_data.db";
 
                 // connect to the new database file
-                using SqliteConnection connection = new($"Data Source={dbFilePath}");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={dbFilePath}");
-                // open the connection
-                connection.Open();
 
                 // create the 'medias' table
                 using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection))
@@ -139,11 +133,9 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
                 Log.Debug("Database data source: " + connection.DataSource);
 
-                connection.Open();
-
                 using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection))
                 {
                     await cmd.ExecuteNonQueryAsync();
@@ -194,9 +186,7 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
-
-                connection.Open();
 
                 using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection))
                 {
@@ -247,8 +237,8 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-                connection.Open();
+
                 await EnsureCreatedAtColumnExists(connection, "messages");
                 using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection);
                 cmd.Parameters.AddWithValue("@post_id", post_id);
@@ -286,8 +276,8 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-                connection.Open();
+
                 await EnsureCreatedAtColumnExists(connection, "posts");
                 using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection);
                 cmd.Parameters.AddWithValue("@post_id", post_id);
@@ -324,8 +314,8 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-                connection.Open();
+
                 await EnsureCreatedAtColumnExists(connection, "stories");
                 using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection);
                 cmd.Parameters.AddWithValue("@post_id", post_id);
@@ -362,8 +352,8 @@ namespace OF_DL.Helpers
         {
             try
             {
-                using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-                connection.Open();
+
                 await EnsureCreatedAtColumnExists(connection, "medias");
                 StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id");
                 if (downloadConfig.DownloadDuplicatedMedia)
@@ -400,22 +390,21 @@ namespace OF_DL.Helpers
         {
             try
             {
-                bool downloaded = false;
+                SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
 
-                using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
-                {
                 StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id");
-                    if(downloadConfig.DownloadDuplicatedMedia)
+                if (downloadConfig.DownloadDuplicatedMedia)
                 {
                     sql.Append(" and api_type=@api_type");
                 }
 
                 connection.Open();
-                    using SqliteCommand cmd = new (sql.ToString(), connection);
+                using SqliteCommand cmd = new(sql.ToString(), connection);
                 cmd.Parameters.AddWithValue("@media_id", media_id);
                 cmd.Parameters.AddWithValue("@api_type", api_type);
-                    downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
+
-                }
+                bool downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
+
                 return downloaded;
             }
             catch (Exception ex)
@@ -435,8 +424,7 @@ namespace OF_DL.Helpers
 
         public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at)
         {
-            using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
+            SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-            connection.Open();
 
             // Construct the update command
             StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id");
@@ -463,24 +451,20 @@ namespace OF_DL.Helpers
 
         public async Task<long> GetStoredFileSize(string folder, long media_id, string api_type)
         {
-            long size;
+            SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-            using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
+
-            {
-                connection.Open();
             using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection);
             cmd.Parameters.AddWithValue("@media_id", media_id);
             cmd.Parameters.AddWithValue("@api_type", api_type);
-                size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
+
-            }
+            long size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
             return size;
         }
 
         public async Task<DateTime?> GetMostRecentPostDate(string folder)
         {
-            DateTime? mostRecentDate = null;
+            SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
-            using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
+
-            {
-                connection.Open();
             using SqliteCommand cmd = new(@"
                     SELECT
 	                    MIN(created_at) AS created_at
@@ -497,13 +481,14 @@ namespace OF_DL.Helpers
 				                    ON P.post_id = m.post_id
 		                    WHERE m.downloaded = 0 
 	                      )", connection);
+
             var scalarValue = await cmd.ExecuteScalarAsync();
-                if(scalarValue != null && scalarValue != DBNull.Value)
+            if (scalarValue != null && scalarValue != DBNull.Value)
             {
-                    mostRecentDate = Convert.ToDateTime(scalarValue);
+                return Convert.ToDateTime(scalarValue);
             }
-            }
+
-            return mostRecentDate;
+            return null;
         }
 
         private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName)
@@ -527,5 +512,35 @@ namespace OF_DL.Helpers
                 await alterCmd.ExecuteNonQueryAsync();
             }
         }
+
+        public static void CloseAllConnections()
+        {
+            foreach (SqliteConnection cn in _connections.Values)
+            {
+                cn?.Close();
+                cn?.Dispose();
+            }
+
+            _connections.Clear();
+        }
+
+        private static async Task<SqliteConnection> GetAndOpenConnectionAsync(string connectionString, int numberOfRetries = 2)
+        {
+            try
+            {
+                SqliteConnection connection = new(connectionString);
+                connection.Open();
+
+                return connection;
+            }
+            catch (Exception)
+            {
+                if (--numberOfRetries <= 0)
+                    throw;
+
+                await Task.Delay(300);
+                return await GetAndOpenConnectionAsync(connectionString, numberOfRetries);
+            }
+        }
     }
 }

OF DL/Helpers/DownloadHelper.cs

@@ -855,7 +855,7 @@ public class DownloadHelper : IDownloadHelper
                 memoryStream.Seek(0, SeekOrigin.Begin);
 
                 MD5 md5 = MD5.Create();
-                byte[] hash = md5.ComputeHash(memoryStream);
+                byte[] hash = await md5.ComputeHashAsync(memoryStream);
                 memoryStream.Seek(0, SeekOrigin.Begin);
                 if (!avatarMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant()))
                 {
@@ -898,7 +898,7 @@ public class DownloadHelper : IDownloadHelper
                 memoryStream.Seek(0, SeekOrigin.Begin);
 
                 MD5 md5 = MD5.Create();
-                byte[] hash = md5.ComputeHash(memoryStream);
+                byte[] hash = await md5.ComputeHashAsync(memoryStream);
                 memoryStream.Seek(0, SeekOrigin.Begin);
                 if (!headerMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant()))
                 {

OF DL/Helpers/Interfaces/IAPIHelper.cs

@@ -18,6 +18,7 @@ namespace OF_DL.Helpers
         Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp);
         Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config);
         Task<List<string>> GetListUsers(string endpoint, IDownloadConfig config);
+        Task<Dictionary<string, int>?> GetUsersFromList(string endpoint, bool includeRestricted, IDownloadConfig config);
         Task<Dictionary<long, string>> GetMedia(MediaType mediatype, string endpoint, string? username, string folder, IDownloadConfig config, List<long> paid_post_ids);
         Task<PaidPostCollection> GetPaidPosts(string endpoint, string folder, string username, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx);
         Task<PostCollection> GetPosts(string endpoint, string folder, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx);

OF DL/OF DL.csproj

@@ -7,6 +7,13 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <ApplicationIcon>Icon\download.ico</ApplicationIcon>
+    <LangVersion>12</LangVersion>
+    <PublishSingleFile>true</PublishSingleFile>
+    <SelfContained>true</SelfContained>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <NoWarn>CS0168;CS0219;CS0472;CS1998;CS8073;CS8600;CS8602;CS8603;CS8604;CS8605;CS8613;CS8618;CS8622;CS8625;CS8629;SYSLIB0021;AsyncFixer01;AsyncFixer02</NoWarn>
   </PropertyGroup>
   
   <ItemGroup>
@@ -37,12 +44,15 @@
   <ItemGroup>
     <None Update="auth.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <ExcludeFromSingleFile>true</ExcludeFromSingleFile>
     </None>
     <None Update="config.conf">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <ExcludeFromSingleFile>true</ExcludeFromSingleFile>
     </None>
     <None Update="rules.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      <ExcludeFromSingleFile>true</ExcludeFromSingleFile>
     </None>
   </ItemGroup>
 

OF DL/Program.cs

@@ -1,7 +1,9 @@
+using Akka.Configuration;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using OF_DL.Entities;
 using OF_DL.Entities.Archived;
+using OF_DL.Entities.Chats;
 using OF_DL.Entities.Messages;
 using OF_DL.Entities.Post;
 using OF_DL.Entities.Purchased;
@@ -13,14 +15,12 @@ using Serilog;
 using Serilog.Core;
 using Serilog.Events;
 using Spectre.Console;
-using System.IO;
+using System.Diagnostics;
 using System.Reflection;
 using System.Runtime.InteropServices;
+using System.Text;
 using System.Text.RegularExpressions;
 using static OF_DL.Entities.Messages.Messages;
-using Akka.Configuration;
-using System.Text;
-using static Akka.Actor.ProviderSelection;
 
 namespace OF_DL;
 
@@ -44,15 +44,15 @@ public class Program
 			AuthHelper authHelper = new();
 			Task setupBrowserTask = authHelper.SetupBrowser(runningInDocker);
 
-			Task.Delay(1000).Wait();
+            await Task.Delay(1000);
 			if (!setupBrowserTask.IsCompleted)
 			{
 				AnsiConsole.MarkupLine($"[yellow]Downloading dependencies. Please wait ...[/]");
 			}
-			setupBrowserTask.Wait();
+            await setupBrowserTask;
 
 			Task<Auth?> getAuthTask = authHelper.GetAuthFromBrowser();
-			Task.Delay(5000).Wait();
+            await Task.Delay(5000);
 			if (!getAuthTask.IsCompleted)
 			{
 				if (runningInDocker)
@@ -117,13 +117,15 @@ public class Program
 
 			AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
 
+            ExitIfOtherProcess();
+
             //Remove config.json and convert to config.conf
             if (File.Exists("config.json"))
             {
                 AnsiConsole.Markup("[green]config.json located successfully!\n[/]");
                 try
                 {
-                    string jsonText = File.ReadAllText("config.json");
+                    string jsonText = await File.ReadAllTextAsync("config.json");
                     var jsonConfig = JsonConvert.DeserializeObject<Entities.Config>(jsonText);
 
                     if (jsonConfig != null)
@@ -219,7 +221,7 @@ public class Program
                         hoconConfig.AppendLine($"  LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\"");
                         hoconConfig.AppendLine("}");
 
-                        File.WriteAllText("config.conf", hoconConfig.ToString());
+                        await File.WriteAllTextAsync("config.conf", hoconConfig.ToString());
                         File.Delete("config.json");
                         AnsiConsole.Markup("[green]config.conf created successfully from config.json!\n[/]");
                     }
@@ -245,7 +247,7 @@ public class Program
 				AnsiConsole.Markup("[green]config.conf located successfully!\n[/]");
 				try
 				{
-					string hoconText = File.ReadAllText("config.conf");
+					string hoconText = await File.ReadAllTextAsync("config.conf");
 
 					var hoconConfig = ConfigurationFactory.ParseString(hoconText);
 
@@ -474,11 +476,48 @@ public class Program
 			if (args is not null && args.Length > 0)
 			{
                 const string NON_INTERACTIVE_ARG = "--non-interactive";
+                const string SPECIFIC_LISTS_ARG = "--specific-lists";
+                const string SPECIFIC_USERS_ARG = "--specific-users";
 
                 if (args.Any(a => NON_INTERACTIVE_ARG.Equals(NON_INTERACTIVE_ARG, StringComparison.OrdinalIgnoreCase)))
                 {
-					cliNonInteractive = true;
+                    AnsiConsole.Markup($"[grey]Non-Interactive Mode enabled through command-line argument![/]\n");
-					Log.Debug("NonInteractiveMode set via command line");
+
+                    config.NonInteractiveMode = true;
+
+                    int indexOfSpecificListsArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_LISTS_ARG, StringComparison.OrdinalIgnoreCase));
+                    int indexOfSpecificUsersArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_USERS_ARG, StringComparison.OrdinalIgnoreCase));
+                    char[] separator = [','];
+
+                    if (indexOfSpecificListsArg >= 0)
+                    {
+                        int indexOfListValues = indexOfSpecificListsArg + 1;
+
+                        string[] strListValues = args.ElementAtOrDefault(indexOfListValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
+                        if (strListValues.Length > 0)
+                        {
+                            config.NonInteractiveSpecificLists = strListValues;
+                            config.NonInteractiveModeListName = string.Empty;
+                        }
+                    }
+
+                    if (indexOfSpecificUsersArg >= 0)
+                    {
+                        int indexOfUserValues = indexOfSpecificUsersArg + 1;
+                        string[] strUserValues = args.ElementAtOrDefault(indexOfUserValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
+                        if (strUserValues.Length > 0)
+                        {
+                            config.NonInteractiveSpecificUsers = strUserValues;
+                        }
+                    }
+                }
+
+                const string OUTPUT_BLOCKED_USERS_ARG = "--output-blocked";
+
+                if (args.Any(a => OUTPUT_BLOCKED_USERS_ARG.Equals(a, StringComparison.OrdinalIgnoreCase)))
+                {
+                    config.NonInteractiveMode = true;
+                    config.OutputBlockedUsers = true;
                 }
 
                 Log.Debug("Additional arguments:");
@@ -646,9 +685,10 @@ public class Program
 				AnsiConsole.Markup("[green]rules.json located successfully!\n[/]");
 				try
 				{
-					JsonConvert.DeserializeObject<DynamicRules>(File.ReadAllText("rules.json"));
+                    string rulesJson = await File.ReadAllTextAsync("rules.json");
+                    DynamicRules? dynamicRules = JsonConvert.DeserializeObject<DynamicRules>(rulesJson);
 					Log.Debug($"Rules.json: ");
-					Log.Debug(JsonConvert.SerializeObject(File.ReadAllText("rules.json"), Formatting.Indented));
+					Log.Debug(JsonConvert.SerializeObject(dynamicRules, Formatting.Indented));
 				}
 				catch (Exception e)
 				{
@@ -812,8 +852,22 @@ public class Program
 			}
 
 			AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]");
+
+            try
+            {
+                if (config.OutputBlockedUsers)
+                {
+                    await DownloadBlockedUsers(apiHelper, config);
+                    return;
+                }
+
                 await DownloadAllData(apiHelper, auth, config);
             }
+            finally
+            {
+                DBHelper.CloseAllConnections();
+            }
+		}
 		catch (Exception ex)
 		{
 			Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
@@ -833,6 +887,23 @@ public class Program
 		}
 	}
 
+    private static async Task DownloadBlockedUsers(APIHelper m_ApiHelper, Entities.Config Config)
+    {
+        const string OUTPUT_FILE = "blocked-users.json";
+
+        Dictionary<string, int>? blockedUsers = await m_ApiHelper.GetBlockedUsers("/users/blocked", Config);
+
+        if (blockedUsers is null || blockedUsers.Count == 0)
+        {
+            AnsiConsole.Markup($"[green]No Blocked Users found.\n[/]");
+        }
+        else
+        {
+            AnsiConsole.Markup($"[green]Found {blockedUsers.Count} Blocked Users, saving to '{OUTPUT_FILE}'\n[/]");
+            string json = JsonConvert.SerializeObject(blockedUsers, Formatting.Indented);
+            await File.WriteAllTextAsync(OUTPUT_FILE, json);
+        }
+    }
 
     private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config)
 	{
@@ -844,11 +915,12 @@ public class Program
 		{
 			DateTime startTime = DateTime.Now;
             Dictionary<string, int> users = new();
-			Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
+
+            AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
+            Dictionary<string, int> subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
 
             Log.Debug("Subscriptions: ");
-
+            foreach (KeyValuePair<string, int> activeSub in subsActive)
-			foreach (KeyValuePair<string, int> activeSub in activeSubs)
             {
 				if (!users.ContainsKey(activeSub.Key))
 				{
@@ -856,12 +928,15 @@ public class Program
 					Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}");
 				}
 			}
+
 			if (Config!.IncludeExpiredSubscriptions)
 			{
 				Log.Debug("Inactive Subscriptions: ");
 
-				Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
+                AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
-				foreach (KeyValuePair<string, int> expiredSub in expiredSubs)
+                Dictionary<string, int> subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
+
+                foreach (KeyValuePair<string, int> expiredSub in subsExpired)
                 {
                     if (!users.ContainsKey(expiredSub.Key))
 					{
@@ -888,12 +963,36 @@ public class Program
 				}
             }
 
-			await dBHelper.CreateUsersDB(users);
+            KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP = new(false, []);
-			KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP;
+            if (Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab)
-			if(Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab)
             {
                 hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, new Dictionary<string, int> { { "PurchasedTab", 0 } });
             }
+            else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificLists is not null && Config.NonInteractiveSpecificLists.Length > 0)
+            {
+                Dictionary<string, int> usersFromLists = new(StringComparer.OrdinalIgnoreCase);
+
+                foreach (string listName in Config.NonInteractiveSpecificLists)
+                {
+                    if (!lists.TryGetValue(listName, out int listId))
+                        continue;
+
+                    AnsiConsole.Markup($"[green]Getting Users from list '{listName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
+                    Dictionary<string, int> list = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
+
+                    foreach ((string username, int id) in list)
+                        usersFromLists.TryAdd(username, id);
+                }
+
+                users = usersFromLists;
+                hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
+            }
+            else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificUsers is not null && Config.NonInteractiveSpecificUsers.Length > 0)
+            {
+                HashSet<string> usernames = [.. Config.NonInteractiveSpecificUsers];
+                users = users.Where(u => usernames.Contains(u.Key)).ToDictionary(u => u.Key, u => u.Value);
+                hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
+            }
             else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName))
             {
                 hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
@@ -901,11 +1000,17 @@ public class Program
             else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName))
             {
                 var listId = lists[Config.NonInteractiveModeListName];
-				var listUsernames = await m_ApiHelper.GetListUsers($"/lists/{listId}/users", Config) ?? [];
+                AnsiConsole.Markup($"[green]Getting Users from list '{Config.NonInteractiveModeListName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
-				var selectedUsers = users.Where(x => listUsernames.Contains(x.Key)).Distinct().ToDictionary(x => x.Key, x => x.Value);
+                users = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
-				hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, selectedUsers);
+                hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
             }
-			else
+
+            if (users.Count <= 0)
+                throw new InvalidOperationException("No users found!");
+
+            await dBHelper.CreateUsersDB(users);
+            			
+			if (hasSelectedUsersKVP.Key == false)
 			{
 				var userSelectionResult = await HandleUserSelection(m_ApiHelper, Config, users, lists);
 
@@ -1024,9 +1129,11 @@ public class Program
 				Log.Debug($"Download path: {p}");
 
 				List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
-				foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
+                int userNum = 1;
+                int userCount = purchasedTabCollections.Count;
+                foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
 				{
-					AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]");
+					AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]");
 					string path = "";
 					if (!string.IsNullOrEmpty(Config.DownloadPath))
 					{
@@ -1136,6 +1243,8 @@ public class Program
 			else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
 			{
                 //Iterate over each user in the list of users
+                int userNum = 1;
+                int userCount = hasSelectedUsersKVP.Value.Count;
                 foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
 				{
 					int paidPostCount = 0;
@@ -1146,7 +1255,7 @@ public class Program
 					int highlightsCount = 0;
 					int messagesCount = 0;
 					int paidMessagesCount = 0;
-					AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]");
+					AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]");
 
 					Log.Debug($"Scraping Data for {user.Key}");
 
@@ -1431,6 +1540,9 @@ public class Program
 	{
 		Log.Debug($"Calling DownloadMessages - {user.Key}");
 
+        AnsiConsole.Markup($"[grey]Getting Unread Chats\n[/]");
+        HashSet<int> unreadChats = await GetUsersWithUnreadChats(downloadContext.ApiHelper, downloadContext.DownloadConfig);
+
         MessageCollection messages = new MessageCollection();
 
         await AnsiConsole.Status()
@@ -1438,6 +1550,13 @@ public class Program
         {
             messages = await downloadContext.ApiHelper.GetMessages($"/chats/{user.Value}/messages", path, downloadContext.DownloadConfig!, ctx);
         });
+
+        if (unreadChats.Contains(user.Value))
+        {
+            AnsiConsole.Markup($"[grey]Restoring unread state\n[/]");
+            await downloadContext.ApiHelper.MarkAsUnread($"/chats/{user.Value}/mark-as-read", downloadContext.DownloadConfig);
+        }
+
         int oldMessagesCount = 0;
 		int newMessagesCount = 0;
 		if (messages != null && messages.Messages.Count > 0)
@@ -2963,7 +3082,7 @@ public class Program
                         hoconConfig.AppendLine($"  LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\"");
                         hoconConfig.AppendLine("}");
 
-                        File.WriteAllText("config.conf", hoconConfig.ToString());
+                        await File.WriteAllTextAsync("config.conf", hoconConfig.ToString());
 
                         string newConfigString = JsonConvert.SerializeObject(newConfig, Formatting.Indented);
 
@@ -3187,6 +3306,17 @@ public class Program
 		}
 	}
 
+    private static async Task<HashSet<int>> GetUsersWithUnreadChats(APIHelper apiHelper, IDownloadConfig currentConfig)
+    {
+        ChatCollection chats = await apiHelper.GetChats($"/chats", currentConfig, onlyUnread: true);
+
+        var unreadChats = chats.Chats
+            .Where(c => c.Value.unreadMessagesCount > 0)
+            .ToList();
+
+        return [.. unreadChats.Select(c => c.Key)];
+    }
+
     static bool ValidateFilePath(string path)
 	{
 		char[] invalidChars = System.IO.Path.GetInvalidPathChars();
@@ -3290,4 +3420,24 @@ public class Program
 
         return Enum.Parse<VideoResolution>("_" + value, ignoreCase: true);
     }
+
+    static void ExitIfOtherProcess()
+    {
+        Assembly entryAssembly = Assembly.GetEntryAssembly();
+        AssemblyName entryAssemblyName = entryAssembly?.GetName();
+
+        if (entryAssemblyName?.Name is null)
+            return;
+
+        Process thisProcess = Process.GetCurrentProcess();
+        Process[] otherProcesses = [.. Process.GetProcessesByName(entryAssemblyName.Name).Where(p => p.Id != thisProcess.Id)];
+
+        if (otherProcesses.Length <= 0)
+            return;
+
+        AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]");
+        Log.Warning("Other OF DL process detected, exiting..");
+
+        Environment.Exit(0);
+    }
 }

Publish_OF-DL.bat

@@ -0,0 +1,33 @@
+@ECHO OFF
+
+ECHO.
+
+ECHO ==============================
+ECHO == Cleaning Output ===========
+ECHO ==============================
+dotnet clean ".\OF DL\OF DL.csproj" -v minimal
+DEL /Q /F ".\Publish"
+
+ECHO.
+
+ECHO ==============================
+ECHO == Publishing OF-DL ==========
+ECHO ==============================
+dotnet publish ".\OF DL\OF DL.csproj" -o ".\Publish" -c Debug
+
+ECHO.
+
+ECHO ==============================
+ECHO == Copy to network drive? ====
+ECHO ==============================
+CHOICE /C yn /m "Copy published files to network drive? "
+
+IF %ERRORLEVEL%==1 (GOTO Copy) ELSE (GOTO Exit)
+
+:Copy
+xcopy .\Publish\* p:\_Utils\OF_DL /I /Y /Q /EXCLUDE:.\excludes.txt
+
+:Exit
+ECHO.
+ECHO.
+PAUSE

docs/config/all-configuration-options.md

@@ -0,0 +1,518 @@
+# All Configuration Options
+
+This page contains detailed information for each configuration option supported by OF-DL. For information about the structure of the `config.conf` file or a simple list of these configuration options, go to the [configuration page](/config/configuration).
+
+## BypassContentForCreatorsWhoNoLongerExist
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: When a creator no longer exists (their account has been deleted), most of their content will be inaccessible.
+Purchased content, however, will still be accessible by downloading media usisng the "Download Purchased Tab" menu option
+or with the [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) config option when downloading media in non-interactive mode.
+
+## CreatorConfigs
+
+Type: `object`
+
+Default: `{}`
+
+Allowed values: An array of Creator Config objects
+
+Description: This configuration options allows you to set file name formats for specific creators.
+This is useful if you want to have different file name formats for different creators. The values set here will override the global values set in the config file
+(see [PaidPostFileNameFormat](#paidpostfilenameformat), [PostFileNameFormat](#postfilenameformat),
+[PaidMessageFileNAmeFormat](#paidmessagefilenameformat), and [MessageFileNameFormat](#messagefilenameformat)).
+For more information on the file name formats, see the [custom filename formats](/config/custom-filename-formats) page.
+
+Example:
+```
+"CreatorConfigs": {
+    "creator_one": {
+        "PaidPostFileNameFormat": "{id}_{mediaid}_{filename}",
+        "PostFileNameFormat": "{username}_{id}_{mediaid}_{mediaCreatedAt}",
+        "PaidMessageFileNameFormat": "{id}_{mediaid}_{createdAt}",
+        "MessageFileNameFormat": "{id}_{mediaid}_{filename}"
+    },
+    "creator_two": {
+        "PaidPostFileNameFormat": "{id}_{mediaid}",
+        "PostFileNameFormat": "{username}_{id}_{mediaid}",
+        "PaidMessageFileNameFormat": "{id}_{mediaid}",
+        "MessageFileNameFormat": "{id}_{mediaid}"
+    }
+}
+```
+
+## CustomDate
+
+Type: `string`
+
+Default: `null`
+
+Allowed values: Any date in `yyyy-mm-dd` format or `null`
+
+Description: [DownloadOnlySpecificDates](#downloadonlyspecificdates) needs to be set to `true` for this to work.
+This date will be used when you are trying to download between/after a certain date. See [DownloadOnlySpecificDates](#downloadonlyspecificdates) and
+[DownloadDateSelection](#downloaddateselection) for more information.
+
+## DisableBrowserAuth
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: The built-in/bundled web browser will not be used to authenticate OF-DL if set to `true`. If set to `true`,
+an `auth.json` file will need to be provided using a [legacy authentication method](/config/auth#legacy-methods).
+If set to `true`, the `auth.json` file will not be deleted if authentication fails. If set to `false` (the default
+behavior), OF-DL will delete the `auth.json` file if authentication fails.
+
+## DownloadArchived
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Posts in the "Archived" tab will be downloaded if set to `true`
+
+## DownloadAudios
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Audios will be downloaded if set to `true`
+
+## DownloadAvatarHeaderPhoto
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Avatar and header images will be downloaded if set to `true`
+
+## DownloadDateSelection
+
+Type: `string`
+
+Default: `"before"`
+
+Allowed values: `"before"`, `"after"`
+
+Description: [DownloadOnlySpecificDates](#downloadonlyspecificdates) needs to be set to `true` for this to work. This will get all posts from before
+the date if set to `"before"`, and all posts from the date you specify up until the current date if set to `"after"`.
+The date you specify will be in the [CustomDate](#customdate) config option.
+
+## DownloadDuplicatedMedia
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: By default (or when set to `false`), the program will not download duplicated media. If set to `true`, duplicated media will be downloaded.
+
+## DownloadHighlights
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Highlights on a user's will be downloaded if set to `true`
+
+## DownloadImages
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Images will be downloaded if set to `true`
+
+## DownloadLimitInMbPerSec
+
+Type: `integer`
+
+Default: `4`
+
+Allowed values: Any positive integer
+
+Description: The download rate in MB per second. This will only be used if [LimitDownloadRate](#limitdownloadrate) is set to `true`.
+
+## DownloadMessages
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Free media within messages (including paid message previews) will be downloaded if set to `true`
+
+## DownloadOnlySpecificDates
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: If set to `true`, posts will be downloaded based on the [DownloadDateSelection](#downloaddateselection) and [CustomDate](#customdate) config options.
+If set to `false`, all posts will be downloaded.
+
+## DownloadPaidMessages
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Paid media within messages (excluding paid message previews) will be downloaded if set to `true`
+
+## DownloadPaidPosts
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Paid posts will be downloaded if set to `true`
+
+## DownloadPath
+
+Type: `string`
+
+Default: `""`
+
+Allowed values: Any valid path
+
+Description: If left blank then content will be downloaded to `__user_data__/sites/OnlyFans/{username}`.
+If you set the download path to `"S:/"`, then content will be downloaded to `S:/{username}`
+
+!!! note
+
+    If you are using a Windows path, you will need to escape the backslashes, e.g. `"C:\\Users\\user\\Downloads\\OnlyFans\\"`
+    Please make sure your path ends with a `/`
+
+## DownloadPosts
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Free posts will be downloaded if set to `true`
+
+## DownloadPostsIncrementally
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: If set to `true`, only new posts will be downloaded from the date of the last post that was downloaded based off what's in the `user_data.db` file.
+If set to `false`, the default behaviour will apply, and all posts will be gathered and compared against the database to see if they need to be downloaded or not.
+
+## DownloadStories
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Stories on a user's profile will be downloaded if set to `true`
+
+## DownloadStreams
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Posts in the "Streams" tab will be downloaded if set to `true`
+
+## DownloadVideos
+
+Type: `boolean`
+
+Default: `true`
+
+Allowed values: `true`, `false`
+
+Description: Videos will be downloaded if set to `true`
+
+## FFmpegPath
+
+Type: `string`
+
+Default: `""`
+
+Allowed values: Any valid path or `""`
+
+Description: This is the path to the FFmpeg executable (`ffmpeg.exe` on Windows and `ffmpeg` on Linux/macOS).
+If the path is not set then the program will try to find it in both the same directory as the OF-DL executable as well
+as the PATH environment variable.
+
+!!! note
+
+    If you are using a Windows path, you will need to escape the backslashes, e.g. `"C:\\ffmpeg\\bin\\ffmpeg.exe"`
+    For example, this is not valid: `"C:\some\path\ffmpeg.exe"`, but `"C:/some/path/ffmpeg.exe"` and `"C:\\some\\path\\ffmpeg.exe"` are both valid.
+
+## FolderPerMessage
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: A folder will be created for each message (containing all the media for that message) if set to `true`.
+When set to `false`, message media will be downloaded into the `Messages/Free` folder.
+
+## FolderPerPaidMessage
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: A folder will be created for each paid message (containing all the media for that message) if set to `true`.
+When set to `false`, paid message media will be downloaded into the `Messages/Paid` folder.
+
+## FolderPerPaidPost
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: A folder will be created for each paid post (containing all the media for that post) if set to `true`.
+When set to `false`, paid post media will be downloaded into the `Posts/Paid` folder.
+
+## FolderPerPost
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: A folder will be created for each post (containing all the media for that post) if set to `true`.
+When set to `false`, post media will be downloaded into the `Posts/Free` folder.
+
+## IgnoreOwnMessages
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: By default (or when set to `false`), messages that were sent by yourself will be added to the metadata DB and any media which has been sent by yourself will be downloaded. If set to `true`, the program will not add messages sent by yourself to the metadata DB and will not download any media which has been sent by yourself.
+
+## IgnoredUsersListName
+
+Type: `string`
+
+Default: `""`
+
+Allowed values: The name of a list of users you have created on OnlyFans or `""`
+
+Description: When set to the name of a list, users in the list will be ignored when scraping content.
+If set to `""` (or an invalid list name), no users will be ignored when scraping content.
+
+## IncludeExpiredSubscriptions
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: If set to `true`, expired subscriptions will appear in the user list under the "Custom" menu option.
+
+## IncludeRestrictedSubscriptions
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: If set to `true`, media from restricted creators will be downloaded. If set to `false`, restricted creators will be ignored.
+
+## LimitDownloadRate
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: If set to `true`, the download rate will be limited to the value set in [DownloadLimitInMbPerSec](#downloadlimitinmbpersec).
+
+## LoggingLevel
+
+Type: `string`
+
+Default: `"Error"`
+
+Allowed values: `"Verbose"`, `"Debug"`, `"Information"`, `"Warning"`, `"Error"`, `"Fatal"`
+
+Description: The level of logging that will be saved to the log files in the `logs` folder.
+When requesting help with an issue, it is recommended to set this to `"Verbose"` and provide the log file.
+
+## MessageFileNameFormat
+
+Type: `string`
+
+Default: `""`
+
+Allowed values: Any valid string
+
+Description: Please refer to [custom filename formats](/config/custom-filename-formats#messagefilenameformat) page to see what fields you can use.
+
+## NonInteractiveMode
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: If set to `true`, the program will run without any input from the user. It will scrape all users automatically
+(unless [NonInteractiveModeListName](#noninteractivemodelistname) or [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) are configured).
+If set to `false`, the default behaviour will apply, and you will be able to choose an option from the menu.
+
+!!! warning
+
+    If NonInteractiveMode is enabled, you will be unable to authenticate OF-DL using the standard authentication method.
+    Before you can run OF-DL in NonInteractiveMode, you must either
+
+    1. Generate an auth.json file by running OF-DL with NonInteractiveMode disabled and authenticating OF-DL using the standard method **OR**
+    2. Generate an auth.json file by using a [legacy authentication method](/config/auth#legacy-methods)
+
+
+## NonInteractiveModeListName
+
+Type: `string`
+
+Default: `""`
+
+Allowed values: The name of a list of users you have created on OnlyFans or `""`
+
+Description: When set to the name of a list, non-interactive mode will download media from the list of users instead of all
+users (when [NonInteractiveMode](#noninteractivemode) is set to `true`). If set to `""`, all users will be scraped
+(unless [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) is configured).
+
+## NonInteractiveModePurchasedTab
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: When set to `true`, non-interactive mode will only download content from the Purchased tab
+(when [NonInteractiveMode](#noninteractivemode) is set to `true`). If set to `false`, all users will be scraped
+(unless [NonInteractiveModeListName](#noninteractivemodelistname) is configured).
+
+## PaidMessageFileNameFormat
+
+Type: `string`
+
+Default: `""`
+
+Allowed values: Any valid string
+
+Description: Please refer to [custom filename formats](/config/custom-filename-formats#paidmessagefilenameformat) page to see what fields you can use.
+
+## PaidPostFileNameFormat
+
+Type: `string`
+
+Default: `""`
+
+Allowed values: Any valid string
+
+Description: Please refer to [custom filename formats](/config/custom-filename-formats#paidpostfilenameformat) page to see what fields you can use.
+
+## PostFileNameFormat
+
+Type: `string`
+
+Default: `""`
+
+Allowed values: Any valid string
+
+Description: Please refer to the [custom filename formats](/config/custom-filename-formats#postfilenameformat) page to see what fields you can use.
+
+## RenameExistingFilesWhenCustomFormatIsSelected
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: When `true`, any current files downloaded will have the current format applied to them.
+When `false`, only new files will have the current format applied to them.
+
+## ShowScrapeSize
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: If set to `true`, the total scrape size will be shown in bytes when downloading posts, messages, etc.
+When set to `false`, the total number of posts, messages, etc. will be shown.
+
+!!! warning
+
+    Setting this to `true` will have an impact on performance as it has to go through each piece of media and get the size
+    from the server, which is a big task and can sometimes get you rate limited.
+
+## SkipAds
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: Posts and messages that contain #ad or free trial links will be ignored if set to `true`
+
+## Timeout
+
+Type: `integer`
+
+Default: `-1`
+
+Allowed values: Any positive integer or `-1`
+
+Description: You won't need to set this, but if you see errors about the configured timeout of 100 seconds elapsing then
+you could set this to be more than 100. It is recommended that you leave this as the default value.

docs/config/auth.md

@@ -1,7 +1,3 @@
----
-sidebar_position: 1
----
-
 # Authentication
 
 ## Current Method (versions >= 1.9.0)

docs/config/cdm.md

@@ -1,8 +1,4 @@
----
+# CDM (optional, but recommended)
-sidebar_position: 4
----
-
-# CDM (optional, but recommended)
 
 Without Widevine/CDM keys, OF DL uses the 3rd party website cdrm-project.org for decrypting DRM videos. With keys, OF DL directly communicates with OnlyFans. It is highly recommended to use keys, both in case the cdrm-project site is having issues (which occur frequently, in our experience) and it will result in faster download speeds, too. However, this is optional, as things will work as long as cdrm-project is functional.
 

docs/config/configuration.md

@@ -1,514 +1,68 @@
----
-sidebar_position: 2
----
-
 # Configuration
 
-The `config.conf` file contains all the options you can change, these options are listed below:
+The `config.conf` file contains all the options you can change. Click on a configuration option below for more
-
+information about what it does, its default value, and the allowed values.
-# Configuration - External Tools
+
-
+- Auth
-## FFmpegPath
+    - [DisableBrowserAuth](/config/all-configuration-options#disablebrowserauth)
-
+
-Type: `string`
+- External
-
+    - [FFmpegPath](/config/all-configuration-options#ffmpegpath)
-Default: `""`
+
-
+- Download
-Allowed values: Any valid path or `""`
+    - [IgnoreOwnMessages](/config/all-configuration-options#ignoreownmessages)
-
+    - [DownloadPostsIncrementally](/config/all-configuration-options#downloadpostsincrementally)
-Description: This is the path to the FFmpeg executable (`ffmpeg.exe` on Windows and `ffmpeg` on Linux/macOS).
+    - [BypassContentForCreatorsWhoNoLongerExist](/config/all-configuration-options#bypasscontentforcreatorswhonolongerexist)
-If the path is not set then the program will try to find it in both the same directory as the OF-DL executable as well
+    - [DownloadDuplicatedMedia](/config/all-configuration-options#downloadduplicatedmedia)
-as the PATH environment variable.
+    - [SkipAds](/config/all-configuration-options#skipads)
-
+    - [DownloadPath](/config/all-configuration-options#downloadpath)
-:::note
+    - [DownloadOnlySpecificDates](/config/all-configuration-options#downloadonlyspecificdates)
-
+    - [DownloadDateSelection](/config/all-configuration-options#downloaddateselection)
-If you are using a Windows path, you will need to escape the backslashes, e.g. `"C:\\ffmpeg\\bin\\ffmpeg.exe"`
+    - [CustomDate](/config/all-configuration-options#customdate)
-For example, this is not valid: `"C:\some\path\ffmpeg.exe"`, but `"C:/some/path/ffmpeg.exe"` and `"C:\\some\\path\\ffmpeg.exe"` are both valid.
+    - [ShowScrapeSize](/config/all-configuration-options#showscrapesize)
-
+    - Media
-:::
+        - [DownloadAvatarHeaderPhoto](/config/all-configuration-options#downloadavatarheaderphoto)
-
+        - [DownloadPaidPosts](/config/all-configuration-options#downloadpaidposts)
-# Configuration - Download Settings
+        - [DownloadPosts](/config/all-configuration-options#downloadposts)
-
+        - [DownloadArchived](/config/all-configuration-options#downloadarchived)
-## DownloadAvatarHeaderPhoto
+        - [DownloadStreams](/config/all-configuration-options#downloadstreams)
-
+        - [DownloadStories](/config/all-configuration-options#downloadstories)
-Type: `boolean`
+        - [DownloadHighlights](/config/all-configuration-options#downloadhighlights)
-
+        - [DownloadMessages](/config/all-configuration-options#downloadmessages)
-Default: `true`
+        - [DownloadPaidMessages](/config/all-configuration-options#downloadpaidmessages)
-
+        - [DownloadImages](/config/all-configuration-options#downloadimages)
-Allowed values: `true`, `false`
+        - [DownloadVideos](/config/all-configuration-options#downloadvideos)
-
+        - [DownloadAudios](/config/all-configuration-options#downloadaudios)
-Description: Avatar and header images will be downloaded if set to `true`
+
-
+- File
-## DownloadPaidPosts
+    - [PaidPostFileNameFormat](/config/all-configuration-options#paidpostfilenameformat)
-
+    - [PostFileNameFormat](/config/all-configuration-options#postfilenameformat)
-Type: `boolean`
+    - [PaidMessageFileNameFormat](/config/all-configuration-options#paidmessagefilenameformat)
-
+    - [MessageFileNameFormat](/config/all-configuration-options#messagefilenameformat)
-Default: `true`
+    - [RenameExistingFilesWhenCustomFormatIsSelected](/config/all-configuration-options#renameexistingfileswhencustomformatisselected)
-
+
-Allowed values: `true`, `false`
+- [CreatorConfigs](/config/all-configuration-options#creatorconfigs)
-
+
-Description: Paid posts will be downloaded if set to `true`
+- Folder
-
+    - [FolderPerPaidPost](/config/all-configuration-options#folderperpaidpost)
-## DownloadPosts
+    - [FolderPerPost](/config/all-configuration-options#folderperpost)
-
+    - [FolderPerPaidMessage](/config/all-configuration-options#folderperpaidmessage)
-Type: `boolean`
+    - [FolderPerMessage](/config/all-configuration-options#folderpermessage)
-
+
-Default: `true`
+- Subscriptions
-
+    - [IncludeExpiredSubscriptions](/config/all-configuration-options#includeexpiredsubscriptions)
-Allowed values: `true`, `false`
+    - [IncludeRestrictedSubscriptions](/config/all-configuration-options#includerestrictedsubscriptions)
-
+    - [IgnoredUsersListName](/config/all-configuration-options#ignoreduserslistname)
-Description: Free posts will be downloaded if set to `true`
+
-
+- Interaction
-## DownloadArchived
+    - [NonInteractiveMode](/config/all-configuration-options#noninteractivemode)
-
+    - [NonInteractiveModeListName](/config/all-configuration-options#noninteractivemodelistname)
-Type: `boolean`
+    - [NonInteractiveModePurchasedTab](/config/all-configuration-options#noninteractivemodepurchasedtab)
-
+
-Default: `true`
+- Performance
-
+    - [Timeout](/config/all-configuration-options#timeout)
-Allowed values: `true`, `false`
+    - [LimitDownloadRate](/config/all-configuration-options#limitdownloadrate)
-
+    - [DownloadLimitInMbPerSec](/config/all-configuration-options#downloadlimitinmbpersec)
-Description: Posts in the "Archived" tab will be downloaded if set to `true`
+
-
+- Logging
-## DownloadStreams
+    - [LoggingLevel](/config/all-configuration-options#logginglevel)
-
-Type: `boolean`
-
-Default: `true`
-
-Allowed values: `true`, `false`
-
-Description: Posts in the "Streams" tab will be downloaded if set to `true`
-
-## DownloadStories
-
-Type: `boolean`
-
-Default: `true`
-
-Allowed values: `true`, `false`
-
-Description: Stories on a user's profile will be downloaded if set to `true`
-
-## DownloadHighlights
-
-Type: `boolean`
-
-Default: `true`
-
-Allowed values: `true`, `false`
-
-Description: Highlights on a user's will be downloaded if set to `true`
-
-## DownloadMessages
-
-Type: `boolean`
-
-Default: `true`
-
-Allowed values: `true`, `false`
-
-Description: Free media within messages (including paid message previews) will be downloaded if set to `true`
-
-## DownloadPaidMessages
-
-Type: `boolean`
-
-Default: `true`
-
-Allowed values: `true`, `false`
-
-Description: Paid media within messages (excluding paid message previews) will be downloaded if set to `true`
-
-## DownloadImages
-
-Type: `boolean`
-
-Default: `true`
-
-Allowed values: `true`, `false`
-
-Description: Images will be downloaded if set to `true`
-
-## DownloadVideos
-
-Type: `boolean`
-
-Default: `true`
-
-Allowed values: `true`, `false`
-
-Description: Videos will be downloaded if set to `true`
-
-## DownloadAudios
-
-Type: `boolean`
-
-Default: `true`
-
-Allowed values: `true`, `false`
-
-Description: Audios will be downloaded if set to `true`
-
-## IgnoreOwnMessages
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: By default (or when set to `false`), messages that were sent by yourself will be added to the metadata DB and any media which has been sent by yourself will be downloaded. If set to `true`, the program will not add messages sent by yourself to the metadata DB and will not download any media which has been sent by yourself.
-
-## DownloadPostsIncrementally
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: If set to `true`, only new posts will be downloaded from the date of the last post that was downloaded based off what's in the `user_data.db` file.
-If set to `false`, the default behaviour will apply, and all posts will be gathered and compared against the database to see if they need to be downloaded or not.
-
-## BypassContentForCreatorsWhoNoLongerExist
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: When a creator no longer exists (their account has been deleted), most of their content will be inaccessible.
-Purchased content, however, will still be accessible by downloading media usisng the "Download Purchased Tab" menu option
-or with the [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) config option when downloading media in non-interactive mode.
-
-## DownloadDuplicatedMedia
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: By default (or when set to `false`), the program will not download duplicated media. If set to `true`, duplicated media will be downloaded.
-
-## SkipAds
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: Posts and messages that contain #ad or free trial links will be ignored if set to `true`
-
-## DownloadPath
-
-Type: `string`
-
-Default: `""`
-
-Allowed values: Any valid path
-
-Description: If left blank then content will be downloaded to `__user_data__/sites/OnlyFans/{username}`.
-If you set the download path to `"S:/"`, then content will be downloaded to `S:/{username}`
-
-:::note
-
-If you are using a Windows path, you will need to escape the backslashes, e.g. `"C:\\Users\\user\\Downloads\\OnlyFans\\"`
-Please make sure your path ends with a `/`
-
-:::
-
-## DownloadOnlySpecificDates
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: If set to `true`, posts will be downloaded based on the [DownloadDateSelection](#downloaddateselection) and [CustomDate](#customdate) config options.
-If set to `false`, all posts will be downloaded.
-
-## DownloadDateSelection
-
-Type: `string`
-
-Default: `"before"`
-
-Allowed values: `"before"`, `"after"`
-
-Description: [DownloadOnlySpecificDates](#downloadonlyspecificdates) needs to be set to `true` for this to work. This will get all posts from before
-the date if set to `"before"`, and all posts from the date you specify up until the current date if set to `"after"`.
-The date you specify will be in the [CustomDate](#customdate) config option.
-
-## CustomDate
-
-Type: `string`
-
-Default: `null`
-
-Allowed values: Any date in `yyyy-mm-dd` format or `null`
-
-Description: [DownloadOnlySpecificDates](#downloadonlyspecificdates) needs to be set to `true` for this to work.
-This date will be used when you are trying to download between/after a certain date. See [DownloadOnlySpecificDates](#downloadonlyspecificdates) and
-[DownloadDateSelection](#downloaddateselection) for more information.
-
-# Configuration - File Settings
-
-## PaidPostFileNameFormat
-
-Type: `string`
-
-Default: `""`
-
-Allowed values: Any valid string
-
-Description: Please refer to [custom filename formats](/config/custom-filename-formats#paidpostfilenameformat) page to see what fields you can use.
-
-## PostFileNameFormat
-
-Type: `string`
-
-Default: `""`
-
-Allowed values: Any valid string
-
-Description: Please refer to the [custom filename formats](/config/custom-filename-formats#postfilenameformat) page to see what fields you can use.
-
-## PaidMessageFileNameFormat
-
-Type: `string`
-
-Default: `""`
-
-Allowed values: Any valid string
-
-Description: Please refer to [custom filename formats](/config/custom-filename-formats#paidmessagefilenameformat) page to see what fields you can use.
-
-## MessageFileNameFormat
-
-Type: `string`
-
-Default: `""`
-
-Allowed values: Any valid string
-
-Description: Please refer to [custom filename formats](/config/custom-filename-formats#messagefilenameformat) page to see what fields you can use.
-
-## RenameExistingFilesWhenCustomFormatIsSelected
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: When `true`, any current files downloaded will have the current format applied to them.
-When `false`, only new files will have the current format applied to them.
-
-# Configuration - Creator-Specific Configurations
-
-## CreatorConfigs
-
-Type: `object`
-
-Default: `{}`
-
-Allowed values: An array of Creator Config objects
-
-Description: This configuration options allows you to set file name formats for specific creators.
-This is useful if you want to have different file name formats for different creators. The values set here will override the global values set in the config file
-(see [PaidPostFileNameFormat](#paidpostfilenameformat), [PostFileNameFormat](#postfilenameformat),
-[PaidMessageFileNAmeFormat](#paidmessagefilenameformat), and [MessageFileNameFormat](#messagefilenameformat)).
-For more information on the file name formats, see the [custom filename formats](/config/custom-filename-formats) page.
-
-Example:
-```
-"CreatorConfigs": {
-    "creator_one": {
-        "PaidPostFileNameFormat": "{id}_{mediaid}_{filename}",
-        "PostFileNameFormat": "{username}_{id}_{mediaid}_{mediaCreatedAt}",
-        "PaidMessageFileNameFormat": "{id}_{mediaid}_{createdAt}",
-        "MessageFileNameFormat": "{id}_{mediaid}_{filename}"
-    },
-    "creator_two": {
-        "PaidPostFileNameFormat": "{id}_{mediaid}",
-        "PostFileNameFormat": "{username}_{id}_{mediaid}",
-        "PaidMessageFileNameFormat": "{id}_{mediaid}",
-        "MessageFileNameFormat": "{id}_{mediaid}"
-    }
-}
-```
-
-# Configuration - Folder Settings
-
-## FolderPerPaidPost
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: A folder will be created for each paid post (containing all the media for that post) if set to `true`.
-When set to `false`, paid post media will be downloaded into the `Posts/Paid` folder.
-
-## FolderPerPost
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: A folder will be created for each post (containing all the media for that post) if set to `true`.
-When set to `false`, post media will be downloaded into the `Posts/Free` folder.
-
-## FolderPerPaidMessage
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: A folder will be created for each paid message (containing all the media for that message) if set to `true`.
-When set to `false`, paid message media will be downloaded into the `Messages/Paid` folder.
-
-## FolderPerMessage
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: A folder will be created for each message (containing all the media for that message) if set to `true`.
-When set to `false`, message media will be downloaded into the `Messages/Free` folder.
-
-# Configuration - Subscription Settings
-
-## IncludeExpiredSubscriptions
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: If set to `true`, expired subscriptions will appear in the user list under the "Custom" menu option.
-
-## IncludeRestrictedSubscriptions
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: If set to `true`, media from restricted creators will be downloaded. If set to `false`, restricted creators will be ignored.
-
-## IgnoredUsersListName
-
-Type: `string`
-
-Default: `""`
-
-Allowed values: The name of a list of users you have created on OnlyFans or `""`
-
-Description: When set to the name of a list, users in the list will be ignored when scraping content.
-If set to `""` (or an invalid list name), no users will be ignored when scraping content.
-
-# Configuration - Interaction Settings
-
-## NonInteractiveMode
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: If set to `true`, the program will run without any input from the user. It will scrape all users automatically
-(unless [NonInteractiveModeListName](#noninteractivemodelistname) or [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) are configured).
-If set to `false`, the default behaviour will apply, and you will be able to choose an option from the menu.
-
-!!! warning
-
-    If NonInteractiveMode is enabled, you will be unable to authenticate OF-DL using the standard authentication method.
-    Before you can run OF-DL in NonInteractiveMode, you must either
-
-    1. Generate an auth.json file by running OF-DL with NonInteractiveMode disabled and authenticating OF-DL using the standard method **OR**
-    2. Generate an auth.json file by using a [legacy authentication method](/config/auth#legacy-methods)
-
-## NonInteractiveModeListName
-
-Type: `string`
-
-Default: `""`
-
-Allowed values: The name of a list of users you have created on OnlyFans or `""`
-
-Description: When set to the name of a list, non-interactive mode will download media from the list of users instead of all
-users (when [NonInteractiveMode](#noninteractivemode) is set to `true`). If set to `""`, all users will be scraped
-(unless [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) is configured).
-
-## NonInteractiveModePurchasedTab
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: When set to `true`, non-interactive mode will only download content from the Purchased tab
-(when [NonInteractiveMode](#noninteractivemode) is set to `true`). If set to `false`, all users will be scraped
-(unless [NonInteractiveModeListName](#noninteractivemodelistname) is configured).
-
-# Configuration - Performance Settings
-
-## Timeout
-
-Type: `integer`
-
-Default: `-1`
-
-Allowed values: Any positive integer or `-1`
-
-Description: You won't need to set this, but if you see errors about the configured timeout of 100 seconds elapsing then
-you could set this to be more than 100. It is recommended that you leave this as the default value.
-
-## LimitDownloadRate
-
-Type: `boolean`
-
-Default: `false`
-
-Allowed values: `true`, `false`
-
-Description: If set to `true`, the download rate will be limited to the value set in [DownloadLimitInMbPerSec](#downloadlimitinmbpersec).
-
-## DownloadLimitInMbPerSec
-
-Type: `integer`
-
-Default: `4`
-
-Allowed values: Any positive integer
-
-Description: The download rate in MB per second. This will only be used if [LimitDownloadRate](#limitdownloadrate) is set to `true`.
-
-# Configuration - Logging/Debug Settings
-
-## LoggingLevel
-
-Type: `string`
-
-Default: `"Error"`
-
-Allowed values: `"Verbose"`, `"Debug"`, `"Information"`, `"Warning"`, `"Error"`, `"Fatal"`
-
-Description: The level of logging that will be saved to the log files in the `logs` folder.
-When requesting help with an issue, it is recommended to set this to `"Verbose"` and provide the log file.

docs/config/custom-filename-formats.md

@@ -1,7 +1,3 @@
----
-sidebar_position: 3
----
-
 # Custom Filename Formats
 
 In the config.conf file you can now specify some custom filename formats that will be used when downloading files. I have had to add 4 new fields to the auth.json file, these are:

docs/installation/docker.md

@@ -1,8 +1,4 @@
----
+# Docker
-sidebar_position: 2
----
-
-# Docker
 
 ## Running OF-DL
 

docs/installation/linux.md

@@ -1,8 +1,4 @@
----
+# Linux
-sidebar_position: 3
----
-
-# Linux
 
 A Linux release of OF-DL is not available at this time, however you can run OF-DL on Linux using Docker.
 Please refer to the [Docker](/installation/docker) page for instructions on how to run OF-DL in a Docker container.
@@ -38,7 +34,7 @@ dotnet publish -p:Version=%VERSION% -c Release
 cd 'OF DL/bin/Release/net8.0'
 ```
 
-- Download the windows release as described on [here](/docs/installation/windows#installation).
+- Download the windows release as described on [here](/installation/windows#installation).
 - Add the `config.json` and `rules.json` files as well as the `cdm` folder to the `OF DL/bin/Release/net8.0` folder.
 
 - Run the application

docs/installation/macos.md

@@ -1,8 +1,4 @@
----
+# macOS
-sidebar_position: 4
----
-
-# macOS
 
 macOS releases of OF-DL are not available at this time, however you can run OF-DL on macOS using Docker.
 Please refer to the [Docker](/installation/docker) page for instructions on how to run OF-DL in a Docker container.

docs/installation/windows.md

@@ -1,8 +1,4 @@
----
+# Windows
-sidebar_position: 1
----
-
-# Windows
 
 ## Requirements
 

docs/running-the-program.md

@@ -1,8 +1,4 @@
----
+# Running the Program
-sidebar_position: 3
----
-
-# Running the Program
 
 Once you are happy you have filled everything in [auth.json](/config/auth) correctly, you can double click OF-DL.exe and you should see a command prompt window appear, it should look something like this:
 

excludes.txt

@@ -0,0 +1,2 @@
+excludes.txt
+rules.json

mkdocs.yml

@@ -1,5 +1,19 @@
 site_name: OF-DL Docs
 site_url: https://docs.ofdl.tools
+nav:
+  - Home: index.md
+  - Running the Program: running-the-program.md
+  - Config:
+    - Authentication: config/auth.md
+    - CDM (optional, but recommended): config/cdm.md
+    - Configuration: config/configuration.md
+    - All Configuration Options: config/all-configuration-options.md
+    - Custom Filename Formats: config/custom-filename-formats.md
+  - Installation:
+    - Windows: installation/windows.md
+    - macOS: installation/macos.md
+    - Linux: installation/linux.md
+    - Docker: installation/docker.md
 theme:
   name: material
   features: