From 7ea0c5f52545886fad450873d03f1e8322997740 Mon Sep 17 00:00:00 2001 From: Casper Sparre Date: Sat, 8 Mar 2025 14:43:10 +0100 Subject: [PATCH] Squashed branch 'dev-before-refactor' --- .editorconfig | 7 + .gitignore | 4 +- OF DL/Entities/Chats/ChatCollection.cs | 7 + OF DL/Entities/Chats/Chats.cs | 20 + OF DL/Entities/Config.cs | 6 + OF DL/Entities/Subscriptions.cs | 1 + OF DL/Helpers/APIHelper.cs | 430 +++++++++++++--- OF DL/Helpers/DBHelper.cs | 264 +++++++--- OF DL/Helpers/DownloadHelper.cs | 4 +- OF DL/Helpers/Interfaces/IAPIHelper.cs | 7 +- OF DL/Helpers/Interfaces/IDBHelper.cs | 3 + OF DL/OF DL.csproj | 11 + OF DL/Program.cs | 663 +++++++++++++++++++------ Publish_OF-DL.bat | 33 ++ excludes.txt | 2 + 15 files changed, 1150 insertions(+), 312 deletions(-) create mode 100644 OF DL/Entities/Chats/ChatCollection.cs create mode 100644 OF DL/Entities/Chats/Chats.cs create mode 100644 Publish_OF-DL.bat create mode 100644 excludes.txt diff --git a/.editorconfig b/.editorconfig index b3672fd..ba94cee 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,5 +5,12 @@ root = true charset = utf-8 indent_style = space indent_size = 4 +tab_width = 4 +end_of_line = crlf insert_final_newline = true trim_trailing_whitespace = true + +[*.{sln,csproj,xml,json,config}] +indent_size = 2 +indent_style = space +tab_width = 2 diff --git a/.gitignore b/.gitignore index 822cbd2..a1bb878 100644 --- a/.gitignore +++ b/.gitignore @@ -370,4 +370,6 @@ FodyWeavers.xsd !.gitea-actions/**/node_modules/ # venv -venv/ \ No newline at end of file +venv/ + +Publish/ diff --git a/OF DL/Entities/Chats/ChatCollection.cs b/OF DL/Entities/Chats/ChatCollection.cs new file mode 100644 index 0000000..86d5325 --- /dev/null +++ b/OF DL/Entities/Chats/ChatCollection.cs @@ -0,0 +1,7 @@ +namespace OF_DL.Entities.Chats +{ + public class ChatCollection + { + public Dictionary Chats { get; set; } = []; + } +} diff --git a/OF DL/Entities/Chats/Chats.cs b/OF DL/Entities/Chats/Chats.cs new file mode 100644 index 0000000..ba695f4 --- /dev/null +++ b/OF DL/Entities/Chats/Chats.cs @@ -0,0 +1,20 @@ +namespace OF_DL.Entities.Chats +{ + public class Chats + { + public List list { get; set; } + public bool hasMore { get; set; } + public int nextOffset { get; set; } + + public class Chat + { + public User withUser { get; set; } + public int unreadMessagesCount { get; set; } + } + + public class User + { + public int id { get; set; } + } + } +} diff --git a/OF DL/Entities/Config.cs b/OF DL/Entities/Config.cs index 0934422..0418043 100644 --- a/OF DL/Entities/Config.cs +++ b/OF DL/Entities/Config.cs @@ -107,6 +107,12 @@ namespace OF_DL.Entities // When enabled, post/message text is stored as-is without XML stripping. [ToggleableConfig] public bool DisableTextSanitization { get; set; } = false; + + public string[] NonInteractiveSpecificUsers { get; set; } = []; + public string[] NonInteractiveSpecificLists { get; set; } = []; + + public bool OutputBlockedUsers { get; set; } + public bool UpdateAllUserInfo { get; set; } } public class CreatorConfig : IFileNameFormatConfig diff --git a/OF DL/Entities/Subscriptions.cs b/OF DL/Entities/Subscriptions.cs index f5bacdd..7d668be 100644 --- a/OF DL/Entities/Subscriptions.cs +++ b/OF DL/Entities/Subscriptions.cs @@ -98,6 +98,7 @@ namespace OF_DL.Entities public object id { get; set; } public long? userId { get; set; } public int? subscriberId { get; set; } + public long? earningId { get; set; } public DateTime? date { get; set; } public int? duration { get; set; } public DateTime? startDate { get; set; } diff --git a/OF DL/Helpers/APIHelper.cs b/OF DL/Helpers/APIHelper.cs index ea209c8..4fb03dd 100644 --- a/OF DL/Helpers/APIHelper.cs +++ b/OF DL/Helpers/APIHelper.cs @@ -2,6 +2,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using OF_DL.Entities; using OF_DL.Entities.Archived; +using OF_DL.Entities.Chats; using OF_DL.Entities.Highlights; using OF_DL.Entities.Lists; using OF_DL.Entities.Messages; @@ -13,6 +14,7 @@ using OF_DL.Enumerations; using OF_DL.Enumurations; using Serilog; using Spectre.Console; +using System.Diagnostics; using System.Globalization; using System.Security.Cryptography; using System.Text; @@ -25,10 +27,15 @@ namespace OF_DL.Helpers; public class APIHelper : IAPIHelper { + private const int MAX_RETRIES = 10; + private const int DELAY_BEFORE_RETRY = 1000; + private static readonly JsonSerializerSettings m_JsonSerializerSettings; private readonly IDBHelper m_DBHelper; private readonly IDownloadConfig downloadConfig; private readonly Auth auth; + private HttpClient httpClient = new(); + private static DateTime? cachedDynamicRulesExpiration; private static DynamicRules? cachedDynamicRules; private const int MaxAttempts = 30; @@ -118,30 +125,58 @@ public class APIHelper : IAPIHelper } - private async Task BuildHeaderAndExecuteRequests(Dictionary getParams, string endpoint, HttpClient client) + private async Task BuildHeaderAndExecuteRequests(Dictionary getParams, string endpoint, HttpClient client, HttpMethod? method = null, int retryCount = 0) { - Log.Debug("Calling BuildHeaderAndExecuteRequests"); + Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", retryCount + 1); - HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint); - using var response = await client.SendAsync(request); - response.EnsureSuccessStatusCode(); - string body = await response.Content.ReadAsStringAsync(); + try + { + HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method); - Log.Debug(body); + Debug.WriteLine($"Executing {request.Method.Method.ToUpper()} request: {request.RequestUri}\r\n\t{GetParamsString(getParams)}"); - return body; + using var response = await client.SendAsync(request); + + if (Debugger.IsAttached && !response.IsSuccessStatusCode) + Debugger.Break(); + + response.EnsureSuccessStatusCode(); + string body = await response.Content.ReadAsStringAsync(); + + Log.Debug(body); + + return body; + } + catch (HttpRequestException ex) + { + if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && retryCount < MAX_RETRIES) + { + await Task.Delay(DELAY_BEFORE_RETRY); + return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, method, ++retryCount); + } + + throw; + } + + static string GetParamsString(Dictionary getParams) + => string.Join(" | ", getParams.Select(kv => $"{kv.Key}={kv.Value}")); } - private async Task BuildHttpRequestMessage(Dictionary getParams, string endpoint) + private async Task BuildHttpRequestMessage(Dictionary getParams, string endpoint, HttpMethod? method = null) { Log.Debug("Calling BuildHttpRequestMessage"); - string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); + method ??= HttpMethod.Get; + + string queryParams = ""; + + if (getParams.Any()) + queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); Dictionary headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams); - HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{queryParams}"); + HttpRequestMessage request = new(method, $"{Constants.API_URL}{endpoint}{queryParams}"); Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}"); @@ -166,18 +201,16 @@ public class APIHelper : IAPIHelper return input.All(char.IsDigit); } - - private static HttpClient GetHttpClient(IDownloadConfig? config = null) + private HttpClient GetHttpClient(IDownloadConfig? config = null) { - var client = new HttpClient(); + httpClient ??= new HttpClient(); if (config?.Timeout != null && config.Timeout > 0) { - client.Timeout = TimeSpan.FromSeconds(config.Timeout.Value); + httpClient.Timeout = TimeSpan.FromSeconds(config.Timeout.Value); } - return client; + return httpClient; } - /// /// this one is used during initialization only /// if the config option is not available then no modificatiotns will be done on the getParams @@ -215,7 +248,7 @@ public class APIHelper : IAPIHelper } - public async Task GetUserInfo(string endpoint) + public async Task GetUserInfo(string username, string endpoint) { Log.Debug($"Calling GetUserInfo: {endpoint}"); @@ -242,6 +275,10 @@ public class APIHelper : IAPIHelper response.EnsureSuccessStatusCode(); var body = await response.Content.ReadAsStringAsync(); user = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); + + if (user is not null && !endpoint.EndsWith("/me")) + await m_DBHelper.UpdateUserInfo(username, user); + return user; } catch (Exception ex) @@ -300,47 +337,44 @@ public class APIHelper : IAPIHelper try { Dictionary users = new(); - Subscriptions subscriptions = new(); + + int limit = 25; + int offset = 0; + + getParams["limit"] = limit.ToString(); + getParams["offset"] = offset.ToString(); Log.Debug("Calling GetAllSubscrptions"); - string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); - - subscriptions = JsonConvert.DeserializeObject(body); - if (subscriptions != null && subscriptions.hasMore) + while (true) { - getParams["offset"] = subscriptions.list.Count.ToString(); + string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient); - while (true) + if (string.IsNullOrWhiteSpace(body)) + break; + + Subscriptions? subscriptions = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); + + if (subscriptions?.list is null) + break; + + foreach (Subscriptions.List item in subscriptions.list) { - Subscriptions newSubscriptions = new(); - string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); + if (users.ContainsKey(item.username)) + continue; - if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]")) - { - newSubscriptions = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); - } - else - { - break; - } + bool isRestricted = item.isRestricted ?? false; + bool isRestrictedButAllowed = isRestricted && includeRestricted; - subscriptions.list.AddRange(newSubscriptions.list); - if (!newSubscriptions.hasMore) - { - break; - } - getParams["offset"] = subscriptions.list.Count.ToString(); + if (!isRestricted || isRestrictedButAllowed) + users.Add(item.username, item.id); } - } - foreach (Subscriptions.List subscription in subscriptions.list) - { - if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted)) - && !users.ContainsKey(subscription.username)) - { - users.Add(subscription.username, subscription.id); - } + if (!subscriptions.hasMore) + break; + + offset += limit; + getParams["offset"] = offset.ToString(); } return users; @@ -363,23 +397,20 @@ public class APIHelper : IAPIHelper { Dictionary getParams = new() { - { "offset", "0" }, - { "limit", "50" }, { "type", "active" }, { "format", "infinite"} }; + Log.Debug("Calling GetActiveSubscriptions"); + return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config); } public async Task?> GetExpiredSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config) { - Dictionary getParams = new() { - { "offset", "0" }, - { "limit", "50" }, { "type", "expired" }, { "format", "infinite"} }; @@ -389,6 +420,86 @@ public class APIHelper : IAPIHelper return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config); } + public async Task?> GetUsersWithProgress(string typeDisplay, string endpoint, StatusContext ctx, string? typeParam, bool offsetByCount) + { + int limit = 50; + int offset = 0; + bool includeRestricted = true; + + Dictionary getParams = new() + { + ["format"] = "infinite", + ["limit"] = limit.ToString(), + ["offset"] = offset.ToString() + }; + + if (!string.IsNullOrWhiteSpace(typeParam)) + getParams["type"] = typeParam; + + try + { + Dictionary users = []; + + Log.Debug("Calling GetUsersWithProgress"); + + bool isLastLoop = false; + while (true) + { + string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient); + + if (string.IsNullOrWhiteSpace(body)) + break; + + Subscriptions? subscriptions = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); + + if (subscriptions?.list is null) + break; + + foreach (Subscriptions.List item in subscriptions.list) + { + if (users.ContainsKey(item.username)) + continue; + + bool isRestricted = item.isRestricted ?? false; + bool isRestrictedButAllowed = isRestricted && includeRestricted; + + if (!isRestricted || isRestrictedButAllowed) + users.Add(item.username, item.id); + } + + ctx.Status($"[red]Getting {typeDisplay} Users\n[/] [red]Found {users.Count}[/]"); + ctx.Spinner(Spinner.Known.Dots); + ctx.SpinnerStyle(Style.Parse("blue")); + + if (isLastLoop) + break; + + if (!subscriptions.hasMore || subscriptions.list.Count == 0) + isLastLoop = true; + + offset += offsetByCount + ? subscriptions.list.Count + : limit; + + getParams["offset"] = offset.ToString(); + } + + return users; + } + catch (Exception ex) + { + Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); + Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); + if (ex.InnerException != null) + { + Console.WriteLine("\nInner Exception:"); + Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); + Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); + } + } + + return null; + } public async Task> GetLists(string endpoint, IDownloadConfig config) { @@ -407,7 +518,7 @@ public class APIHelper : IAPIHelper Dictionary lists = new(); while (true) { - string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); + string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); if (body == null) { @@ -472,7 +583,7 @@ public class APIHelper : IAPIHelper while (true) { - var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); + var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); if (body == null) { break; @@ -516,6 +627,66 @@ public class APIHelper : IAPIHelper } + public async Task?> GetUsersFromList(string endpoint, bool includeRestricted, IDownloadConfig config) + { + var model = new { list = new[] { new { id = int.MaxValue, username = string.Empty, isRestricted = false, isBlocked = false } }, nextOffset = 0, hasMore = false }; + + Log.Debug($"Calling GetUsersFromList - {endpoint}"); + + int limit = 50; + int offset = 0; + + Dictionary getParams = new() + { + { "offset", offset.ToString() }, + { "limit", limit.ToString() }, + { "format", "infinite" } + }; + + try + { + Dictionary users = []; + + while (true) + { + string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); + + if (string.IsNullOrWhiteSpace(body)) + break; + + var data = JsonConvert.DeserializeAnonymousType(body, model, m_JsonSerializerSettings); + + if (data is null) + break; + + foreach (var item in data.list) + { + if (users.ContainsKey(item.username)) + continue; + + bool isRestricted = item.isRestricted; + bool isRestrictedButAllowed = isRestricted && includeRestricted; + + if (!isRestricted || isRestrictedButAllowed) + users.Add(item.username, item.id); + } + + if (!data.hasMore) + break; + + + offset += data.nextOffset; + getParams["offset"] = offset.ToString(); + } + + return users; + } + catch (Exception ex) + { + throw; + } + } + public async Task> GetMedia(MediaType mediatype, string endpoint, string? username, @@ -557,7 +728,7 @@ public class APIHelper : IAPIHelper break; } - var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); + var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); if (mediatype == MediaType.Stories) @@ -730,7 +901,7 @@ public class APIHelper : IAPIHelper } - public async Task GetPaidPosts(string endpoint, string folder, string username, IDownloadConfig config, List paid_post_ids, StatusContext ctx) + public async Task GetPaidPosts(string endpoint, string folder, string username, long userId, IDownloadConfig config, List paid_post_ids, StatusContext ctx) { Log.Debug($"Calling GetPaidPosts - {username}"); @@ -739,13 +910,13 @@ public class APIHelper : IAPIHelper Purchased paidPosts = new(); PaidPostCollection paidPostCollection = new(); int post_limit = 50; + int offset = 0; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "skip_users", "all" }, - { "order", "publish_date_desc" }, { "format", "infinite" }, - { "author", username } + { "author", username }, }; var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); @@ -755,9 +926,10 @@ public class APIHelper : IAPIHelper ctx.SpinnerStyle(Style.Parse("blue")); if (paidPosts != null && paidPosts.hasMore) { - getParams["offset"] = paidPosts.list.Count.ToString(); while (true) { + offset += post_limit; + getParams["offset"] = offset.ToString(); Purchased newPaidPosts = new(); @@ -772,7 +944,6 @@ public class APIHelper : IAPIHelper { break; } - getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); } } @@ -781,6 +952,9 @@ public class APIHelper : IAPIHelper { if (purchase.responseType == "post" && purchase.media != null && purchase.media.Count > 0) { + if (purchase.fromUser.id != userId) + continue; // Ensures only posts from current model are included + List previewids = new(); if (purchase.previews != null) { @@ -938,7 +1112,7 @@ public class APIHelper : IAPIHelper ref getParams, downloadAsOf); - var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); + var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); posts = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); ctx.Status($"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); @@ -1096,7 +1270,7 @@ public class APIHelper : IAPIHelper { "skip_users", "all" } }; - var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); + var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); singlePost = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); if (singlePost != null) @@ -1153,7 +1327,7 @@ public class APIHelper : IAPIHelper } break; case VideoResolution._240: - if(medium.videoSources != null) + if (medium.videoSources != null) { if (!string.IsNullOrEmpty(medium.videoSources._240)) { @@ -1252,7 +1426,7 @@ public class APIHelper : IAPIHelper ref getParams, config.CustomDate); - var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); + var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); streams = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); @@ -1526,7 +1700,7 @@ public class APIHelper : IAPIHelper { { "limit", post_limit.ToString() }, { "order", "desc" }, - { "skip_users", "all" } + { "skip_users", "all" }, }; var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); @@ -1536,9 +1710,10 @@ public class APIHelper : IAPIHelper ctx.SpinnerStyle(Style.Parse("blue")); if (messages.hasMore) { - getParams["id"] = messages.list[^1].id.ToString(); while (true) { + getParams["id"] = messages.list[^1].id.ToString(); + Messages newmessages = new(); var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); @@ -1552,7 +1727,6 @@ public class APIHelper : IAPIHelper { break; } - getParams["id"] = newmessages.list[newmessages.list.Count - 1].id.ToString(); } } @@ -1880,7 +2054,7 @@ public class APIHelper : IAPIHelper } - public async Task GetPaidMessages(string endpoint, string folder, string username, IDownloadConfig config, StatusContext ctx) + public async Task GetPaidMessages(string endpoint, string folder, string username, long userId, IDownloadConfig config, StatusContext ctx) { Log.Debug($"Calling GetPaidMessages - {username}"); @@ -1889,13 +2063,14 @@ public class APIHelper : IAPIHelper Purchased paidMessages = new(); PaidMessageCollection paidMessageCollection = new(); int post_limit = 50; + int offset = 0; Dictionary getParams = new() { { "limit", post_limit.ToString() }, - { "order", "publish_date_desc" }, + { "skip_users", "all" }, { "format", "infinite" }, + { "offset", offset.ToString() }, { "author", username }, - { "skip_users", "all" } }; var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); @@ -1905,9 +2080,11 @@ public class APIHelper : IAPIHelper ctx.SpinnerStyle(Style.Parse("blue")); if (paidMessages != null && paidMessages.hasMore) { - getParams["offset"] = paidMessages.list.Count.ToString(); while (true) { + offset += post_limit; + getParams["offset"] = offset.ToString(); + string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); Purchased newpaidMessages = new(); Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams); @@ -1919,12 +2096,14 @@ public class APIHelper : IAPIHelper { looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); } + using (var loopresponse = await loopclient.SendAsync(looprequest)) { loopresponse.EnsureSuccessStatusCode(); var loopbody = await loopresponse.Content.ReadAsStringAsync(); newpaidMessages = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); } + paidMessages.list.AddRange(newpaidMessages.list); ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); @@ -1933,16 +2112,21 @@ public class APIHelper : IAPIHelper { break; } - getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); } } if (paidMessages.list != null && paidMessages.list.Count > 0) { + long ownUserId = Convert.ToInt64(auth.USER_ID); + long[] validUserIds = [ownUserId, userId]; + foreach (Purchased.List purchase in paidMessages.list.Where(p => p.responseType == "message").OrderByDescending(p => p.postedAt ?? p.createdAt)) { - if (!config.IgnoreOwnMessages || purchase.fromUser.id != Convert.ToInt32(auth.USER_ID)) + if (!config.IgnoreOwnMessages || purchase.fromUser.id != ownUserId) { + if (!validUserIds.Contains(purchase.fromUser.id)) + continue; // Ensures only messages from current model (or self) are included + if (purchase.postedAt != null) { await m_DBHelper.AddMessage(folder, purchase.id, purchase.text != null ? purchase.text : string.Empty, purchase.price != null ? purchase.price : "0", true, false, purchase.postedAt.Value, purchase.fromUser.id); @@ -2190,11 +2374,11 @@ public class APIHelper : IAPIHelper { JObject user = await GetUserInfoById($"/users/list?x[]={purchase.fromUser.id}"); - if(user is null) + if (user is null) { if (!config.BypassContentForCreatorsWhoNoLongerExist) { - if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}")) + if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}")) { purchasedTabUsers.Add($"Deleted User - {purchase.fromUser.id}", purchase.fromUser.id); } @@ -2244,7 +2428,7 @@ public class APIHelper : IAPIHelper { if (!config.BypassContentForCreatorsWhoNoLongerExist) { - if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}")) + if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}")) { purchasedTabUsers.Add($"Deleted User - {purchase.author.id}", purchase.author.id); } @@ -2647,6 +2831,94 @@ public class APIHelper : IAPIHelper return null; } + public async Task GetChats(string endpoint, IDownloadConfig config, bool onlyUnread) + { + Log.Debug($"Calling GetChats - {endpoint}"); + + try + { + Chats chats = new(); + ChatCollection collection = new(); + + int limit = 60; + Dictionary getParams = new() + { + { "limit", $"{limit}" }, + { "offset", "0" }, + { "skip_users", "all" }, + { "order", "recent" } + }; + + if (onlyUnread) + getParams["filter"] = "unread"; + + string body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); + chats = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); + + if (chats.hasMore) + { + getParams["offset"] = $"{chats.nextOffset}"; + + while (true) + { + string loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); + Chats newChats = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); + + chats.list.AddRange(newChats.list); + + if (!newChats.hasMore) + break; + + getParams["offset"] = $"{newChats.nextOffset}"; + } + } + + foreach (Chats.Chat chat in chats.list) + collection.Chats.TryAdd(chat.withUser.id, chat); + + return collection; + } + catch (Exception ex) + { + Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); + Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); + if (ex.InnerException != null) + { + Console.WriteLine("\nInner Exception:"); + Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); + Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); + } + } + + return null; + } + + public async Task MarkAsUnread(string endpoint, IDownloadConfig config) + { + Log.Debug($"Calling MarkAsUnread - {endpoint}"); + + try + { + var result = new { success = false }; + + string body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(config), HttpMethod.Delete); + result = JsonConvert.DeserializeAnonymousType(body, result); + + if (result?.success != true) + Console.WriteLine($"Failed to mark chat as unread! Endpoint: {endpoint}"); + } + catch (Exception ex) + { + Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); + Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); + if (ex.InnerException != null) + { + Console.WriteLine("\nInner Exception:"); + Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); + Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); + } + } + } public async Task GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp) { @@ -2762,7 +3034,7 @@ public class APIHelper : IAPIHelper using var response = await client.SendAsync(request); - Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}"); + Log.Debug($"CDRM Project Response (Attempt {attempt}): {await response.Content.ReadAsStringAsync()}"); response.EnsureSuccessStatusCode(); var body = await response.Content.ReadAsStringAsync(); diff --git a/OF DL/Helpers/DBHelper.cs b/OF DL/Helpers/DBHelper.cs index 697fa71..99e9cd7 100644 --- a/OF DL/Helpers/DBHelper.cs +++ b/OF DL/Helpers/DBHelper.cs @@ -1,18 +1,14 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using OF_DL.Enumurations; -using System.IO; using Microsoft.Data.Sqlite; -using Serilog; using OF_DL.Entities; +using Serilog; +using System.Text; namespace OF_DL.Helpers { public class DBHelper : IDBHelper { + private static readonly Dictionary _connections = []; + private readonly IDownloadConfig downloadConfig; public DBHelper(IDownloadConfig downloadConfig) @@ -32,9 +28,7 @@ namespace OF_DL.Helpers string dbFilePath = $"{folder}/Metadata/user_data.db"; // connect to the new database file - using SqliteConnection connection = new($"Data Source={dbFilePath}"); - // open the connection - connection.Open(); + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={dbFilePath}"); // create the 'medias' table using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection)) @@ -139,39 +133,44 @@ namespace OF_DL.Helpers { try { - using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db"); + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db"); Log.Debug("Database data source: " + connection.DataSource); - connection.Open(); - - using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection)) + using (SqliteCommand cmdUsers = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection)) { - await cmd.ExecuteNonQueryAsync(); + await cmdUsers.ExecuteNonQueryAsync(); + } + + using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, about VARCHAR NULL, expires_on TIMESTAMP NULL, photo_count INT NOT NULL, video_count INT NOT NULL, PRIMARY KEY(user_id));", connection)) + { + await cmdInfo.ExecuteNonQueryAsync(); + } + + using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info_blob (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, blob TEXT NULL, PRIMARY KEY(user_id));", connection)) + { + await cmdInfo.ExecuteNonQueryAsync(); } Log.Debug("Adding missing creators"); foreach (KeyValuePair user in users) { - using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection)) + using SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection); + checkCmd.Parameters.AddWithValue("@userId", user.Value); + + using var reader = await checkCmd.ExecuteReaderAsync(); + + if (!reader.Read()) { - checkCmd.Parameters.AddWithValue("@userId", user.Value); - using (var reader = await checkCmd.ExecuteReaderAsync()) - { - if (!reader.Read()) - { - using (SqliteCommand insertCmd = new($"INSERT INTO users (user_id, username) VALUES (@userId, @username);", connection)) - { - insertCmd.Parameters.AddWithValue("@userId", user.Value); - insertCmd.Parameters.AddWithValue("@username", user.Key); - await insertCmd.ExecuteNonQueryAsync(); - Log.Debug("Inserted new creator: " + user.Key); - } - } - else - { - Log.Debug("Creator " + user.Key + " already exists"); - } - } + using SqliteCommand insertCmd = new($"INSERT INTO users (user_id, username) VALUES (@userId, @username);", connection); + insertCmd.Parameters.AddWithValue("@userId", user.Value); + insertCmd.Parameters.AddWithValue("@username", user.Key); + + await insertCmd.ExecuteNonQueryAsync(); + Log.Debug("Inserted new creator: " + user.Key); + } + else + { + Log.Debug("Creator " + user.Key + " already exists"); } } @@ -194,9 +193,7 @@ namespace OF_DL.Helpers { try { - using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db"); - - connection.Open(); + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db"); using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection)) { @@ -243,12 +240,92 @@ namespace OF_DL.Helpers } } + public async Task> GetUsers() + { + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db"); + using SqliteCommand cmd = new("SELECT user_id, username FROM users", connection); + using SqliteDataReader reader = cmd.ExecuteReader(); + + Dictionary result = new(StringComparer.OrdinalIgnoreCase); + + while (reader.Read()) + { + long userId = reader.GetInt64(0); + string username = reader.GetString(1); + + result[username] = userId; + } + + return result; + } + + public async Task UpdateUserInfo(string username, User? user) + { + if (user?.id is null) + return; + + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db"); + Log.Debug("Database data source: " + connection.DataSource); + + await UpdateAsync(); + await UpdateBlobAsync(); + + async Task UpdateAsync() + { + using SqliteCommand cmdInfo = new( + "INSERT OR REPLACE INTO user_info (user_id, name, about, expires_on, photo_count, video_count) " + + "VALUES (@userId, @name, @about, @expiresOn, @photoCount, @videoCount);", + connection + ); + + cmdInfo.Parameters.AddWithValue("@userId", user.id); + cmdInfo.Parameters.AddWithValue("@name", user.name ?? user.username ?? username); + cmdInfo.Parameters.AddWithValue("@about", user.about); + cmdInfo.Parameters.AddWithValue("@expiresOn", user.subscribedByExpireDate); + cmdInfo.Parameters.AddWithValue("@photoCount", user.photosCount ?? 0); + cmdInfo.Parameters.AddWithValue("@videoCount", user.videosCount ?? 0); + + try + { + await cmdInfo.ExecuteNonQueryAsync(); + Log.Debug("Inserted or updated creator info: {Username:l}", user.username); + } + catch (Exception ex) + { + Log.Warning(ex, "Failed to update User Info for: {Username:l}", user.username); + } + } + + async Task UpdateBlobAsync() + { + using SqliteCommand cmdInfo = new( + "INSERT OR REPLACE INTO user_info_blob (user_id, name, blob) " + + "VALUES (@userId, @name, @blob);", + connection + ); + + cmdInfo.Parameters.AddWithValue("@userId", user.id); + cmdInfo.Parameters.AddWithValue("@name", user.name ?? user.username ?? username); + cmdInfo.Parameters.AddWithValue("@blob", Newtonsoft.Json.JsonConvert.SerializeObject(user)); + + try + { + await cmdInfo.ExecuteNonQueryAsync(); + Log.Debug("Inserted or updated creator blob: {Username:l}", user.username); + } + catch (Exception ex) + { + Log.Warning(ex, "Failed to update User Info Blob for: {Username:l}", user.username); + } + } + } + public async Task AddMessage(string folder, long post_id, string message_text, string price, bool is_paid, bool is_archived, DateTime created_at, long user_id) { try { - using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); - connection.Open(); + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); + await EnsureCreatedAtColumnExists(connection, "messages"); using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection); cmd.Parameters.AddWithValue("@post_id", post_id); @@ -286,8 +363,8 @@ namespace OF_DL.Helpers { try { - using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); - connection.Open(); + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); + await EnsureCreatedAtColumnExists(connection, "posts"); using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection); cmd.Parameters.AddWithValue("@post_id", post_id); @@ -324,8 +401,8 @@ namespace OF_DL.Helpers { try { - using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); - connection.Open(); + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); + await EnsureCreatedAtColumnExists(connection, "stories"); using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection); cmd.Parameters.AddWithValue("@post_id", post_id); @@ -362,8 +439,8 @@ namespace OF_DL.Helpers { try { - using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); - connection.Open(); + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); + await EnsureCreatedAtColumnExists(connection, "medias"); StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id"); if (downloadConfig.DownloadDuplicatedMedia) @@ -400,22 +477,21 @@ namespace OF_DL.Helpers { try { - bool downloaded = false; + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); - using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db")) + StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id"); + if (downloadConfig.DownloadDuplicatedMedia) { - StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id"); - if(downloadConfig.DownloadDuplicatedMedia) - { - sql.Append(" and api_type=@api_type"); - } - - connection.Open(); - using SqliteCommand cmd = new (sql.ToString(), connection); - cmd.Parameters.AddWithValue("@media_id", media_id); - cmd.Parameters.AddWithValue("@api_type", api_type); - downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync()); + sql.Append(" and api_type=@api_type"); } + + connection.Open(); + using SqliteCommand cmd = new(sql.ToString(), connection); + cmd.Parameters.AddWithValue("@media_id", media_id); + cmd.Parameters.AddWithValue("@api_type", api_type); + + bool downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync()); + return downloaded; } catch (Exception ex) @@ -435,8 +511,7 @@ namespace OF_DL.Helpers public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at) { - using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); - connection.Open(); + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); // Construct the update command StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id"); @@ -463,25 +538,21 @@ namespace OF_DL.Helpers public async Task GetStoredFileSize(string folder, long media_id, string api_type) { - long size; - using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db")) - { - connection.Open(); - using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection); - cmd.Parameters.AddWithValue("@media_id", media_id); - cmd.Parameters.AddWithValue("@api_type", api_type); - size = Convert.ToInt64(await cmd.ExecuteScalarAsync()); - } + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); + + using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection); + cmd.Parameters.AddWithValue("@media_id", media_id); + cmd.Parameters.AddWithValue("@api_type", api_type); + + long size = Convert.ToInt64(await cmd.ExecuteScalarAsync()); return size; } public async Task GetMostRecentPostDate(string folder) { - DateTime? mostRecentDate = null; - using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db")) - { - connection.Open(); - using SqliteCommand cmd = new(@" + SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); + + using SqliteCommand cmd = new(@" SELECT MIN(created_at) AS created_at FROM ( @@ -497,13 +568,14 @@ namespace OF_DL.Helpers ON P.post_id = m.post_id WHERE m.downloaded = 0 )", connection); - var scalarValue = await cmd.ExecuteScalarAsync(); - if(scalarValue != null && scalarValue != DBNull.Value) - { - mostRecentDate = Convert.ToDateTime(scalarValue); - } + + var scalarValue = await cmd.ExecuteScalarAsync(); + if (scalarValue != null && scalarValue != DBNull.Value) + { + return Convert.ToDateTime(scalarValue); } - return mostRecentDate; + + return null; } private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName) @@ -527,5 +599,35 @@ namespace OF_DL.Helpers await alterCmd.ExecuteNonQueryAsync(); } } + + public static void CloseAllConnections() + { + foreach (SqliteConnection cn in _connections.Values) + { + cn?.Close(); + cn?.Dispose(); + } + + _connections.Clear(); + } + + private static async Task GetAndOpenConnectionAsync(string connectionString, int numberOfRetries = 2) + { + try + { + SqliteConnection connection = new(connectionString); + connection.Open(); + + return connection; + } + catch (Exception) + { + if (--numberOfRetries <= 0) + throw; + + await Task.Delay(300); + return await GetAndOpenConnectionAsync(connectionString, numberOfRetries); + } + } } } diff --git a/OF DL/Helpers/DownloadHelper.cs b/OF DL/Helpers/DownloadHelper.cs index f5ad8e2..a62f878 100644 --- a/OF DL/Helpers/DownloadHelper.cs +++ b/OF DL/Helpers/DownloadHelper.cs @@ -921,7 +921,7 @@ public class DownloadHelper : IDownloadHelper memoryStream.Seek(0, SeekOrigin.Begin); MD5 md5 = MD5.Create(); - byte[] hash = md5.ComputeHash(memoryStream); + byte[] hash = await md5.ComputeHashAsync(memoryStream); memoryStream.Seek(0, SeekOrigin.Begin); if (!avatarMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant())) { @@ -964,7 +964,7 @@ public class DownloadHelper : IDownloadHelper memoryStream.Seek(0, SeekOrigin.Begin); MD5 md5 = MD5.Create(); - byte[] hash = md5.ComputeHash(memoryStream); + byte[] hash = await md5.ComputeHashAsync(memoryStream); memoryStream.Seek(0, SeekOrigin.Begin); if (!headerMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant())) { diff --git a/OF DL/Helpers/Interfaces/IAPIHelper.cs b/OF DL/Helpers/Interfaces/IAPIHelper.cs index d94d509..7262d77 100644 --- a/OF DL/Helpers/Interfaces/IAPIHelper.cs +++ b/OF DL/Helpers/Interfaces/IAPIHelper.cs @@ -18,17 +18,18 @@ namespace OF_DL.Helpers Task GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp); Task> GetLists(string endpoint, IDownloadConfig config); Task> GetListUsers(string endpoint, IDownloadConfig config); + Task?> GetUsersFromList(string endpoint, bool includeRestricted, IDownloadConfig config); Task> GetMedia(MediaType mediatype, string endpoint, string? username, string folder, IDownloadConfig config, List paid_post_ids); - Task GetPaidPosts(string endpoint, string folder, string username, IDownloadConfig config, List paid_post_ids, StatusContext ctx); + Task GetPaidPosts(string endpoint, string folder, string username, long userId, IDownloadConfig config, List paid_post_ids, StatusContext ctx); Task GetPosts(string endpoint, string folder, IDownloadConfig config, List paid_post_ids, StatusContext ctx); Task GetPost(string endpoint, string folder, IDownloadConfig config); Task GetStreams(string endpoint, string folder, IDownloadConfig config, List paid_post_ids, StatusContext ctx); Task GetArchived(string endpoint, string folder, IDownloadConfig config, StatusContext ctx); Task GetMessages(string endpoint, string folder, IDownloadConfig config, StatusContext ctx); - Task GetPaidMessages(string endpoint, string folder, string username, IDownloadConfig config, StatusContext ctx); + Task GetPaidMessages(string endpoint, string folder, string username, long userId, IDownloadConfig config, StatusContext ctx); Task> GetPurchasedTabUsers(string endpoint, IDownloadConfig config, Dictionary users); Task> GetPurchasedTab(string endpoint, string folder, IDownloadConfig config, Dictionary users); - Task GetUserInfo(string endpoint); + Task GetUserInfo(string username, string endpoint); Task GetUserInfoById(string endpoint); Dictionary GetDynamicHeaders(string path, string queryParam); Task> GetActiveSubscriptions(string endpoint, bool includeRestrictedSubscriptions, IDownloadConfig config); diff --git a/OF DL/Helpers/Interfaces/IDBHelper.cs b/OF DL/Helpers/Interfaces/IDBHelper.cs index ab55249..cb91fb3 100644 --- a/OF DL/Helpers/Interfaces/IDBHelper.cs +++ b/OF DL/Helpers/Interfaces/IDBHelper.cs @@ -1,3 +1,5 @@ +using OF_DL.Entities; + namespace OF_DL.Helpers { public interface IDBHelper @@ -13,5 +15,6 @@ namespace OF_DL.Helpers Task GetStoredFileSize(string folder, long media_id, string api_type); Task CheckDownloaded(string folder, long media_id, string api_type); Task GetMostRecentPostDate(string folder); + Task UpdateUserInfo(string username, User? user); } } diff --git a/OF DL/OF DL.csproj b/OF DL/OF DL.csproj index 2ee9f51..508a54d 100644 --- a/OF DL/OF DL.csproj +++ b/OF DL/OF DL.csproj @@ -7,8 +7,15 @@ enable enable Icon\download.ico + 12 + true + true + + CS0168;CS0219;CS0472;CS1998;CS8073;CS8600;CS8602;CS8603;CS8604;CS8605;CS8613;CS8618;CS8622;CS8625;CS8629;SYSLIB0021;AsyncFixer01;AsyncFixer02 + + @@ -24,6 +31,7 @@ + @@ -37,12 +45,15 @@ Always + true Always + true Always + true diff --git a/OF DL/Program.cs b/OF DL/Program.cs index ae0e740..671facb 100644 --- a/OF DL/Program.cs +++ b/OF DL/Program.cs @@ -1,7 +1,9 @@ +using Akka.Configuration; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using OF_DL.Entities; using OF_DL.Entities.Archived; +using OF_DL.Entities.Chats; using OF_DL.Entities.Messages; using OF_DL.Entities.Post; using OF_DL.Entities.Purchased; @@ -10,17 +12,16 @@ using OF_DL.Enumerations; using OF_DL.Enumurations; using OF_DL.Helpers; using Serilog; +using Serilog.Context; using Serilog.Core; using Serilog.Events; using Spectre.Console; -using System.IO; +using System.Diagnostics; using System.Reflection; using System.Runtime.InteropServices; +using System.Text; using System.Text.RegularExpressions; using static OF_DL.Entities.Messages.Messages; -using Akka.Configuration; -using System.Text; -using static Akka.Actor.ProviderSelection; namespace OF_DL; @@ -44,15 +45,15 @@ public class Program AuthHelper authHelper = new(); Task setupBrowserTask = authHelper.SetupBrowser(runningInDocker); - Task.Delay(1000).Wait(); + await Task.Delay(1000); if (!setupBrowserTask.IsCompleted) { AnsiConsole.MarkupLine($"[yellow]Downloading dependencies. Please wait ...[/]"); } - setupBrowserTask.Wait(); + await setupBrowserTask; Task getAuthTask = authHelper.GetAuthFromBrowser(); - Task.Delay(5000).Wait(); + await Task.Delay(5000); if (!getAuthTask.IsCompleted) { if (runningInDocker) @@ -82,7 +83,7 @@ public class Program AnsiConsole.MarkupLine($"[red]Press any key to exit.[/]"); Log.Error(e, "auth invalid after attempt to get auth from browser"); - Environment.Exit(2); + ExitWithCode(2); } if (auth == null) @@ -94,7 +95,7 @@ public class Program AnsiConsole.MarkupLine($"[red]Press any key to exit.[/]"); Log.Error("auth invalid after attempt to get auth from browser"); - Environment.Exit(2); + ExitWithCode(2); } else { @@ -111,21 +112,28 @@ public class Program levelSwitch.MinimumLevel = LogEventLevel.Error; //set initial level (until we've read from config) Log.Logger = new LoggerConfiguration() + .Enrich.FromLogContext() + .Enrich.WithProperty("Application", "OF_DL") + .Enrich.WithProperty("StartTime", $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} ") + .Enrich.WithProperty("MachineName", Environment.MachineName) .MinimumLevel.ControlledBy(levelSwitch) - .WriteTo.File("logs/OFDL.txt", rollingInterval: RollingInterval.Day) + .WriteTo.File("logs/OFDL.txt", rollingInterval: RollingInterval.Day, restrictedToMinimumLevel: LogEventLevel.Error) + .WriteTo.Seq("https://seq.cajetan.dk") .CreateLogger(); AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red)); AnsiConsole.Markup("Documentation: [link]https://docs.ofdl.tools/[/]\n"); AnsiConsole.Markup("Discord server: [link]https://discord.com/invite/6bUW8EJ53j[/]\n\n"); + ExitIfOtherProcess(); + //Remove config.json and convert to config.conf if (File.Exists("config.json")) { AnsiConsole.Markup("[green]config.json located successfully!\n[/]"); try { - string jsonText = File.ReadAllText("config.json"); + string jsonText = await File.ReadAllTextAsync("config.json"); var jsonConfig = JsonConvert.DeserializeObject(jsonText); if (jsonConfig != null) @@ -226,7 +234,7 @@ public class Program hoconConfig.AppendLine($" LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\""); hoconConfig.AppendLine("}"); - File.WriteAllText("config.conf", hoconConfig.ToString()); + await File.WriteAllTextAsync("config.conf", hoconConfig.ToString()); File.Delete("config.json"); AnsiConsole.Markup("[green]config.conf created successfully from config.json!\n[/]"); } @@ -242,7 +250,8 @@ public class Program { Console.ReadKey(); } - Environment.Exit(3); + + ExitWithCode(3); } } @@ -252,7 +261,7 @@ public class Program AnsiConsole.Markup("[green]config.conf located successfully!\n[/]"); try { - string hoconText = File.ReadAllText("config.conf"); + string hoconText = await File.ReadAllTextAsync("config.conf"); var hoconConfig = ConfigurationFactory.ParseString(hoconText); @@ -371,7 +380,8 @@ public class Program { Console.ReadKey(); } - Environment.Exit(3); + + ExitWithCode(3); } } else @@ -482,21 +492,77 @@ public class Program { Console.ReadKey(); } - Environment.Exit(3); + + ExitWithCode(3); } if (args is not null && args.Length > 0) { - const string NON_INTERACTIVE_ARG = "--non-interactive"; + const string NON_INTERACTIVE_ARG = "--non-interactive"; + const string SPECIFIC_LISTS_ARG = "--specific-lists"; + const string SPECIFIC_USERS_ARG = "--specific-users"; - if (args.Any(a => NON_INTERACTIVE_ARG.Equals(NON_INTERACTIVE_ARG, StringComparison.OrdinalIgnoreCase))) - { - cliNonInteractive = true; - Log.Debug("NonInteractiveMode set via command line"); - } + if (args.Any(a => NON_INTERACTIVE_ARG.Equals(NON_INTERACTIVE_ARG, StringComparison.OrdinalIgnoreCase))) + { + AnsiConsole.Markup($"[grey]Non-Interactive Mode enabled through command-line argument![/]\n"); - Log.Debug("Additional arguments:"); + config.NonInteractiveMode = true; + + Log.Logger = Log.Logger.ForContext("Mode", "NonInteractiveMode"); + + int indexOfSpecificListsArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_LISTS_ARG, StringComparison.OrdinalIgnoreCase)); + int indexOfSpecificUsersArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_USERS_ARG, StringComparison.OrdinalIgnoreCase)); + char[] separator = [',']; + + if (indexOfSpecificListsArg >= 0) + { + int indexOfListValues = indexOfSpecificListsArg + 1; + + string[] strListValues = args.ElementAtOrDefault(indexOfListValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? []; + if (strListValues.Length > 0) + { + config.NonInteractiveSpecificLists = strListValues; + config.NonInteractiveModeListName = string.Empty; + + Log.Logger = Log.Logger.ForContext("NonInteractiveSpecificLists", string.Join(",", strListValues)); + } + } + + if (indexOfSpecificUsersArg >= 0) + { + int indexOfUserValues = indexOfSpecificUsersArg + 1; + string[] strUserValues = args.ElementAtOrDefault(indexOfUserValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? []; + if (strUserValues.Length > 0) + { + config.NonInteractiveSpecificUsers = strUserValues; + + Log.Logger = Log.Logger.ForContext("NonInteractiveSpecificUsers", string.Join(",", strUserValues)); + } + } + } + + const string OUTPUT_BLOCKED_USERS_ARG = "--output-blocked"; + + if (args.Any(a => OUTPUT_BLOCKED_USERS_ARG.Equals(a, StringComparison.OrdinalIgnoreCase))) + { + config.NonInteractiveMode = true; + config.OutputBlockedUsers = true; + + Log.Logger = Log.Logger.ForContext("Mode", "OutputBlockedUsers"); + } + + const string UPDATE_ALL_USER_INFO_ARG = "--update-userinfo"; + + if (args.Any(a => UPDATE_ALL_USER_INFO_ARG.Equals(a, StringComparison.OrdinalIgnoreCase))) + { + config.NonInteractiveMode = true; + config.UpdateAllUserInfo = true; + + Log.Logger = Log.Logger.ForContext("Mode", "UpdateAllUserInfo"); + } + + Log.Debug("Additional arguments:"); foreach (string argument in args) { Log.Debug(argument); @@ -521,7 +587,8 @@ public class Program { Console.ReadKey(); } - Environment.Exit(1); + + ExitWithCode(1); } else { @@ -606,8 +673,8 @@ public class Program Log.Information("Auth file found but could not be deserialized"); if (!config!.DisableBrowserAuth) { - Log.Debug("Deleting auth.json"); - File.Delete("auth.json"); + //Log.Debug("Deleting auth.json"); + //File.Delete("auth.json"); } if (cliNonInteractive) @@ -618,7 +685,7 @@ public class Program AnsiConsole.MarkupLine($"[red]Press any key to exit.[/]"); Console.ReadKey(); - Environment.Exit(2); + ExitWithCode(2); } @@ -634,7 +701,7 @@ public class Program AnsiConsole.MarkupLine($"[red]Press any key to exit.[/]"); Console.ReadKey(); - Environment.Exit(2); + ExitWithCode(2); } } } @@ -648,7 +715,7 @@ public class Program AnsiConsole.MarkupLine($"[red]Press any key to exit.[/]"); Console.ReadKey(); - Environment.Exit(2); + ExitWithCode(2); } if (!config!.DisableBrowserAuth) @@ -663,7 +730,7 @@ public class Program AnsiConsole.MarkupLine($"[red]Press any key to exit.[/]"); Console.ReadKey(); - Environment.Exit(2); + ExitWithCode(2); } } @@ -675,9 +742,10 @@ public class Program AnsiConsole.Markup("[green]rules.json located successfully!\n[/]"); try { - JsonConvert.DeserializeObject(File.ReadAllText("rules.json")); + string rulesJson = await File.ReadAllTextAsync("rules.json"); + DynamicRules? dynamicRules = JsonConvert.DeserializeObject(rulesJson); Log.Debug($"Rules.json: "); - Log.Debug(JsonConvert.SerializeObject(File.ReadAllText("rules.json"), Formatting.Indented)); + Log.Debug(JsonConvert.SerializeObject(dynamicRules, Formatting.Indented)); } catch (Exception e) { @@ -691,7 +759,8 @@ public class Program { Console.ReadKey(); } - Environment.Exit(2); + + ExitWithCode(2); } } @@ -834,7 +903,8 @@ public class Program { Console.ReadKey(); } - Environment.Exit(4); + + ExitWithCode(4); } if (!File.Exists(Path.Join(WidevineClient.Widevine.Constants.DEVICES_FOLDER, WidevineClient.Widevine.Constants.DEVICE_NAME, "device_client_id_blob"))) @@ -867,7 +937,7 @@ public class Program //Check if auth is valid var apiHelper = new APIHelper(auth, config); - Entities.User? validate = await apiHelper.GetUserInfo($"/users/me"); + Entities.User? validate = await apiHelper.GetUserInfo(string.Empty, $"/users/me"); if (validate == null || (validate?.name == null && validate?.username == null)) { Log.Error("Auth failed"); @@ -875,10 +945,10 @@ public class Program auth = null; if (!config!.DisableBrowserAuth) { - if (File.Exists("auth.json")) - { - File.Delete("auth.json"); - } + //if (File.Exists("auth.json")) + //{ + // File.Delete("auth.json"); + //} } if (!cliNonInteractive && !config!.DisableBrowserAuth) @@ -890,13 +960,35 @@ public class Program { AnsiConsole.MarkupLine($"\n[red]Auth failed. Please try again or use other authentication methods detailed here:[/]\n"); AnsiConsole.MarkupLine($"[link]https://docs.ofdl.tools/config/auth[/]\n"); - Console.ReadKey(); - Environment.Exit(2); - } + + Console.ReadKey(); + + ExitWithCode(2); + } } - AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]"); - await DownloadAllData(apiHelper, auth, config); + Log.Information("Logged In successfully as {Name:l} ({Username:l})", validate.name, validate.username); + AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} ({validate.username})\n[/]"); + + try + { + if (config.OutputBlockedUsers) + { + await DownloadBlockedOrExpiredUsers(apiHelper, config); + return; + } + else if (config.UpdateAllUserInfo) + { + await UpdateAlluserInfo(apiHelper, config); + return; + } + + await DownloadAllData(apiHelper, auth, config); + } + finally + { + DBHelper.CloseAllConnections(); + } } catch (Exception ex) { @@ -912,13 +1004,117 @@ public class Program if (!cliNonInteractive) { Console.ReadKey(); - } - Environment.Exit(5); + } + + ExitWithCode(5); } + finally + { + Console.WriteLine(); + + AnsiConsole.Markup($"Exiting after successful run..\n"); + await Task.Delay(2000); + + Log.CloseAndFlush(); + } } + private static async Task DownloadBlockedOrExpiredUsers(APIHelper m_ApiHelper, Entities.Config Config) + { + const string OUTPUT_FILE_BLOCKED = "blocked-users.json"; + const string OUTPUT_FILE_EXPIRED = "expired-users.json"; - private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config) + await GetUsers("Blocked", "/users/blocked", OUTPUT_FILE_BLOCKED); + await GetUsers("Expired", "/subscriptions/subscribes", OUTPUT_FILE_EXPIRED, typeParam: "expired", offsetByCount: false); + + async Task GetUsers(string typeDisplay, string uri, string outputFile, string? typeParam = null, bool offsetByCount = true) + { + Dictionary? users = null; + + await AnsiConsole + .Status() + .StartAsync($"[red]Getting {typeDisplay} Users[/]", async ctx => + { + users = await m_ApiHelper.GetUsersWithProgress(typeDisplay, uri, ctx, typeParam, offsetByCount); + }); + + Console.WriteLine(); + + if (users is null || users.Count == 0) + { + AnsiConsole.Markup($"[green]No {typeDisplay} Users found.\n[/]"); + } + else + { + AnsiConsole.Markup($"[green]Found {users.Count} {typeDisplay} Users, saving to '{outputFile}'\n[/]"); + string json = JsonConvert.SerializeObject(users, Formatting.Indented); + await File.WriteAllTextAsync(outputFile, json); + } + } + } + + private static async Task UpdateAlluserInfo(APIHelper m_ApiHelper, Entities.Config Config) + { + DBHelper dbHelper = new(Config); + + await dbHelper.CreateUsersDB([]); + + Dictionary users = await dbHelper.GetUsers(); + + Console.WriteLine(); + Log.Information("Updating User Info for '{UserCount}' users", users.Count); + AnsiConsole.Markup($"[green]Updating User Info for '{users.Count}' users\n[/]"); + + Console.WriteLine(); + await AnsiConsole.Progress() + .Columns(new ProgressBarColumn(), new PercentageColumn(), new TaskDescriptionColumn { Alignment = Justify.Left }) + .StartAsync(RunUpdateAsync); + + async Task RunUpdateAsync(ProgressContext context) + { + ProgressTask updateTask = null; + + int maxUsernameLength = users.Keys.Max(s => s.Length); + + foreach ((string username, long userId) in users) + { + string description = $"Updating '{username}'".PadRight(11 + maxUsernameLength); + double prevValue = updateTask?.Value ?? 0; + + updateTask = context.AddTask(description, true, users.Count); + updateTask.Value = prevValue; + + using (LogContext.PushProperty("Username", username)) + using (LogContext.PushProperty("UserId", userId)) + using (LogContext.PushProperty("UserNum", prevValue+1)) + using (LogContext.PushProperty("UserTotal", users.Count)) + { + try + { + Log.Information("[{UserNum:0} of {UserTotal}] Updating User Info for for: {Username:l}"); + User? user_info = await m_ApiHelper.GetUserInfo(username, $"/users/{username}"); + await dbHelper.UpdateUserInfo(username, user_info); + + updateTask.Description = $"{description} - COMPLETE"; + } + catch (Exception ex) + { + Log.Warning(ex, "[{UserNum:0} of {UserTotal}] Failed to update User Info for: {Username:l}"); + AnsiConsole.Markup($"[red]Failed to update User Info for '{username}'\n[/]"); + + updateTask.Description = $"{description} - FAILED: {ex.Message}"; + } + finally + { + updateTask.Increment(1); + updateTask.StopTask(); + } + } + } + } + } + + private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config) { DBHelper dBHelper = new DBHelper(Config); @@ -927,27 +1123,33 @@ public class Program do { DateTime startTime = DateTime.Now; - Dictionary users = new(); - Dictionary activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config); + Dictionary users = new(); - Log.Debug("Subscriptions: "); + Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", Config.IncludeRestrictedSubscriptions); + AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); + Dictionary subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? []; - foreach (KeyValuePair activeSub in activeSubs) - { + Log.Debug("Subscriptions: "); + foreach (KeyValuePair activeSub in subsActive) + { if (!users.ContainsKey(activeSub.Key)) { users.Add(activeSub.Key, activeSub.Value); Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}"); } } + if (Config!.IncludeExpiredSubscriptions) { Log.Debug("Inactive Subscriptions: "); - Dictionary expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config); - foreach (KeyValuePair expiredSub in expiredSubs) - { - if (!users.ContainsKey(expiredSub.Key)) + Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", Config.IncludeRestrictedSubscriptions); + AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); + Dictionary subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? []; + + foreach (KeyValuePair expiredSub in subsExpired) + { + if (!users.ContainsKey(expiredSub.Key)) { users.Add(expiredSub.Key, expiredSub.Value); Log.Debug($"Name: {expiredSub.Key} ID: {expiredSub.Value}"); @@ -970,26 +1172,58 @@ public class Program var ignoredUsernames = await m_ApiHelper.GetListUsers($"/lists/{ignoredUsersListId}/users", Config) ?? []; users = users.Where(x => !ignoredUsernames.Contains(x.Key)).ToDictionary(x => x.Key, x => x.Value); } - } + } - await dBHelper.CreateUsersDB(users); - KeyValuePair> hasSelectedUsersKVP; - if(Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab) - { - hasSelectedUsersKVP = new KeyValuePair>(true, new Dictionary { { "PurchasedTab", 0 } }); - } - else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName)) - { - hasSelectedUsersKVP = new KeyValuePair>(true, users); - } - else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName)) - { - var listId = lists[Config.NonInteractiveModeListName]; - var listUsernames = await m_ApiHelper.GetListUsers($"/lists/{listId}/users", Config) ?? []; - var selectedUsers = users.Where(x => listUsernames.Contains(x.Key)).Distinct().ToDictionary(x => x.Key, x => x.Value); - hasSelectedUsersKVP = new KeyValuePair>(true, selectedUsers); - } - else + KeyValuePair> hasSelectedUsersKVP = new(false, []); + if (Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab) + { + hasSelectedUsersKVP = new KeyValuePair>(true, new Dictionary { { "PurchasedTab", 0 } }); + } + else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificLists is not null && Config.NonInteractiveSpecificLists.Length > 0) + { + Dictionary usersFromLists = new(StringComparer.OrdinalIgnoreCase); + + foreach (string listName in Config.NonInteractiveSpecificLists) + { + if (!lists.TryGetValue(listName, out long listId)) + continue; + + Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", listName, Config.IncludeRestrictedSubscriptions); + AnsiConsole.Markup($"[green]Getting Users from list '{listName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); + Dictionary list = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config); + + foreach ((string username, long id) in list) + usersFromLists.TryAdd(username, id); + } + + users = usersFromLists; + hasSelectedUsersKVP = new KeyValuePair>(true, users); + } + else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificUsers is not null && Config.NonInteractiveSpecificUsers.Length > 0) + { + HashSet usernames = [.. Config.NonInteractiveSpecificUsers]; + users = users.Where(u => usernames.Contains(u.Key)).ToDictionary(u => u.Key, u => u.Value); + hasSelectedUsersKVP = new KeyValuePair>(true, users); + } + else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName)) + { + hasSelectedUsersKVP = new KeyValuePair>(true, users); + } + else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName)) + { + var listId = lists[Config.NonInteractiveModeListName]; + Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", Config.NonInteractiveModeListName, Config.IncludeRestrictedSubscriptions); + AnsiConsole.Markup($"[green]Getting Users from list '{Config.NonInteractiveModeListName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); + users = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config); + hasSelectedUsersKVP = new KeyValuePair>(true, users); + } + + if (users.Count <= 0) + throw new InvalidOperationException("No users found!"); + + await dBHelper.CreateUsersDB(users); + + if (hasSelectedUsersKVP.Key == false) { var userSelectionResult = await HandleUserSelection(m_ApiHelper, Config, users, lists); @@ -1090,7 +1324,7 @@ public class Program Log.Debug($"Folder for {user.Key} already created"); } - Entities.User user_info = await m_ApiHelper.GetUserInfo($"/users/{user.Key}"); + Entities.User user_info = await m_ApiHelper.GetUserInfo(user.Key, $"/users/{user.Key}"); await dBHelper.CreateDB(path); } @@ -1108,9 +1342,13 @@ public class Program Log.Debug($"Download path: {p}"); List purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid/all", p, Config, users); - foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections) + int userNum = 1; + int userCount = purchasedTabCollections.Count; + foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections) { - AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]"); + Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", purchasedTabCollection.Username, ++userNum, userCount); + AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum} of {userCount})\n[/]"); + string path = ""; if (!string.IsNullOrEmpty(Config.DownloadPath)) { @@ -1219,8 +1457,21 @@ public class Program } else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged")) { - //Iterate over each user in the list of users - foreach (KeyValuePair user in hasSelectedUsersKVP.Value) + int totalNewPaidPostCount = 0; + int totalNewPostCount = 0; + int totalNewArchivedCount = 0; + int totalNewStreamsCount = 0; + int totalNewStoriesCount = 0; + int totalNewHighlightsCount = 0; + int totalNewMessagesCount = 0; + int totalNewPaidMessagesCount = 0; + + //Iterate over each user in the list of users + int userNum = 0; + int userCount = hasSelectedUsersKVP.Value.Count; + + LoggerWithConfigContext(config).Information("Scraping Data for {UserCount} user(s)", userCount); + foreach (KeyValuePair user in hasSelectedUsersKVP.Value) { int paidPostCount = 0; int postCount = 0; @@ -1230,9 +1481,20 @@ public class Program int highlightsCount = 0; int messagesCount = 0; int paidMessagesCount = 0; - AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]"); - Log.Debug($"Scraping Data for {user.Key}"); + int newPaidPostCount = 0; + int newPostCount = 0; + int newArchivedCount = 0; + int newStreamsCount = 0; + int newStoriesCount = 0; + int newHighlightsCount = 0; + int newMessagesCount = 0; + int newPaidMessagesCount = 0; + + DateTime userStartTime = DateTime.Now; + + Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", user.Key, ++userNum, userCount); + AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum} of {userCount})\n[/]"); string path = ""; if (!string.IsNullOrEmpty(Config.DownloadPath)) @@ -1264,71 +1526,102 @@ public class Program var downloadContext = new DownloadContext(Auth, Config, GetCreatorFileNameFormatConfig(Config, user.Key), m_ApiHelper, dBHelper); - if (Config.DownloadAvatarHeaderPhoto) + User? user_info = await m_ApiHelper.GetUserInfo(user.Key, $"/users/{user.Key}"); + + if (Config.DownloadAvatarHeaderPhoto && user_info != null) { - Entities.User? user_info = await m_ApiHelper.GetUserInfo($"/users/{user.Key}"); - if (user_info != null) - { - await downloadContext.DownloadHelper.DownloadAvatarHeader(user_info.avatar, user_info.header, path, user.Key); - } + await downloadContext.DownloadHelper.DownloadAvatarHeader(user_info.avatar, user_info.header, path, user.Key); } if (Config.DownloadPaidPosts) { - paidPostCount = await DownloadPaidPosts(downloadContext, hasSelectedUsersKVP, user, paidPostCount, path); + (paidPostCount, newPaidPostCount) = await DownloadPaidPosts(downloadContext, hasSelectedUsersKVP, user, paidPostCount, path); + totalNewPaidPostCount += newPaidPostCount; } if (Config.DownloadPosts) { - postCount = await DownloadFreePosts(downloadContext, hasSelectedUsersKVP, user, postCount, path); - } + (postCount, newPostCount) = await DownloadFreePosts(downloadContext, hasSelectedUsersKVP, user, postCount, path); + totalNewPostCount += newPostCount; + } - if (Config.DownloadArchived) + if (Config.DownloadArchived) { - archivedCount = await DownloadArchived(downloadContext, hasSelectedUsersKVP, user, archivedCount, path); - } + (archivedCount, newArchivedCount) = await DownloadArchived(downloadContext, hasSelectedUsersKVP, user, archivedCount, path); + totalNewArchivedCount += newArchivedCount; + } - if (Config.DownloadStreams) + if (Config.DownloadStreams) { - streamsCount = await DownloadStreams(downloadContext, hasSelectedUsersKVP, user, streamsCount, path); - } + (streamsCount, newStreamsCount) = await DownloadStreams(downloadContext, hasSelectedUsersKVP, user, streamsCount, path); + totalNewStreamsCount += newStreamsCount; + } - if (Config.DownloadStories) + if (Config.DownloadStories) { - storiesCount = await DownloadStories(downloadContext, user, storiesCount, path); - } + (storiesCount, newStoriesCount) = await DownloadStories(downloadContext, user, storiesCount, path); + totalNewStoriesCount += newStoriesCount; + } - if (Config.DownloadHighlights) + if (Config.DownloadHighlights) { - highlightsCount = await DownloadHighlights(downloadContext, user, highlightsCount, path); - } + (highlightsCount, newHighlightsCount) = await DownloadHighlights(downloadContext, user, highlightsCount, path); + totalNewHighlightsCount += newHighlightsCount; + } - if (Config.DownloadMessages) + if (Config.DownloadMessages) { - messagesCount = await DownloadMessages(downloadContext, hasSelectedUsersKVP, user, messagesCount, path); - } + (messagesCount, newMessagesCount) = await DownloadMessages(downloadContext, hasSelectedUsersKVP, user, messagesCount, path); + totalNewMessagesCount += newMessagesCount; + } - if (Config.DownloadPaidMessages) + if (Config.DownloadPaidMessages) { - paidMessagesCount = await DownloadPaidMessages(downloadContext, hasSelectedUsersKVP, user, paidMessagesCount, path); - } + (paidMessagesCount, newPaidMessagesCount) = await DownloadPaidMessages(downloadContext, hasSelectedUsersKVP, user, paidMessagesCount, path); + totalNewPaidMessagesCount += newPaidMessagesCount; + } - AnsiConsole.Markup("\n"); + AnsiConsole.Markup("\n"); AnsiConsole.Write(new BreakdownChart() - .FullSize() - .AddItem("Paid Posts", paidPostCount, Color.Red) - .AddItem("Posts", postCount, Color.Blue) - .AddItem("Archived", archivedCount, Color.Green) - .AddItem("Streams", streamsCount, Color.Purple) - .AddItem("Stories", storiesCount, Color.Yellow) - .AddItem("Highlights", highlightsCount, Color.Orange1) - .AddItem("Messages", messagesCount, Color.LightGreen) - .AddItem("Paid Messages", paidMessagesCount, Color.Aqua)); + .FullSize() + .AddItem("Paid Posts", paidPostCount, Color.Red) + .AddItem("Posts", postCount, Color.Blue) + .AddItem("Archived", archivedCount, Color.Green) + .AddItem("Streams", streamsCount, Color.Purple) + .AddItem("Stories", storiesCount, Color.Yellow) + .AddItem("Highlights", highlightsCount, Color.Orange1) + .AddItem("Messages", messagesCount, Color.LightGreen) + .AddItem("Paid Messages", paidMessagesCount, Color.Aqua)); AnsiConsole.Markup("\n"); - } + + DateTime userEndTime = DateTime.Now; + TimeSpan userTotalTime = userEndTime - userStartTime; + + Log.ForContext("Paid Posts", newPaidPostCount) + .ForContext("Posts", newPostCount) + .ForContext("Archived", newArchivedCount) + .ForContext("Streams", newStreamsCount) + .ForContext("Stories", newStoriesCount) + .ForContext("Highlights", newHighlightsCount) + .ForContext("Messages", newMessagesCount) + .ForContext("Paid Messages", newPaidMessagesCount) + .Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes", user.Key, userTotalTime.TotalMinutes); + } + DateTime endTime = DateTime.Now; TimeSpan totalTime = endTime - startTime; - AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]"); + Log.ForContext("Paid Posts", totalNewPaidPostCount) + .ForContext("Posts", totalNewPostCount) + .ForContext("Archived", totalNewArchivedCount) + .ForContext("Streams", totalNewStreamsCount) + .ForContext("Stories", totalNewStoriesCount) + .ForContext("Highlights", totalNewHighlightsCount) + .ForContext("Messages", totalNewMessagesCount) + .ForContext("Paid Messages", totalNewPaidMessagesCount) + .Information("Scrape Completed in {TotalMinutes:0.00} minutes", totalTime.TotalMinutes); + AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]"); + + await Task.Delay(2000); } else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged")) { @@ -1378,7 +1671,7 @@ public class Program return combinedConfig; } - private static async Task DownloadPaidMessages(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int paidMessagesCount, string path) + private static async Task<(int, int)> DownloadPaidMessages(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int paidMessagesCount, string path) { Log.Debug($"Calling DownloadPaidMessages - {user.Key}"); @@ -1387,7 +1680,7 @@ public class Program await AnsiConsole.Status() .StartAsync("[red]Getting Paid Messages[/]", async ctx => { - paidMessageCollection = await downloadContext.ApiHelper.GetPaidMessages("/posts/paid/chat", path, user.Key, downloadContext.DownloadConfig!, ctx); + paidMessageCollection = await downloadContext.ApiHelper.GetPaidMessages("/posts/paid/chat", path, user.Key, user.Value, downloadContext.DownloadConfig!, ctx); }); int oldPaidMessagesCount = 0; int newPaidMessagesCount = 0; @@ -1508,13 +1801,16 @@ public class Program AnsiConsole.Markup($"[red]Found 0 Paid Messages\n[/]"); } - return paidMessagesCount; + return (paidMessagesCount, newPaidMessagesCount); } - private static async Task DownloadMessages(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int messagesCount, string path) + private static async Task<(int, int)> DownloadMessages(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int messagesCount, string path) { Log.Debug($"Calling DownloadMessages - {user.Key}"); + AnsiConsole.Markup($"[grey]Getting Unread Chats\n[/]"); + HashSet unreadChats = await GetUsersWithUnreadChats(downloadContext.ApiHelper, downloadContext.DownloadConfig); + MessageCollection messages = new MessageCollection(); await AnsiConsole.Status() @@ -1522,7 +1818,14 @@ public class Program { messages = await downloadContext.ApiHelper.GetMessages($"/chats/{user.Value}/messages", path, downloadContext.DownloadConfig!, ctx); }); - int oldMessagesCount = 0; + + if (unreadChats.Contains(user.Value)) + { + AnsiConsole.Markup($"[grey]Restoring unread state\n[/]"); + await downloadContext.ApiHelper.MarkAsUnread($"/chats/{user.Value}/mark-as-read", downloadContext.DownloadConfig); + } + + int oldMessagesCount = 0; int newMessagesCount = 0; if (messages != null && messages.Messages.Count > 0) { @@ -1641,10 +1944,10 @@ public class Program AnsiConsole.Markup($"[red]Found 0 Messages\n[/]"); } - return messagesCount; + return (messagesCount, newMessagesCount); } - private static async Task DownloadHighlights(IDownloadContext downloadContext, KeyValuePair user, int highlightsCount, string path) + private static async Task<(int, int)> DownloadHighlights(IDownloadContext downloadContext, KeyValuePair user, int highlightsCount, string path) { Log.Debug($"Calling DownloadHighlights - {user.Key}"); @@ -1698,10 +2001,10 @@ public class Program Log.Debug($"Found 0 Highlights"); } - return highlightsCount; + return (highlightsCount, newHighlightsCount); } - private static async Task DownloadStories(IDownloadContext downloadContext, KeyValuePair user, int storiesCount, string path) + private static async Task<(int, int)> DownloadStories(IDownloadContext downloadContext, KeyValuePair user, int storiesCount, string path) { Log.Debug($"Calling DownloadStories - {user.Key}"); @@ -1755,10 +2058,10 @@ public class Program Log.Debug($"Found 0 Stories"); } - return storiesCount; + return (storiesCount, newStoriesCount); } - private static async Task DownloadArchived(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int archivedCount, string path) + private static async Task<(int,int)> DownloadArchived(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int archivedCount, string path) { Log.Debug($"Calling DownloadArchived - {user.Key}"); @@ -1888,10 +2191,10 @@ public class Program AnsiConsole.Markup($"[red]Found 0 Archived Posts\n[/]"); } - return archivedCount; + return (archivedCount, newArchivedCount); } - private static async Task DownloadFreePosts(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int postCount, string path) + private static async Task<(int,int)> DownloadFreePosts(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int postCount, string path) { Log.Debug($"Calling DownloadFreePosts - {user.Key}"); @@ -1909,7 +2212,7 @@ public class Program { AnsiConsole.Markup($"[red]Found 0 Posts\n[/]"); Log.Debug($"Found 0 Posts"); - return 0; + return (0,0); } AnsiConsole.Markup($"[red]Found {posts.Posts.Count} Media from {posts.PostObjects.Count} Posts\n[/]"); @@ -2028,10 +2331,10 @@ public class Program AnsiConsole.Markup($"[red]Posts Already Downloaded: {oldPostCount} New Posts Downloaded: {newPostCount}[/]\n"); Log.Debug("Posts Already Downloaded: {oldPostCount} New Posts Downloaded: {newPostCount}"); - return postCount; + return (postCount, newPostCount); } - private static async Task DownloadPaidPosts(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int paidPostCount, string path) + private static async Task<(int,int)> DownloadPaidPosts(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int paidPostCount, string path) { Log.Debug($"Calling DownloadPaidPosts - {user.Key}"); @@ -2040,7 +2343,7 @@ public class Program await AnsiConsole.Status() .StartAsync("[red]Getting Paid Posts[/]", async ctx => { - purchasedPosts = await downloadContext.ApiHelper.GetPaidPosts("/posts/paid/post", path, user.Key, downloadContext.DownloadConfig!, paid_post_ids, ctx); + purchasedPosts = await downloadContext.ApiHelper.GetPaidPosts("/posts/paid/post", path, user.Key, user.Value, downloadContext.DownloadConfig!, paid_post_ids, ctx); }); int oldPaidPostCount = 0; @@ -2049,7 +2352,7 @@ public class Program { AnsiConsole.Markup($"[red]Found 0 Paid Posts\n[/]"); Log.Debug("Found 0 Paid Posts"); - return 0; + return (0,0); } AnsiConsole.Markup($"[red]Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts\n[/]"); @@ -2161,7 +2464,7 @@ public class Program }); AnsiConsole.Markup($"[red]Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}[/]\n"); Log.Debug($"Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}"); - return paidPostCount; + return (paidPostCount, newPaidPostCount); } private static async Task DownloadPaidPostsPurchasedTab(IDownloadContext downloadContext, PaidPostCollection purchasedPosts, KeyValuePair user, int paidPostCount, string path, Dictionary users) @@ -2413,7 +2716,7 @@ public class Program return paidMessagesCount; } - private static async Task DownloadStreams(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int streamsCount, string path) + private static async Task<(int, int)> DownloadStreams(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int streamsCount, string path) { Log.Debug($"Calling DownloadStreams - {user.Key}"); @@ -2431,7 +2734,7 @@ public class Program { AnsiConsole.Markup($"[red]Found 0 Streams\n[/]"); Log.Debug($"Found 0 Streams"); - return 0; + return (0,0); } AnsiConsole.Markup($"[red]Found {streams.Streams.Count} Media from {streams.StreamObjects.Count} Streams\n[/]"); @@ -2549,7 +2852,7 @@ public class Program }); AnsiConsole.Markup($"[red]Streams Already Downloaded: {oldStreamsCount} New Streams Downloaded: {newStreamsCount}[/]\n"); Log.Debug($"Streams Already Downloaded: {oldStreamsCount} New Streams Downloaded: {newStreamsCount}"); - return streamsCount; + return (streamsCount, newStreamsCount); } private static async Task DownloadPaidMessage(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, string username, int paidMessagesCount, string path, long message_id) @@ -3164,7 +3467,7 @@ public class Program hoconConfig.AppendLine($" LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\""); hoconConfig.AppendLine("}"); - File.WriteAllText("config.conf", hoconConfig.ToString()); + await File.WriteAllTextAsync("config.conf", hoconConfig.ToString()); string newConfigString = JsonConvert.SerializeObject(newConfig, Formatting.Indented); @@ -3391,7 +3694,18 @@ public class Program } } - static bool ValidateFilePath(string path) + private static async Task> GetUsersWithUnreadChats(APIHelper apiHelper, IDownloadConfig currentConfig) + { + ChatCollection chats = await apiHelper.GetChats($"/chats", currentConfig, onlyUnread: true); + + var unreadChats = chats.Chats + .Where(c => c.Value.unreadMessagesCount > 0) + .ToList(); + + return [.. unreadChats.Select(c => c.Key)]; + } + + static bool ValidateFilePath(string path) { char[] invalidChars = System.IO.Path.GetInvalidPathChars(); char[] foundInvalidChars = path.Where(c => invalidChars.Contains(c)).ToArray(); @@ -3483,7 +3797,7 @@ public class Program AnsiConsole.Markup($"[red]{settingName} is not unique enough, please make sure you include either '{{mediaId}}' or '{{filename}}' to ensure that files are not overwritten with the same filename.[/]\n"); AnsiConsole.Markup("[red]Press any key to continue.[/]\n"); Console.ReadKey(); - Environment.Exit(2); + ExitWithCode(2); } } @@ -3494,4 +3808,61 @@ public class Program return Enum.Parse("_" + value, ignoreCase: true); } + + static void ExitIfOtherProcess() + { + Assembly entryAssembly = Assembly.GetEntryAssembly(); + AssemblyName entryAssemblyName = entryAssembly?.GetName(); + + if (entryAssemblyName?.Name is null) + return; + + Process thisProcess = Process.GetCurrentProcess(); + Process[] otherProcesses = [.. Process.GetProcessesByName(entryAssemblyName.Name).Where(p => p.Id != thisProcess.Id)]; + + if (otherProcesses.Length <= 0) + return; + + AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]"); + Log.Warning("Other OF DL process detected, exiting.."); + + ExitWithCode(0); + } + + static void ExitWithCode(int exitCode) + { + Console.WriteLine(); + + (string colorStart, string colorEnd) = exitCode != 0 ? ("[red]", "[]") : ("", ""); + AnsiConsole.Markup($"{colorStart}Exiting run with Code '{exitCode}'..{colorEnd}\n"); + + Log.CloseAndFlush(); + Log.CloseAndFlush(); + Task.Delay(5000).GetAwaiter().GetResult(); + + Environment.Exit(exitCode); + } + + static ILogger LoggerWithConfigContext(Entities.Config config) + => Log.Logger.ForContext(nameof(Entities.Config.DownloadPath), config.DownloadPath) + .ForContext(nameof(Entities.Config.DownloadPosts), config.DownloadPosts) + .ForContext(nameof(Entities.Config.DownloadPaidPosts), config.DownloadPaidPosts) + .ForContext(nameof(Entities.Config.DownloadMessages), config.DownloadMessages) + .ForContext(nameof(Entities.Config.DownloadPaidMessages), config.DownloadPaidMessages) + .ForContext(nameof(Entities.Config.DownloadStories), config.DownloadStories) + .ForContext(nameof(Entities.Config.DownloadStreams), config.DownloadStreams) + .ForContext(nameof(Entities.Config.DownloadHighlights), config.DownloadHighlights) + .ForContext(nameof(Entities.Config.DownloadArchived), config.DownloadArchived) + .ForContext(nameof(Entities.Config.DownloadAvatarHeaderPhoto), config.DownloadAvatarHeaderPhoto) + .ForContext(nameof(Entities.Config.DownloadImages), config.DownloadImages) + .ForContext(nameof(Entities.Config.DownloadVideos), config.DownloadVideos) + .ForContext(nameof(Entities.Config.DownloadAudios), config.DownloadAudios) + .ForContext(nameof(Entities.Config.IgnoreOwnMessages), config.IgnoreOwnMessages) + .ForContext(nameof(Entities.Config.DownloadPostsIncrementally), config.DownloadPostsIncrementally) + .ForContext(nameof(Entities.Config.BypassContentForCreatorsWhoNoLongerExist), config.BypassContentForCreatorsWhoNoLongerExist) + .ForContext(nameof(Entities.Config.SkipAds), config.SkipAds) + .ForContext(nameof(Entities.Config.IncludeExpiredSubscriptions), config.IncludeExpiredSubscriptions) + .ForContext(nameof(Entities.Config.IncludeRestrictedSubscriptions), config.IncludeRestrictedSubscriptions) + .ForContext(nameof(Entities.Config.NonInteractiveSpecificLists), config.NonInteractiveSpecificLists) + .ForContext(nameof(Entities.Config.NonInteractiveSpecificUsers), config.NonInteractiveSpecificUsers); } diff --git a/Publish_OF-DL.bat b/Publish_OF-DL.bat new file mode 100644 index 0000000..c421cd0 --- /dev/null +++ b/Publish_OF-DL.bat @@ -0,0 +1,33 @@ +@ECHO OFF + +ECHO. + +ECHO ============================== +ECHO == Cleaning Output =========== +ECHO ============================== +dotnet clean ".\OF DL\OF DL.csproj" -v minimal +DEL /Q /F ".\Publish" + +ECHO. + +ECHO ============================== +ECHO == Publishing OF-DL ========== +ECHO ============================== +dotnet publish ".\OF DL\OF DL.csproj" -o ".\Publish" -c Debug + +ECHO. + +ECHO ============================== +ECHO == Copy to network drive? ==== +ECHO ============================== +CHOICE /C yn /m "Copy published files to network drive? " + +IF %ERRORLEVEL%==1 (GOTO Copy) ELSE (GOTO Exit) + +:Copy +xcopy .\Publish\* p:\_Utils\OF_DL /I /Y /Q /EXCLUDE:.\excludes.txt + +:Exit +ECHO. +ECHO. +PAUSE \ No newline at end of file diff --git a/excludes.txt b/excludes.txt new file mode 100644 index 0000000..297e244 --- /dev/null +++ b/excludes.txt @@ -0,0 +1,2 @@ +excludes.txt +rules.json \ No newline at end of file