From a2d57ba2f594ee96c1523333baa653e5cb3c66c3 Mon Sep 17 00:00:00 2001 From: Casper Sparre Date: Wed, 8 Oct 2025 22:07:37 +0200 Subject: [PATCH] Added additional info logging with new media counts per model --- OF DL/Program.cs | 132 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 96 insertions(+), 36 deletions(-) diff --git a/OF DL/Program.cs b/OF DL/Program.cs index d9db2b9..28936c3 100644 --- a/OF DL/Program.cs +++ b/OF DL/Program.cs @@ -112,7 +112,8 @@ public class Program Log.Logger = new LoggerConfiguration() .Enrich.WithProperty("Application", "OF_DL") - .Enrich.WithProperty("StartTime", DateTime.Now) + .Enrich.WithProperty("StartTime", $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} ") + .Enrich.WithProperty("MachineName", Environment.MachineName) .MinimumLevel.ControlledBy(levelSwitch) .WriteTo.File("logs/OFDL.txt", rollingInterval: RollingInterval.Day) .WriteTo.Seq("https://seq.cajetan.dk") @@ -861,7 +862,8 @@ public class Program } } - AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]"); + Log.Information("Logged In successfully as {Name:l} ({Username:l})", validate.name, validate.username); + AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} ({validate.username})\n[/]"); try { @@ -942,6 +944,7 @@ public class Program DateTime startTime = DateTime.Now; Dictionary users = new(); + Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", Config.IncludeRestrictedSubscriptions); AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); Dictionary subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? []; @@ -959,6 +962,7 @@ public class Program { Log.Debug("Inactive Subscriptions: "); + Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", Config.IncludeRestrictedSubscriptions); AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); Dictionary subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? []; @@ -1003,6 +1007,7 @@ public class Program if (!lists.TryGetValue(listName, out int listId)) continue; + Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", listName, Config.IncludeRestrictedSubscriptions); AnsiConsole.Markup($"[green]Getting Users from list '{listName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); Dictionary list = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config); @@ -1026,6 +1031,7 @@ public class Program else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName)) { var listId = lists[Config.NonInteractiveModeListName]; + Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", Config.NonInteractiveModeListName, Config.IncludeRestrictedSubscriptions); AnsiConsole.Markup($"[green]Getting Users from list '{Config.NonInteractiveModeListName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); users = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config); hasSelectedUsersKVP = new KeyValuePair>(true, users); @@ -1155,11 +1161,13 @@ public class Program Log.Debug($"Download path: {p}"); List purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users); - int userNum = 1; + int userNum = 0; int userCount = purchasedTabCollections.Count; foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections) { - AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]"); + Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", purchasedTabCollection.Username, ++userNum, userCount); + AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum} of {userCount})\n[/]"); + string path = ""; if (!string.IsNullOrEmpty(Config.DownloadPath)) { @@ -1269,8 +1277,10 @@ public class Program else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged")) { //Iterate over each user in the list of users - int userNum = 1; + int userNum = 0; int userCount = hasSelectedUsersKVP.Value.Count; + + LoggerWithConfigContext(config).Information("Scraping Data for {UserCount} user(s)", userCount); foreach (KeyValuePair user in hasSelectedUsersKVP.Value) { int paidPostCount = 0; @@ -1281,9 +1291,20 @@ public class Program int highlightsCount = 0; int messagesCount = 0; int paidMessagesCount = 0; - AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]"); - Log.Debug($"Scraping Data for {user.Key}"); + int newPaidPostCount = 0; + int newPostCount = 0; + int newArchivedCount = 0; + int newStreamsCount = 0; + int newStoriesCount = 0; + int newHighlightsCount = 0; + int newMessagesCount = 0; + int newPaidMessagesCount = 0; + + DateTime userStartTime = DateTime.Now; + + Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", user.Key, ++userNum, userCount); + AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum} of {userCount})\n[/]"); string path = ""; if (!string.IsNullOrEmpty(Config.DownloadPath)) @@ -1326,42 +1347,42 @@ public class Program if (Config.DownloadPaidPosts) { - paidPostCount = await DownloadPaidPosts(downloadContext, hasSelectedUsersKVP, user, paidPostCount, path); + (paidPostCount, newPaidPostCount) = await DownloadPaidPosts(downloadContext, hasSelectedUsersKVP, user, paidPostCount, path); } if (Config.DownloadPosts) { - postCount = await DownloadFreePosts(downloadContext, hasSelectedUsersKVP, user, postCount, path); + (postCount, newPostCount) = await DownloadFreePosts(downloadContext, hasSelectedUsersKVP, user, postCount, path); } if (Config.DownloadArchived) { - archivedCount = await DownloadArchived(downloadContext, hasSelectedUsersKVP, user, archivedCount, path); + (archivedCount, newArchivedCount) = await DownloadArchived(downloadContext, hasSelectedUsersKVP, user, archivedCount, path); } if (Config.DownloadStreams) { - streamsCount = await DownloadStreams(downloadContext, hasSelectedUsersKVP, user, streamsCount, path); + (streamsCount, newStreamsCount) = await DownloadStreams(downloadContext, hasSelectedUsersKVP, user, streamsCount, path); } if (Config.DownloadStories) { - storiesCount = await DownloadStories(downloadContext, user, storiesCount, path); + (storiesCount, newStoriesCount) = await DownloadStories(downloadContext, user, storiesCount, path); } if (Config.DownloadHighlights) { - highlightsCount = await DownloadHighlights(downloadContext, user, highlightsCount, path); + (highlightsCount, newHighlightsCount) = await DownloadHighlights(downloadContext, user, highlightsCount, path); } if (Config.DownloadMessages) { - messagesCount = await DownloadMessages(downloadContext, hasSelectedUsersKVP, user, messagesCount, path); + (messagesCount, newMessagesCount) = await DownloadMessages(downloadContext, hasSelectedUsersKVP, user, messagesCount, path); } if (Config.DownloadPaidMessages) { - paidMessagesCount = await DownloadPaidMessages(downloadContext, hasSelectedUsersKVP, user, paidMessagesCount, path); + (paidMessagesCount, newPaidMessagesCount) = await DownloadPaidMessages(downloadContext, hasSelectedUsersKVP, user, paidMessagesCount, path); } AnsiConsole.Markup("\n"); @@ -1376,10 +1397,25 @@ public class Program .AddItem("Messages", messagesCount, Color.LightGreen) .AddItem("Paid Messages", paidMessagesCount, Color.Aqua)); AnsiConsole.Markup("\n"); - } + + DateTime userEndTime = DateTime.Now; + TimeSpan userTotalTime = userEndTime - userStartTime; + + Log.ForContext("Paid Posts", newPaidPostCount) + .ForContext("Posts", newPostCount) + .ForContext("Archived", newArchivedCount) + .ForContext("Streams", newStreamsCount) + .ForContext("Stories", newStoriesCount) + .ForContext("Highlights", newHighlightsCount) + .ForContext("Messages", newMessagesCount) + .ForContext("Paid Messages", newPaidMessagesCount) + .Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes", user.Key, userTotalTime.TotalMinutes); + } + DateTime endTime = DateTime.Now; TimeSpan totalTime = endTime - startTime; - AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]"); + Log.Information("Scrape Completed in {TotalMinutes:0.00} minutes", totalTime.TotalMinutes); + AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]"); } else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged")) { @@ -1429,7 +1465,7 @@ public class Program return combinedConfig; } - private static async Task DownloadPaidMessages(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int paidMessagesCount, string path) + private static async Task<(int, int)> DownloadPaidMessages(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int paidMessagesCount, string path) { Log.Debug($"Calling DownloadPaidMessages - {user.Key}"); @@ -1559,10 +1595,10 @@ public class Program AnsiConsole.Markup($"[red]Found 0 Paid Messages\n[/]"); } - return paidMessagesCount; + return (paidMessagesCount, newPaidMessagesCount); } - private static async Task DownloadMessages(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int messagesCount, string path) + private static async Task<(int, int)> DownloadMessages(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int messagesCount, string path) { Log.Debug($"Calling DownloadMessages - {user.Key}"); @@ -1702,10 +1738,10 @@ public class Program AnsiConsole.Markup($"[red]Found 0 Messages\n[/]"); } - return messagesCount; + return (messagesCount, newMessagesCount); } - private static async Task DownloadHighlights(IDownloadContext downloadContext, KeyValuePair user, int highlightsCount, string path) + private static async Task<(int, int)> DownloadHighlights(IDownloadContext downloadContext, KeyValuePair user, int highlightsCount, string path) { Log.Debug($"Calling DownloadHighlights - {user.Key}"); @@ -1759,10 +1795,10 @@ public class Program Log.Debug($"Found 0 Highlights"); } - return highlightsCount; + return (highlightsCount, newHighlightsCount); } - private static async Task DownloadStories(IDownloadContext downloadContext, KeyValuePair user, int storiesCount, string path) + private static async Task<(int, int)> DownloadStories(IDownloadContext downloadContext, KeyValuePair user, int storiesCount, string path) { Log.Debug($"Calling DownloadStories - {user.Key}"); @@ -1816,10 +1852,10 @@ public class Program Log.Debug($"Found 0 Stories"); } - return storiesCount; + return (storiesCount, newStoriesCount); } - private static async Task DownloadArchived(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int archivedCount, string path) + private static async Task<(int,int)> DownloadArchived(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int archivedCount, string path) { Log.Debug($"Calling DownloadArchived - {user.Key}"); @@ -1949,10 +1985,10 @@ public class Program AnsiConsole.Markup($"[red]Found 0 Archived Posts\n[/]"); } - return archivedCount; + return (archivedCount, newArchivedCount); } - private static async Task DownloadFreePosts(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int postCount, string path) + private static async Task<(int,int)> DownloadFreePosts(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int postCount, string path) { Log.Debug($"Calling DownloadFreePosts - {user.Key}"); @@ -1970,7 +2006,7 @@ public class Program { AnsiConsole.Markup($"[red]Found 0 Posts\n[/]"); Log.Debug($"Found 0 Posts"); - return 0; + return (0,0); } AnsiConsole.Markup($"[red]Found {posts.Posts.Count} Media from {posts.PostObjects.Count} Posts\n[/]"); @@ -2089,10 +2125,10 @@ public class Program AnsiConsole.Markup($"[red]Posts Already Downloaded: {oldPostCount} New Posts Downloaded: {newPostCount}[/]\n"); Log.Debug("Posts Already Downloaded: {oldPostCount} New Posts Downloaded: {newPostCount}"); - return postCount; + return (postCount, newPostCount); } - private static async Task DownloadPaidPosts(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int paidPostCount, string path) + private static async Task<(int,int)> DownloadPaidPosts(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int paidPostCount, string path) { Log.Debug($"Calling DownloadPaidPosts - {user.Key}"); @@ -2110,7 +2146,7 @@ public class Program { AnsiConsole.Markup($"[red]Found 0 Paid Posts\n[/]"); Log.Debug("Found 0 Paid Posts"); - return 0; + return (0,0); } AnsiConsole.Markup($"[red]Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts\n[/]"); @@ -2222,7 +2258,7 @@ public class Program }); AnsiConsole.Markup($"[red]Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}[/]\n"); Log.Debug($"Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}"); - return paidPostCount; + return (paidPostCount, newPaidPostCount); } private static async Task DownloadPaidPostsPurchasedTab(IDownloadContext downloadContext, PaidPostCollection purchasedPosts, KeyValuePair user, int paidPostCount, string path, Dictionary users) @@ -2474,7 +2510,7 @@ public class Program return paidMessagesCount; } - private static async Task DownloadStreams(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int streamsCount, string path) + private static async Task<(int, int)> DownloadStreams(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, int streamsCount, string path) { Log.Debug($"Calling DownloadStreams - {user.Key}"); @@ -2492,7 +2528,7 @@ public class Program { AnsiConsole.Markup($"[red]Found 0 Streams\n[/]"); Log.Debug($"Found 0 Streams"); - return 0; + return (0,0); } AnsiConsole.Markup($"[red]Found {streams.Streams.Count} Media from {streams.StreamObjects.Count} Streams\n[/]"); @@ -2610,7 +2646,7 @@ public class Program }); AnsiConsole.Markup($"[red]Streams Already Downloaded: {oldStreamsCount} New Streams Downloaded: {newStreamsCount}[/]\n"); Log.Debug($"Streams Already Downloaded: {oldStreamsCount} New Streams Downloaded: {newStreamsCount}"); - return streamsCount; + return (streamsCount, newStreamsCount); } private static async Task DownloadPaidMessage(IDownloadContext downloadContext, KeyValuePair> hasSelectedUsersKVP, string username, int paidMessagesCount, string path, long message_id) @@ -3466,6 +3502,30 @@ public class Program AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]"); Log.Warning("Other OF DL process detected, exiting.."); + Log.CloseAndFlush(); Environment.Exit(0); } + + static ILogger LoggerWithConfigContext(Entities.Config config) + => Log.Logger.ForContext(nameof(Entities.Config.DownloadPath), config.DownloadPath) + .ForContext(nameof(Entities.Config.DownloadPosts), config.DownloadPosts) + .ForContext(nameof(Entities.Config.DownloadPaidPosts), config.DownloadPaidPosts) + .ForContext(nameof(Entities.Config.DownloadMessages), config.DownloadMessages) + .ForContext(nameof(Entities.Config.DownloadPaidMessages), config.DownloadPaidMessages) + .ForContext(nameof(Entities.Config.DownloadStories), config.DownloadStories) + .ForContext(nameof(Entities.Config.DownloadStreams), config.DownloadStreams) + .ForContext(nameof(Entities.Config.DownloadHighlights), config.DownloadHighlights) + .ForContext(nameof(Entities.Config.DownloadArchived), config.DownloadArchived) + .ForContext(nameof(Entities.Config.DownloadAvatarHeaderPhoto), config.DownloadAvatarHeaderPhoto) + .ForContext(nameof(Entities.Config.DownloadImages), config.DownloadImages) + .ForContext(nameof(Entities.Config.DownloadVideos), config.DownloadVideos) + .ForContext(nameof(Entities.Config.DownloadAudios), config.DownloadAudios) + .ForContext(nameof(Entities.Config.IgnoreOwnMessages), config.IgnoreOwnMessages) + .ForContext(nameof(Entities.Config.DownloadPostsIncrementally), config.DownloadPostsIncrementally) + .ForContext(nameof(Entities.Config.BypassContentForCreatorsWhoNoLongerExist), config.BypassContentForCreatorsWhoNoLongerExist) + .ForContext(nameof(Entities.Config.SkipAds), config.SkipAds) + .ForContext(nameof(Entities.Config.IncludeExpiredSubscriptions), config.IncludeExpiredSubscriptions) + .ForContext(nameof(Entities.Config.IncludeRestrictedSubscriptions), config.IncludeRestrictedSubscriptions) + .ForContext(nameof(Entities.Config.NonInteractiveSpecificLists), config.NonInteractiveSpecificLists) + .ForContext(nameof(Entities.Config.NonInteractiveSpecificUsers), config.NonInteractiveSpecificUsers); }