forked from sim0n00ps/OF-DL
Added additional info logging with new media counts per model
This commit is contained in:
parent
a70fa0aa2f
commit
7bf6271102
130
OF DL/Program.cs
130
OF DL/Program.cs
@ -112,7 +112,8 @@ public class Program
|
||||
|
||||
Log.Logger = new LoggerConfiguration()
|
||||
.Enrich.WithProperty("Application", "OF_DL")
|
||||
.Enrich.WithProperty("StartTime", DateTime.Now)
|
||||
.Enrich.WithProperty("StartTime", $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} ")
|
||||
.Enrich.WithProperty("MachineName", Environment.MachineName)
|
||||
.MinimumLevel.ControlledBy(levelSwitch)
|
||||
.WriteTo.File("logs/OFDL.txt", rollingInterval: RollingInterval.Day)
|
||||
.WriteTo.Seq("https://seq.cajetan.dk")
|
||||
@ -861,7 +862,8 @@ public class Program
|
||||
}
|
||||
}
|
||||
|
||||
AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]");
|
||||
Log.Information("Logged In successfully as {Name:l} ({Username:l})", validate.name, validate.username);
|
||||
AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} ({validate.username})\n[/]");
|
||||
|
||||
try
|
||||
{
|
||||
@ -942,6 +944,7 @@ public class Program
|
||||
DateTime startTime = DateTime.Now;
|
||||
Dictionary<string, int> users = new();
|
||||
|
||||
Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", Config.IncludeRestrictedSubscriptions);
|
||||
AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
||||
Dictionary<string, int> subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
|
||||
|
||||
@ -959,6 +962,7 @@ public class Program
|
||||
{
|
||||
Log.Debug("Inactive Subscriptions: ");
|
||||
|
||||
Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", Config.IncludeRestrictedSubscriptions);
|
||||
AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
||||
Dictionary<string, int> subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
|
||||
|
||||
@ -1003,6 +1007,7 @@ public class Program
|
||||
if (!lists.TryGetValue(listName, out int listId))
|
||||
continue;
|
||||
|
||||
Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", listName, Config.IncludeRestrictedSubscriptions);
|
||||
AnsiConsole.Markup($"[green]Getting Users from list '{listName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
||||
Dictionary<string, int> list = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
|
||||
|
||||
@ -1026,6 +1031,7 @@ public class Program
|
||||
else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName))
|
||||
{
|
||||
var listId = lists[Config.NonInteractiveModeListName];
|
||||
Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", Config.NonInteractiveModeListName, Config.IncludeRestrictedSubscriptions);
|
||||
AnsiConsole.Markup($"[green]Getting Users from list '{Config.NonInteractiveModeListName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
||||
users = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
|
||||
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
|
||||
@ -1159,7 +1165,9 @@ public class Program
|
||||
int userCount = purchasedTabCollections.Count;
|
||||
foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
|
||||
{
|
||||
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]");
|
||||
Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", purchasedTabCollection.Username, ++userNum, userCount);
|
||||
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum} of {userCount})\n[/]");
|
||||
|
||||
string path = "";
|
||||
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
||||
{
|
||||
@ -1269,8 +1277,10 @@ public class Program
|
||||
else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
||||
{
|
||||
//Iterate over each user in the list of users
|
||||
int userNum = 1;
|
||||
int userNum = 0;
|
||||
int userCount = hasSelectedUsersKVP.Value.Count;
|
||||
|
||||
LoggerWithConfigContext(config).Information("Scraping Data for {UserCount} user(s)", userCount);
|
||||
foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
|
||||
{
|
||||
int paidPostCount = 0;
|
||||
@ -1281,9 +1291,20 @@ public class Program
|
||||
int highlightsCount = 0;
|
||||
int messagesCount = 0;
|
||||
int paidMessagesCount = 0;
|
||||
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]");
|
||||
|
||||
Log.Debug($"Scraping Data for {user.Key}");
|
||||
int newPaidPostCount = 0;
|
||||
int newPostCount = 0;
|
||||
int newArchivedCount = 0;
|
||||
int newStreamsCount = 0;
|
||||
int newStoriesCount = 0;
|
||||
int newHighlightsCount = 0;
|
||||
int newMessagesCount = 0;
|
||||
int newPaidMessagesCount = 0;
|
||||
|
||||
DateTime userStartTime = DateTime.Now;
|
||||
|
||||
Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", user.Key, ++userNum, userCount);
|
||||
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum} of {userCount})\n[/]");
|
||||
|
||||
string path = "";
|
||||
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
||||
@ -1326,42 +1347,42 @@ public class Program
|
||||
|
||||
if (Config.DownloadPaidPosts)
|
||||
{
|
||||
paidPostCount = await DownloadPaidPosts(downloadContext, hasSelectedUsersKVP, user, paidPostCount, path);
|
||||
(paidPostCount, newPaidPostCount) = await DownloadPaidPosts(downloadContext, hasSelectedUsersKVP, user, paidPostCount, path);
|
||||
}
|
||||
|
||||
if (Config.DownloadPosts)
|
||||
{
|
||||
postCount = await DownloadFreePosts(downloadContext, hasSelectedUsersKVP, user, postCount, path);
|
||||
(postCount, newPostCount) = await DownloadFreePosts(downloadContext, hasSelectedUsersKVP, user, postCount, path);
|
||||
}
|
||||
|
||||
if (Config.DownloadArchived)
|
||||
{
|
||||
archivedCount = await DownloadArchived(downloadContext, hasSelectedUsersKVP, user, archivedCount, path);
|
||||
(archivedCount, newArchivedCount) = await DownloadArchived(downloadContext, hasSelectedUsersKVP, user, archivedCount, path);
|
||||
}
|
||||
|
||||
if (Config.DownloadStreams)
|
||||
{
|
||||
streamsCount = await DownloadStreams(downloadContext, hasSelectedUsersKVP, user, streamsCount, path);
|
||||
(streamsCount, newStreamsCount) = await DownloadStreams(downloadContext, hasSelectedUsersKVP, user, streamsCount, path);
|
||||
}
|
||||
|
||||
if (Config.DownloadStories)
|
||||
{
|
||||
storiesCount = await DownloadStories(downloadContext, user, storiesCount, path);
|
||||
(storiesCount, newStoriesCount) = await DownloadStories(downloadContext, user, storiesCount, path);
|
||||
}
|
||||
|
||||
if (Config.DownloadHighlights)
|
||||
{
|
||||
highlightsCount = await DownloadHighlights(downloadContext, user, highlightsCount, path);
|
||||
(highlightsCount, newHighlightsCount) = await DownloadHighlights(downloadContext, user, highlightsCount, path);
|
||||
}
|
||||
|
||||
if (Config.DownloadMessages)
|
||||
{
|
||||
messagesCount = await DownloadMessages(downloadContext, hasSelectedUsersKVP, user, messagesCount, path);
|
||||
(messagesCount, newMessagesCount) = await DownloadMessages(downloadContext, hasSelectedUsersKVP, user, messagesCount, path);
|
||||
}
|
||||
|
||||
if (Config.DownloadPaidMessages)
|
||||
{
|
||||
paidMessagesCount = await DownloadPaidMessages(downloadContext, hasSelectedUsersKVP, user, paidMessagesCount, path);
|
||||
(paidMessagesCount, newPaidMessagesCount) = await DownloadPaidMessages(downloadContext, hasSelectedUsersKVP, user, paidMessagesCount, path);
|
||||
}
|
||||
|
||||
AnsiConsole.Markup("\n");
|
||||
@ -1376,10 +1397,25 @@ public class Program
|
||||
.AddItem("Messages", messagesCount, Color.LightGreen)
|
||||
.AddItem("Paid Messages", paidMessagesCount, Color.Aqua));
|
||||
AnsiConsole.Markup("\n");
|
||||
}
|
||||
|
||||
DateTime userEndTime = DateTime.Now;
|
||||
TimeSpan userTotalTime = userEndTime - userStartTime;
|
||||
|
||||
Log.ForContext("Paid Posts", newPaidPostCount)
|
||||
.ForContext("Posts", newPostCount)
|
||||
.ForContext("Archived", newArchivedCount)
|
||||
.ForContext("Streams", newStreamsCount)
|
||||
.ForContext("Stories", newStoriesCount)
|
||||
.ForContext("Highlights", newHighlightsCount)
|
||||
.ForContext("Messages", newMessagesCount)
|
||||
.ForContext("Paid Messages", newPaidMessagesCount)
|
||||
.Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes", user.Key, userTotalTime.TotalMinutes);
|
||||
}
|
||||
|
||||
DateTime endTime = DateTime.Now;
|
||||
TimeSpan totalTime = endTime - startTime;
|
||||
AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]");
|
||||
Log.Information("Scrape Completed in {TotalMinutes:0.00} minutes", totalTime.TotalMinutes);
|
||||
AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]");
|
||||
}
|
||||
else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
||||
{
|
||||
@ -1429,7 +1465,7 @@ public class Program
|
||||
return combinedConfig;
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadPaidMessages(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int paidMessagesCount, string path)
|
||||
private static async Task<(int, int)> DownloadPaidMessages(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int paidMessagesCount, string path)
|
||||
{
|
||||
Log.Debug($"Calling DownloadPaidMessages - {user.Key}");
|
||||
|
||||
@ -1559,10 +1595,10 @@ public class Program
|
||||
AnsiConsole.Markup($"[red]Found 0 Paid Messages\n[/]");
|
||||
}
|
||||
|
||||
return paidMessagesCount;
|
||||
return (paidMessagesCount, newPaidMessagesCount);
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadMessages(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int messagesCount, string path)
|
||||
private static async Task<(int, int)> DownloadMessages(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int messagesCount, string path)
|
||||
{
|
||||
Log.Debug($"Calling DownloadMessages - {user.Key}");
|
||||
|
||||
@ -1702,10 +1738,10 @@ public class Program
|
||||
AnsiConsole.Markup($"[red]Found 0 Messages\n[/]");
|
||||
}
|
||||
|
||||
return messagesCount;
|
||||
return (messagesCount, newMessagesCount);
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadHighlights(IDownloadContext downloadContext, KeyValuePair<string, int> user, int highlightsCount, string path)
|
||||
private static async Task<(int, int)> DownloadHighlights(IDownloadContext downloadContext, KeyValuePair<string, int> user, int highlightsCount, string path)
|
||||
{
|
||||
Log.Debug($"Calling DownloadHighlights - {user.Key}");
|
||||
|
||||
@ -1759,10 +1795,10 @@ public class Program
|
||||
Log.Debug($"Found 0 Highlights");
|
||||
}
|
||||
|
||||
return highlightsCount;
|
||||
return (highlightsCount, newHighlightsCount);
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadStories(IDownloadContext downloadContext, KeyValuePair<string, int> user, int storiesCount, string path)
|
||||
private static async Task<(int, int)> DownloadStories(IDownloadContext downloadContext, KeyValuePair<string, int> user, int storiesCount, string path)
|
||||
{
|
||||
Log.Debug($"Calling DownloadStories - {user.Key}");
|
||||
|
||||
@ -1816,10 +1852,10 @@ public class Program
|
||||
Log.Debug($"Found 0 Stories");
|
||||
}
|
||||
|
||||
return storiesCount;
|
||||
return (storiesCount, newStoriesCount);
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadArchived(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int archivedCount, string path)
|
||||
private static async Task<(int,int)> DownloadArchived(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int archivedCount, string path)
|
||||
{
|
||||
Log.Debug($"Calling DownloadArchived - {user.Key}");
|
||||
|
||||
@ -1949,10 +1985,10 @@ public class Program
|
||||
AnsiConsole.Markup($"[red]Found 0 Archived Posts\n[/]");
|
||||
}
|
||||
|
||||
return archivedCount;
|
||||
return (archivedCount, newArchivedCount);
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadFreePosts(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int postCount, string path)
|
||||
private static async Task<(int,int)> DownloadFreePosts(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int postCount, string path)
|
||||
{
|
||||
Log.Debug($"Calling DownloadFreePosts - {user.Key}");
|
||||
|
||||
@ -1970,7 +2006,7 @@ public class Program
|
||||
{
|
||||
AnsiConsole.Markup($"[red]Found 0 Posts\n[/]");
|
||||
Log.Debug($"Found 0 Posts");
|
||||
return 0;
|
||||
return (0,0);
|
||||
}
|
||||
|
||||
AnsiConsole.Markup($"[red]Found {posts.Posts.Count} Media from {posts.PostObjects.Count} Posts\n[/]");
|
||||
@ -2089,10 +2125,10 @@ public class Program
|
||||
AnsiConsole.Markup($"[red]Posts Already Downloaded: {oldPostCount} New Posts Downloaded: {newPostCount}[/]\n");
|
||||
Log.Debug("Posts Already Downloaded: {oldPostCount} New Posts Downloaded: {newPostCount}");
|
||||
|
||||
return postCount;
|
||||
return (postCount, newPostCount);
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadPaidPosts(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int paidPostCount, string path)
|
||||
private static async Task<(int,int)> DownloadPaidPosts(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int paidPostCount, string path)
|
||||
{
|
||||
Log.Debug($"Calling DownloadPaidPosts - {user.Key}");
|
||||
|
||||
@ -2110,7 +2146,7 @@ public class Program
|
||||
{
|
||||
AnsiConsole.Markup($"[red]Found 0 Paid Posts\n[/]");
|
||||
Log.Debug("Found 0 Paid Posts");
|
||||
return 0;
|
||||
return (0,0);
|
||||
}
|
||||
|
||||
AnsiConsole.Markup($"[red]Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts\n[/]");
|
||||
@ -2222,7 +2258,7 @@ public class Program
|
||||
});
|
||||
AnsiConsole.Markup($"[red]Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}[/]\n");
|
||||
Log.Debug($"Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}");
|
||||
return paidPostCount;
|
||||
return (paidPostCount, newPaidPostCount);
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadPaidPostsPurchasedTab(IDownloadContext downloadContext, PaidPostCollection purchasedPosts, KeyValuePair<string, int> user, int paidPostCount, string path, Dictionary<string, int> users)
|
||||
@ -2474,7 +2510,7 @@ public class Program
|
||||
return paidMessagesCount;
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadStreams(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int streamsCount, string path)
|
||||
private static async Task<(int, int)> DownloadStreams(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, KeyValuePair<string, int> user, int streamsCount, string path)
|
||||
{
|
||||
Log.Debug($"Calling DownloadStreams - {user.Key}");
|
||||
|
||||
@ -2492,7 +2528,7 @@ public class Program
|
||||
{
|
||||
AnsiConsole.Markup($"[red]Found 0 Streams\n[/]");
|
||||
Log.Debug($"Found 0 Streams");
|
||||
return 0;
|
||||
return (0,0);
|
||||
}
|
||||
|
||||
AnsiConsole.Markup($"[red]Found {streams.Streams.Count} Media from {streams.StreamObjects.Count} Streams\n[/]");
|
||||
@ -2610,7 +2646,7 @@ public class Program
|
||||
});
|
||||
AnsiConsole.Markup($"[red]Streams Already Downloaded: {oldStreamsCount} New Streams Downloaded: {newStreamsCount}[/]\n");
|
||||
Log.Debug($"Streams Already Downloaded: {oldStreamsCount} New Streams Downloaded: {newStreamsCount}");
|
||||
return streamsCount;
|
||||
return (streamsCount, newStreamsCount);
|
||||
}
|
||||
|
||||
private static async Task<int> DownloadPaidMessage(IDownloadContext downloadContext, KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP, string username, int paidMessagesCount, string path, long message_id)
|
||||
@ -3466,6 +3502,30 @@ public class Program
|
||||
AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]");
|
||||
Log.Warning("Other OF DL process detected, exiting..");
|
||||
|
||||
Log.CloseAndFlush();
|
||||
Environment.Exit(0);
|
||||
}
|
||||
|
||||
static ILogger LoggerWithConfigContext(Entities.Config config)
|
||||
=> Log.Logger.ForContext(nameof(Entities.Config.DownloadPath), config.DownloadPath)
|
||||
.ForContext(nameof(Entities.Config.DownloadPosts), config.DownloadPosts)
|
||||
.ForContext(nameof(Entities.Config.DownloadPaidPosts), config.DownloadPaidPosts)
|
||||
.ForContext(nameof(Entities.Config.DownloadMessages), config.DownloadMessages)
|
||||
.ForContext(nameof(Entities.Config.DownloadPaidMessages), config.DownloadPaidMessages)
|
||||
.ForContext(nameof(Entities.Config.DownloadStories), config.DownloadStories)
|
||||
.ForContext(nameof(Entities.Config.DownloadStreams), config.DownloadStreams)
|
||||
.ForContext(nameof(Entities.Config.DownloadHighlights), config.DownloadHighlights)
|
||||
.ForContext(nameof(Entities.Config.DownloadArchived), config.DownloadArchived)
|
||||
.ForContext(nameof(Entities.Config.DownloadAvatarHeaderPhoto), config.DownloadAvatarHeaderPhoto)
|
||||
.ForContext(nameof(Entities.Config.DownloadImages), config.DownloadImages)
|
||||
.ForContext(nameof(Entities.Config.DownloadVideos), config.DownloadVideos)
|
||||
.ForContext(nameof(Entities.Config.DownloadAudios), config.DownloadAudios)
|
||||
.ForContext(nameof(Entities.Config.IgnoreOwnMessages), config.IgnoreOwnMessages)
|
||||
.ForContext(nameof(Entities.Config.DownloadPostsIncrementally), config.DownloadPostsIncrementally)
|
||||
.ForContext(nameof(Entities.Config.BypassContentForCreatorsWhoNoLongerExist), config.BypassContentForCreatorsWhoNoLongerExist)
|
||||
.ForContext(nameof(Entities.Config.SkipAds), config.SkipAds)
|
||||
.ForContext(nameof(Entities.Config.IncludeExpiredSubscriptions), config.IncludeExpiredSubscriptions)
|
||||
.ForContext(nameof(Entities.Config.IncludeRestrictedSubscriptions), config.IncludeRestrictedSubscriptions)
|
||||
.ForContext(nameof(Entities.Config.NonInteractiveSpecificLists), config.NonInteractiveSpecificLists)
|
||||
.ForContext(nameof(Entities.Config.NonInteractiveSpecificUsers), config.NonInteractiveSpecificUsers);
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user