forked from sim0n00ps/OF-DL
Compare commits
2 Commits
849f8d7f5f
...
af6a9818d6
| Author | SHA1 | Date | |
|---|---|---|---|
| af6a9818d6 | |||
| f3815ba993 |
@ -31,6 +31,7 @@
|
|||||||
<PackageReference Include="Serilog" Version="4.2.0" />
|
<PackageReference Include="Serilog" Version="4.2.0" />
|
||||||
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
|
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
|
||||||
<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
|
<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
|
||||||
|
<PackageReference Include="Serilog.Sinks.Seq" Version="9.0.0"/>
|
||||||
<PackageReference Include="System.Reactive" Version="6.0.1" />
|
<PackageReference Include="System.Reactive" Version="6.0.1" />
|
||||||
<PackageReference Include="xFFmpeg.NET" Version="7.2.0" />
|
<PackageReference Include="xFFmpeg.NET" Version="7.2.0" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|||||||
@ -111,8 +111,11 @@ public class Program
|
|||||||
levelSwitch.MinimumLevel = LogEventLevel.Error; //set initial level (until we've read from config)
|
levelSwitch.MinimumLevel = LogEventLevel.Error; //set initial level (until we've read from config)
|
||||||
|
|
||||||
Log.Logger = new LoggerConfiguration()
|
Log.Logger = new LoggerConfiguration()
|
||||||
|
.Enrich.WithProperty("Application", "OF_DL")
|
||||||
|
.Enrich.WithProperty("StartTime", DateTime.Now)
|
||||||
.MinimumLevel.ControlledBy(levelSwitch)
|
.MinimumLevel.ControlledBy(levelSwitch)
|
||||||
.WriteTo.File("logs/OFDL.txt", rollingInterval: RollingInterval.Day)
|
.WriteTo.File("logs/OFDL.txt", rollingInterval: RollingInterval.Day)
|
||||||
|
.WriteTo.Seq("https://seq.cajetan.dk")
|
||||||
.CreateLogger();
|
.CreateLogger();
|
||||||
|
|
||||||
AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
|
AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
|
||||||
@ -858,7 +861,8 @@ public class Program
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]");
|
Log.Information("Logged In successfully as {Name:l} ({Username:l})", validate.name, validate.username);
|
||||||
|
AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} ({validate.username})\n[/]");
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@ -939,6 +943,7 @@ public class Program
|
|||||||
DateTime startTime = DateTime.Now;
|
DateTime startTime = DateTime.Now;
|
||||||
Dictionary<string, int> users = new();
|
Dictionary<string, int> users = new();
|
||||||
|
|
||||||
|
Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", Config.IncludeRestrictedSubscriptions);
|
||||||
AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
||||||
Dictionary<string, int> subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
|
Dictionary<string, int> subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
|
||||||
|
|
||||||
@ -956,6 +961,7 @@ public class Program
|
|||||||
{
|
{
|
||||||
Log.Debug("Inactive Subscriptions: ");
|
Log.Debug("Inactive Subscriptions: ");
|
||||||
|
|
||||||
|
Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", Config.IncludeRestrictedSubscriptions);
|
||||||
AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
||||||
Dictionary<string, int> subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
|
Dictionary<string, int> subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
|
||||||
|
|
||||||
@ -1000,6 +1006,7 @@ public class Program
|
|||||||
if (!lists.TryGetValue(listName, out int listId))
|
if (!lists.TryGetValue(listName, out int listId))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
|
Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", listName, Config.IncludeRestrictedSubscriptions);
|
||||||
AnsiConsole.Markup($"[green]Getting Users from list '{listName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
AnsiConsole.Markup($"[green]Getting Users from list '{listName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
||||||
Dictionary<string, int> list = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
|
Dictionary<string, int> list = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
|
||||||
|
|
||||||
@ -1023,6 +1030,7 @@ public class Program
|
|||||||
else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName))
|
else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName))
|
||||||
{
|
{
|
||||||
var listId = lists[Config.NonInteractiveModeListName];
|
var listId = lists[Config.NonInteractiveModeListName];
|
||||||
|
Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", Config.NonInteractiveModeListName, Config.IncludeRestrictedSubscriptions);
|
||||||
AnsiConsole.Markup($"[green]Getting Users from list '{Config.NonInteractiveModeListName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
AnsiConsole.Markup($"[green]Getting Users from list '{Config.NonInteractiveModeListName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
|
||||||
users = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
|
users = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
|
||||||
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
|
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
|
||||||
@ -1152,11 +1160,13 @@ public class Program
|
|||||||
Log.Debug($"Download path: {p}");
|
Log.Debug($"Download path: {p}");
|
||||||
|
|
||||||
List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
|
List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
|
||||||
int userNum = 1;
|
int userNum = 0;
|
||||||
int userCount = purchasedTabCollections.Count;
|
int userCount = purchasedTabCollections.Count;
|
||||||
foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
|
foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
|
||||||
{
|
{
|
||||||
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]");
|
Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", purchasedTabCollection.Username, ++userNum, userCount);
|
||||||
|
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum} of {userCount})\n[/]");
|
||||||
|
|
||||||
string path = "";
|
string path = "";
|
||||||
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
||||||
{
|
{
|
||||||
@ -1266,8 +1276,10 @@ public class Program
|
|||||||
else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
||||||
{
|
{
|
||||||
//Iterate over each user in the list of users
|
//Iterate over each user in the list of users
|
||||||
int userNum = 1;
|
int userNum = 0;
|
||||||
int userCount = hasSelectedUsersKVP.Value.Count;
|
int userCount = hasSelectedUsersKVP.Value.Count;
|
||||||
|
|
||||||
|
LoggerWithConfigContext(config).Information("Scraping Data for {UserCount} user(s)", userCount);
|
||||||
foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
|
foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
|
||||||
{
|
{
|
||||||
int paidPostCount = 0;
|
int paidPostCount = 0;
|
||||||
@ -1278,9 +1290,9 @@ public class Program
|
|||||||
int highlightsCount = 0;
|
int highlightsCount = 0;
|
||||||
int messagesCount = 0;
|
int messagesCount = 0;
|
||||||
int paidMessagesCount = 0;
|
int paidMessagesCount = 0;
|
||||||
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]");
|
|
||||||
|
|
||||||
Log.Debug($"Scraping Data for {user.Key}");
|
Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", user.Key, ++userNum, userCount);
|
||||||
|
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum} of {userCount})\n[/]");
|
||||||
|
|
||||||
string path = "";
|
string path = "";
|
||||||
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
||||||
@ -1373,10 +1385,17 @@ public class Program
|
|||||||
.AddItem("Messages", messagesCount, Color.LightGreen)
|
.AddItem("Messages", messagesCount, Color.LightGreen)
|
||||||
.AddItem("Paid Messages", paidMessagesCount, Color.Aqua));
|
.AddItem("Paid Messages", paidMessagesCount, Color.Aqua));
|
||||||
AnsiConsole.Markup("\n");
|
AnsiConsole.Markup("\n");
|
||||||
}
|
|
||||||
|
Log.Information(
|
||||||
|
"Scraped Data for '{Username:l}' [P: {PostCount}] [PP: {PaidPostCount}] [M: {MessagesCount}] [PM: {PaidMessagesCount}]",
|
||||||
|
user.Key, postCount, paidPostCount, messagesCount, paidMessagesCount, storiesCount, archivedCount
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
DateTime endTime = DateTime.Now;
|
DateTime endTime = DateTime.Now;
|
||||||
TimeSpan totalTime = endTime - startTime;
|
TimeSpan totalTime = endTime - startTime;
|
||||||
AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]");
|
Log.Information("Scrape Completed in {TotalMinutes:0.00} minutes", totalTime.TotalMinutes);
|
||||||
|
AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]");
|
||||||
}
|
}
|
||||||
else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
||||||
{
|
{
|
||||||
@ -3465,4 +3484,25 @@ public class Program
|
|||||||
|
|
||||||
Environment.Exit(0);
|
Environment.Exit(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static ILogger LoggerWithConfigContext(Entities.Config config)
|
||||||
|
=> Log.Logger.ForContext(nameof(Entities.Config.DownloadPath), config.DownloadPath)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadPosts), config.DownloadPosts)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadPaidPosts), config.DownloadPaidPosts)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadMessages), config.DownloadMessages)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadPaidMessages), config.DownloadPaidMessages)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadStories), config.DownloadStories)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadStreams), config.DownloadStreams)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadHighlights), config.DownloadHighlights)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadArchived), config.DownloadArchived)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadAvatarHeaderPhoto), config.DownloadAvatarHeaderPhoto)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadImages), config.DownloadImages)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadVideos), config.DownloadVideos)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadAudios), config.DownloadAudios)
|
||||||
|
.ForContext(nameof(Entities.Config.IgnoreOwnMessages), config.IgnoreOwnMessages)
|
||||||
|
.ForContext(nameof(Entities.Config.DownloadPostsIncrementally), config.DownloadPostsIncrementally)
|
||||||
|
.ForContext(nameof(Entities.Config.BypassContentForCreatorsWhoNoLongerExist), config.BypassContentForCreatorsWhoNoLongerExist)
|
||||||
|
.ForContext(nameof(Entities.Config.SkipAds), config.SkipAds)
|
||||||
|
.ForContext(nameof(Entities.Config.IncludeExpiredSubscriptions), config.IncludeExpiredSubscriptions)
|
||||||
|
.ForContext(nameof(Entities.Config.IncludeRestrictedSubscriptions), config.IncludeRestrictedSubscriptions);
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user