reptail/OF-DL
1
0
Fork 0
forked from sim0n00ps/OF-DL
Code Pull Requests Activity

Added output of new download counts in DownloadOrchestrationService

Browse Source
This commit is contained in:
Casper Sparre 2026-02-18 22:30:20 +01:00
parent ae5ce7e491
commit f9c625a4be
3 changed files with 35 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
Cajetan.OF-DL/Worker.cs
View File

@ -177,20 +177,21 @@ internal class Worker(IServiceProvider serviceProvider)
devicePrivateKeyMissing: _devicePrivateKeyMissing, devicePrivateKeyMissing: _devicePrivateKeyMissing,
eventHandler: eventHandler eventHandler: eventHandler
); );
CreatorDownloadResult newResults = results.NewDownloads ?? results;
totalResults.Add(results); totalResults.Add(newResults);
DateTime userEndTime = DateTime.Now; DateTime userEndTime = DateTime.Now;
TimeSpan userTotalTime = userEndTime - userStartTime; TimeSpan userTotalTime = userEndTime - userStartTime;
Log.ForContext("Paid Posts", results.PaidPostCount) Log.ForContext("Paid Posts", newResults.PaidPostCount)
.ForContext("Posts", results.PostCount) .ForContext("Posts", newResults.PostCount)
.ForContext("Archived", results.ArchivedCount) .ForContext("Archived", newResults.ArchivedCount)
.ForContext("Streams", results.StreamsCount) .ForContext("Streams", newResults.StreamsCount)
.ForContext("Stories", results.StoriesCount) .ForContext("Stories", newResults.StoriesCount)
.ForContext("Highlights", results.HighlightsCount) .ForContext("Highlights", newResults.HighlightsCount)
.ForContext("Messages", results.MessagesCount) .ForContext("Messages", newResults.MessagesCount)
.ForContext("Paid Messages", results.PaidMessagesCount) .ForContext("Paid Messages", newResults.PaidMessagesCount)
.ForContext("Username", username) .ForContext("Username", username)
.ForContext("TotalMinutes", userTotalTime.TotalMinutes) .ForContext("TotalMinutes", userTotalTime.TotalMinutes)
.Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes"); .Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes");
@ -220,6 +221,8 @@ internal class Worker(IServiceProvider serviceProvider)
.ForContext("Paid Messages", totalResults.PaidMessagesCount) .ForContext("Paid Messages", totalResults.PaidMessagesCount)
.ForContext("TotalMinutes", totalTime.TotalMinutes) .ForContext("TotalMinutes", totalTime.TotalMinutes)
.Information("Scrape Completed in {TotalMinutes:0.00} minutes"); .Information("Scrape Completed in {TotalMinutes:0.00} minutes");
await Task.Delay(2000);
} }
private async Task OutputBlockedUsersAsync() private async Task OutputBlockedUsersAsync()

2
OF DL.Core/Models/Downloads/CreatorDownloadResult.cs
View File

@ -17,6 +17,8 @@ public class CreatorDownloadResult
public int MessagesCount { get; set; } public int MessagesCount { get; set; }
public int PaidMessagesCount { get; set; } public int PaidMessagesCount { get; set; }
public CreatorDownloadResult? NewDownloads { get; set; }
} }
public class UserListResult public class UserListResult

28
OF DL.Core/Services/DownloadOrchestrationService.cs
View File

@ -160,6 +160,7 @@ public class DownloadOrchestrationService(
{ {
Config config = configService.CurrentConfig; Config config = configService.CurrentConfig;
CreatorDownloadResult counts = new(); CreatorDownloadResult counts = new();
CreatorDownloadResult newCounts = new();
eventHandler.OnUserStarting(username); eventHandler.OnUserStarting(username);
Log.Debug($"Scraping Data for {username}"); Log.Debug($"Scraping Data for {username}");
@ -185,7 +186,8 @@ public class DownloadOrchestrationService(
posts => posts.PaidPosts.Values.ToList(), posts => posts.PaidPosts.Values.ToList(),
async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users, async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter), clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
eventHandler); eventHandler,
n => newCounts.PaidPostCount = n);
} }
if (config.DownloadPosts) if (config.DownloadPosts)
@ -202,7 +204,8 @@ public class DownloadOrchestrationService(
posts => posts.Posts.Values.ToList(), posts => posts.Posts.Values.ToList(),
async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users, async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter), clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
eventHandler); eventHandler,
n => newCounts.PostCount = n);
} }
if (config.DownloadArchived) if (config.DownloadArchived)
@ -215,7 +218,8 @@ public class DownloadOrchestrationService(
archived => archived.ArchivedPosts.Values.ToList(), archived => archived.ArchivedPosts.Values.ToList(),
async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users, async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter), clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter),
eventHandler); eventHandler,
n => newCounts.ArchivedCount = n);
} }
if (config.DownloadStreams) if (config.DownloadStreams)
@ -228,7 +232,8 @@ public class DownloadOrchestrationService(
streams => streams.Streams.Values.ToList(), streams => streams.Streams.Values.ToList(),
async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users, async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter), clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter),
eventHandler); eventHandler,
n => newCounts.StreamsCount = n);
} }
if (config.DownloadStories) if (config.DownloadStories)
@ -252,6 +257,7 @@ public class DownloadOrchestrationService(
eventHandler.OnDownloadComplete("Stories", result); eventHandler.OnDownloadComplete("Stories", result);
counts.StoriesCount = result.TotalCount; counts.StoriesCount = result.TotalCount;
newCounts.StoriesCount = result.NewDownloads;
} }
else else
{ {
@ -280,6 +286,7 @@ public class DownloadOrchestrationService(
eventHandler.OnDownloadComplete("Highlights", result); eventHandler.OnDownloadComplete("Highlights", result);
counts.HighlightsCount = result.TotalCount; counts.HighlightsCount = result.TotalCount;
newCounts.HighlightsCount = result.NewDownloads;
} }
else else
{ {
@ -297,7 +304,8 @@ public class DownloadOrchestrationService(
messages => messages.Messages.Values.ToList(), messages => messages.Messages.Values.ToList(),
async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users, async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter), clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter),
eventHandler); eventHandler,
n => newCounts.MessagesCount = n);
} }
if (config.DownloadPaidMessages) if (config.DownloadPaidMessages)
@ -310,10 +318,13 @@ public class DownloadOrchestrationService(
paidMessages => paidMessages.PaidMessages.Values.ToList(), paidMessages => paidMessages.PaidMessages.Values.ToList(),
async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users, async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter), clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter),
eventHandler); eventHandler,
n => newCounts.PaidMessagesCount = n);
} }
eventHandler.OnUserComplete(username, counts); eventHandler.OnUserComplete(username, counts);
counts.NewDownloads = newCounts;
return counts; return counts;
} }
@ -612,7 +623,8 @@ public class DownloadOrchestrationService(
Func<T, int> getObjectCount, Func<T, int> getObjectCount,
Func<T, List<string>?> getUrls, Func<T, List<string>?> getUrls,
Func<T, IProgressReporter, Task<DownloadResult>> downloadData, Func<T, IProgressReporter, Task<DownloadResult>> downloadData,
IDownloadEventHandler eventHandler) IDownloadEventHandler eventHandler,
Action<int>? newPostAssignmentAction)
{ {
T data = await eventHandler.WithStatusAsync($"Getting {contentType}", T data = await eventHandler.WithStatusAsync($"Getting {contentType}",
async statusReporter => await fetchData(statusReporter)); async statusReporter => await fetchData(statusReporter));
@ -643,6 +655,8 @@ public class DownloadOrchestrationService(
Log.Debug( Log.Debug(
$"{contentType} Already Downloaded: {result.ExistingDownloads} New {contentType} Downloaded: {result.NewDownloads}"); $"{contentType} Already Downloaded: {result.ExistingDownloads} New {contentType} Downloaded: {result.NewDownloads}");
newPostAssignmentAction?.Invoke(result.NewDownloads);
return result.TotalCount; return result.TotalCount;
} }
} }