Compare commits

...

9 Commits

24 changed files with 1241 additions and 33 deletions

View File

@ -0,0 +1,45 @@
using OF_DL.Models.Downloads;
namespace OF_DL.CLI;
public interface ICajetanDownloadEventHandler : IDownloadEventHandler
{
void OnMessage(string message, string color);
}
public class CajetanDownloadEventHandler : ICajetanDownloadEventHandler
{
private readonly SpectreDownloadEventHandler _eventHandler = new();
public void OnContentFound(string contentType, int mediaCount, int objectCount)
=> _eventHandler.OnContentFound(contentType, mediaCount, objectCount);
public void OnDownloadComplete(string contentType, DownloadResult result)
=> _eventHandler.OnDownloadComplete(contentType, result);
public void OnMessage(string message)
=> _eventHandler.OnMessage(message);
public void OnMessage(string message, string color)
=> AnsiConsole.Markup($"[{color.ToLowerInvariant()}]{Markup.Escape(message)}\n[/]");
public void OnNoContentFound(string contentType)
=> _eventHandler.OnNoContentFound(contentType);
public void OnPurchasedTabUserComplete(string username, int paidPostCount, int paidMessagesCount)
=> _eventHandler.OnPurchasedTabUserComplete(username, paidPostCount, paidMessagesCount);
public void OnScrapeComplete(TimeSpan elapsed)
=> _eventHandler.OnScrapeComplete(elapsed);
public void OnUserComplete(string username, CreatorDownloadResult result)
=> _eventHandler.OnUserComplete(username, result);
public void OnUserStarting(string username) { }
public Task<T> WithProgressAsync<T>(string description, long maxValue, bool showSize, Func<IProgressReporter, Task<T>> work)
=> _eventHandler.WithProgressAsync(description, maxValue, showSize, work);
public Task<T> WithStatusAsync<T>(string statusMessage, Func<IStatusReporter, Task<T>> work)
=> _eventHandler.WithStatusAsync(statusMessage, work);
}

View File

@ -0,0 +1,6 @@
namespace OF_DL.Exceptions;
public sealed class ExitCodeException(int exitCode) : Exception
{
public int ExitCode { get; } = exitCode;
}

View File

@ -0,0 +1,11 @@
namespace OF_DL.Models;
public class CajetanConfig
{
public string[] NonInteractiveSpecificLists { get; set; } = [];
public string[] NonInteractiveSpecificUsers { get; set; } = [];
public EMode Mode { get; set; } = EMode.None;
public string ErrorMessage { get; set; } = string.Empty;
}

View File

@ -0,0 +1,19 @@
namespace OF_DL.Models.Dtos.Messages;
public class ChatsDto
{
[JsonProperty("list")] public List<ChatItemDto> List { get; set; } = [];
[JsonProperty("hasMore")] public bool HasMore { get; set; }
[JsonProperty("nextOffset")] public int NextOffset { get; set; }
}
public class ChatItemDto
{
[JsonProperty("withUser")] public ChatUserDto WithUser { get; set; } = new();
[JsonProperty("unreadMessagesCount")] public int UnreadMessagesCount { get; set; }
}
public class ChatUserDto
{
[JsonProperty("id")] public long Id { get; set; }
}

View File

@ -0,0 +1,9 @@
namespace OF_DL.Models;
public enum EMode
{
None,
DownloadCreatorContent,
OutputBlockedUsers,
UpdateAllUserInfo
}

View File

@ -0,0 +1,19 @@
namespace OF_DL.Models.Downloads;
internal static class DownloadsExtensions
{
extension(CreatorDownloadResult result)
{
public void Add(CreatorDownloadResult other)
{
result.PaidPostCount += other.PaidPostCount;
result.PostCount += other.PostCount;
result.ArchivedCount += other.ArchivedCount;
result.StreamsCount += other.StreamsCount;
result.StoriesCount += other.StoriesCount;
result.HighlightsCount += other.HighlightsCount;
result.MessagesCount += other.MessagesCount;
result.PaidMessagesCount += other.PaidMessagesCount;
}
}
}

View File

@ -1,10 +1,20 @@
using System.Diagnostics;
using System.Reflection;
using Microsoft.Extensions.DependencyInjection;
AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
await RunAsync(args);
ServiceCollection services = await ConfigureServices(args);
static async Task RunAsync(string[] args)
{
ServiceCollection services = await ConfigureServices(args);
ServiceProvider serviceProvider = services.BuildServiceProvider();
ExitIfOtherProcess(serviceProvider);
Worker worker = serviceProvider.GetRequiredService<Worker>();
await worker.RunAsync();
}
static async Task<ServiceCollection> ConfigureServices(string[] args)
{
@ -22,30 +32,160 @@ static async Task<ServiceCollection> ConfigureServices(string[] args)
if (!await configService.LoadConfigurationAsync(args))
{
AnsiConsole.MarkupLine("\n[red]config.conf is not valid, check your syntax![/]\n");
if (!configService.IsCliNonInteractive)
{
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
Console.ReadKey();
}
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
Console.ReadKey();
exitHelper.ExitWithCode(3);
}
if (configService.CurrentConfig.NonInteractiveMode is false)
{
AnsiConsole.MarkupLine("\n[red]Cannot run in Interactive Mode![/]\n");
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
Console.ReadKey();
exitHelper.ExitWithCode(3);
}
AnsiConsole.Markup("[green]config.conf located successfully!\n[/]");
if (!ParseCommandlineArgs(args, configService.CurrentConfig, out CajetanConfig cajetanConfig))
{
AnsiConsole.MarkupLine($"\n[red]{cajetanConfig.ErrorMessage}[/]\n");
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
Console.ReadKey();
exitHelper.ExitWithCode(3);
}
// Set up full dependency injection with loaded config
services = [];
services.AddSingleton(loggingService);
services.AddSingleton(configService);
services.AddSingleton(exitHelper);
services.AddSingleton(cajetanConfig);
services.AddSingleton<IAuthService, AuthService>();
services.AddSingleton<IApiService, ApiService>();
services.AddSingleton<IDbService, DbService>();
services.AddSingleton<IDownloadService, DownloadService>();
services.AddSingleton<IFileNameService, FileNameService>();
services.AddSingleton<IStartupService, StartupService>();
services.AddSingleton<IDownloadOrchestrationService, DownloadOrchestrationService>();
services.AddSingleton<Program>();
services.AddSingleton<IFileNameService, FileNameService>();
services.AddSingleton<ICajetanDownloadService, CajetanDownloadService>();
services.AddSingleton<IDownloadService>(sp => sp.GetRequiredService<ICajetanDownloadService>());
services.AddSingleton<ICajetanApiService, CajetanApiService>();
services.AddSingleton<IApiService>(sp => sp.GetRequiredService<ICajetanApiService>());
services.AddSingleton<ICajetanDbService, CajetanDbService>();
services.AddSingleton<IDbService>(sp => sp.GetRequiredService<ICajetanDbService>());
services.AddSingleton<ICajetanDownloadOrchestrationService, CajetanDownloadOrchestrationService>();
services.AddSingleton<IDownloadOrchestrationService>(sp => sp.GetRequiredService<ICajetanDownloadOrchestrationService>());
services.AddSingleton<ICajetanDownloadEventHandler, CajetanDownloadEventHandler>();
services.AddSingleton<IDownloadEventHandler>(sp => sp.GetRequiredService<ICajetanDownloadEventHandler>());
services.AddSingleton<Worker>();
return services;
}
static bool ParseCommandlineArgs(string[] args, Config currentConfig, out CajetanConfig parsedConfig)
{
const string SPECIFIC_LISTS_ARG = "--specific-lists";
const string SPECIFIC_USERS_ARG = "--specific-users";
const string OUTPUT_BLOCKED_USERS_ARG = "--output-blocked";
const string UPDATE_ALL_USER_INFO_ARG = "--update-userinfo";
parsedConfig = new();
if (ParseListAndUserArguments(ref parsedConfig))
return true;
if (ParseFlagArgument(OUTPUT_BLOCKED_USERS_ARG, EMode.OutputBlockedUsers, ref parsedConfig))
return true;
if (ParseFlagArgument(UPDATE_ALL_USER_INFO_ARG, EMode.UpdateAllUserInfo, ref parsedConfig))
return true;
parsedConfig.ErrorMessage = "No mode argument provided!";
return false;
bool ParseListAndUserArguments(ref CajetanConfig parsedConfig)
{
bool hasSpecificListsArg = ParseCommaSeparatedListArgument(SPECIFIC_LISTS_ARG, ref parsedConfig, (c, v) =>
{
c.NonInteractiveSpecificLists = v;
Log.Logger = Log.Logger.ForContext(nameof(CajetanConfig.NonInteractiveSpecificLists), string.Join(",", v));
});
if (hasSpecificListsArg)
return true;
bool hasSpecificUsersArg = ParseCommaSeparatedListArgument(SPECIFIC_USERS_ARG, ref parsedConfig, (c, v) =>
{
c.NonInteractiveSpecificUsers = v;
Log.Logger = Log.Logger.ForContext(nameof(CajetanConfig.NonInteractiveSpecificUsers), string.Join(",", v));
});
if (hasSpecificUsersArg)
return true;
return false;
}
bool ParseCommaSeparatedListArgument(string argName, ref CajetanConfig parsedConfig, Action<CajetanConfig, string[]> assignmentFunc)
{
char[] separator = [','];
int indexOfArg = Array.FindIndex(args, a => argName.Equals(a, StringComparison.OrdinalIgnoreCase));
if (indexOfArg < 0)
return false;
int indexOfListValues = indexOfArg + 1;
string[] strListValues = args.ElementAtOrDefault(indexOfListValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
if (strListValues.Length == 0)
return false;
assignmentFunc(parsedConfig, strListValues);
parsedConfig.Mode = EMode.DownloadCreatorContent;
Log.Logger = Log.Logger.ForContext("Mode", $"{EMode.DownloadCreatorContent}");
return true;
}
bool ParseFlagArgument(string argName, EMode modeIfArgIsSet, ref CajetanConfig parsedConfig)
{
if (!args.Any(a => argName.Equals(a, StringComparison.OrdinalIgnoreCase)))
return false;
currentConfig.NonInteractiveMode = true;
parsedConfig.Mode = modeIfArgIsSet;
Log.Logger = Log.Logger.ForContext("Mode", $"{modeIfArgIsSet}");
return true;
}
}
static void ExitIfOtherProcess(ServiceProvider serviceProvider)
{
Assembly? entryAssembly = Assembly.GetEntryAssembly();
AssemblyName? entryAssemblyName = entryAssembly?.GetName();
if (entryAssemblyName?.Name is null)
return;
Process thisProcess = Process.GetCurrentProcess();
Process[] otherProcesses = [.. Process.GetProcessesByName(entryAssemblyName.Name).Where(p => p.Id != thisProcess.Id)];
if (otherProcesses.Length <= 0)
return;
AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]");
Log.Warning("Other OF DL process detected, exiting..");
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(0);
}

View File

@ -1,9 +1,18 @@
global using Spectre.Console;
global using Newtonsoft.Json;
global using OF_DL;
global using OF_DL.CLI;
global using OF_DL.Crypto;
global using OF_DL.Enumerations;
global using OF_DL.Exceptions;
global using OF_DL.Helpers;
global using OF_DL.Models;
global using OF_DL.Models.Config;
global using OF_DL.Models.Downloads;
global using OF_DL.Services;
global using OF_DL.Utils;
global using Serilog;
global using Serilog.Context;
global using Spectre.Console;
global using MessageDtos = OF_DL.Models.Dtos.Messages;
global using MessageEntities = OF_DL.Models.Entities.Messages;
global using SubscriptionDtos = OF_DL.Models.Dtos.Subscriptions;
global using UserDtos = OF_DL.Models.Dtos.Users;
global using UserEntities = OF_DL.Models.Entities.Users;

View File

@ -0,0 +1,8 @@
{
"profiles": {
"Cajetan.OF-DL": {
"commandName": "Project",
"commandLineArgs": "--non-interactive --specific-lists Capture"
}
}
}

View File

@ -0,0 +1,303 @@
namespace OF_DL.Services;
public class CajetanApiService(IAuthService authService, IConfigService configService, ICajetanDbService dbService, ICajetanDownloadEventHandler eventHandler)
: ApiService(authService, configService, dbService), ICajetanApiService
{
private readonly ICajetanDownloadEventHandler _eventHandler = eventHandler;
public new async Task<UserEntities.User?> GetUserInfo(string endpoint)
{
UserEntities.UserInfo? userInfo = await GetDetailedUserInfoAsync(endpoint);
if (userInfo is not null && !endpoint.EndsWith("/me"))
await dbService.UpdateUserInfoAsync(userInfo);
return userInfo;
}
public async Task<UserEntities.UserInfo?> GetDetailedUserInfoAsync(string endpoint)
{
Log.Debug($"Calling GetDetailedUserInfo: {endpoint}");
if (!HasSignedRequestAuth())
return null;
try
{
UserEntities.UserInfo userInfo = new();
Dictionary<string, string> getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() }, { "order", "publish_date_asc" }
};
HttpClient client = new();
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
using HttpResponseMessage response = await client.SendAsync(request);
if (!response.IsSuccessStatusCode)
return userInfo;
response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync();
UserDtos.UserDto? userDto = JsonConvert.DeserializeObject<UserDtos.UserDto>(body, s_mJsonSerializerSettings);
userInfo = FromDto(userDto);
return userInfo;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return null;
}
public async Task<Dictionary<string, long>> GetUsersWithProgressAsync(string typeDisplay, string endpoint, string? typeParam, bool offsetByCount)
{
Dictionary<string, long> usersOfType = await _eventHandler.WithStatusAsync(
statusMessage: $"Getting {typeDisplay} Users",
work: FetchAsync
);
return usersOfType;
async Task<Dictionary<string, long>> FetchAsync(IStatusReporter statusReporter)
{
Dictionary<string, long> users = [];
int limit = 50;
int offset = 0;
bool includeRestricted = true;
Dictionary<string, string> getParams = new()
{
["format"] = "infinite",
["limit"] = limit.ToString(),
["offset"] = offset.ToString()
};
if (!string.IsNullOrWhiteSpace(typeParam))
getParams["type"] = typeParam;
try
{
Log.Debug("Calling GetUsersWithProgress");
HttpClient client = GetHttpClient();
bool isLastLoop = false;
while (true)
{
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, client);
if (string.IsNullOrWhiteSpace(body))
break;
SubscriptionDtos.SubscriptionsDto? subscriptions = DeserializeJson<SubscriptionDtos.SubscriptionsDto>(body, s_mJsonSerializerSettings);
if (subscriptions?.List is null)
break;
foreach (SubscriptionDtos.ListItemDto item in subscriptions.List)
{
if (string.IsNullOrWhiteSpace(item?.Username))
continue;
if (users.ContainsKey(item.Username))
continue;
bool isRestricted = item.IsRestricted ?? false;
bool isRestrictedButAllowed = isRestricted && includeRestricted;
if (!isRestricted || isRestrictedButAllowed)
users.Add(item.Username, item.Id);
}
statusReporter.ReportStatus($"[blue]Getting {typeDisplay} Users\n[/] [blue]Found {users.Count}[/]");
if (isLastLoop)
break;
if (!subscriptions.HasMore || subscriptions.List.Count == 0)
isLastLoop = true;
offset += offsetByCount
? subscriptions.List.Count
: limit;
getParams["offset"] = offset.ToString();
}
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return users;
}
}
public new async Task<MessageEntities.MessageCollection> GetMessages(string endpoint, string folder, IStatusReporter statusReporter)
{
(bool couldExtract, long userId) = ExtractUserId(endpoint);
_eventHandler.OnMessage("Getting Unread Chats", "grey");
HashSet<long> usersWithUnread = couldExtract ? await GetUsersWithUnreadMessagesAsync() : [];
MessageEntities.MessageCollection messages = await base.GetMessages(endpoint, folder, statusReporter);
if (usersWithUnread.Contains(userId))
{
_eventHandler.OnMessage("Restoring unread state", "grey");
await MarkAsUnreadAsync($"chats/{userId}/mark-as-read");
}
return messages;
static (bool couldExtract, long userId) ExtractUserId(string endpoint)
{
string withoutChatsAndMessages = endpoint
.Replace("chats", "", StringComparison.OrdinalIgnoreCase)
.Replace("messages", "", StringComparison.OrdinalIgnoreCase);
string trimmed = withoutChatsAndMessages.Trim(' ', '/', '\\');
if (long.TryParse(trimmed, out long userId))
return (true, userId);
return (false, default);
}
}
public async Task<HashSet<long>> GetUsersWithUnreadMessagesAsync()
{
MessageDtos.ChatsDto unreadChats = await GetChatsAsync("", onlyUnread: true);
HashSet<long> userWithUnread = [];
foreach (MessageDtos.ChatItemDto chatItem in unreadChats.List)
{
if (chatItem?.WithUser?.Id is null)
continue;
if (chatItem.UnreadMessagesCount <= 0)
continue;
userWithUnread.Add(chatItem.WithUser.Id);
}
return userWithUnread;
}
public async Task MarkAsUnreadAsync(string endpoint)
{
Log.Debug($"Calling MarkAsUnread - {endpoint}");
try
{
var result = new { success = false };
string? body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(), HttpMethod.Delete);
if (!string.IsNullOrWhiteSpace(body))
result = JsonConvert.DeserializeAnonymousType(body, result);
if (result?.success != true)
_eventHandler.OnMessage($"Failed to mark chat as unread! Endpoint: {endpoint}", "yellow");
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
}
private async Task<MessageDtos.ChatsDto> GetChatsAsync(string endpoint, bool onlyUnread)
{
Log.Debug($"Calling GetChats - {endpoint}");
MessageDtos.ChatsDto allChats = new();
try
{
int limit = 60;
Dictionary<string, string> getParams = new()
{
{ "limit", $"{limit}" },
{ "offset", "0" },
{ "skip_users", "all" },
{ "order", "recent" }
};
if (onlyUnread)
getParams["filter"] = "unread";
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
MessageDtos.ChatsDto? chats = DeserializeJson<MessageDtos.ChatsDto>(body, s_mJsonSerializerSettings);
if (chats is null)
return allChats;
if (chats.HasMore)
{
getParams["offset"] = $"{chats.NextOffset}";
while (true)
{
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
MessageDtos.ChatsDto? newChats = DeserializeJson<MessageDtos.ChatsDto>(loopbody, s_mJsonSerializerSettings);
if (newChats is null)
break;
allChats.List.AddRange(newChats.List);
if (!newChats.HasMore)
break;
getParams["offset"] = $"{newChats.NextOffset}";
}
}
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return allChats;
}
private static UserEntities.UserInfo FromDto(UserDtos.UserDto? userDto)
{
if (userDto is null)
return new();
return new()
{
Id = userDto.Id,
Avatar = userDto.Avatar,
Header = userDto.Header,
Name = userDto.Name,
Username = userDto.Username,
SubscribePrice = userDto.SubscribePrice,
CurrentSubscribePrice = userDto.CurrentSubscribePrice,
IsPaywallRequired = userDto.IsPaywallRequired,
IsRestricted = userDto.IsRestricted,
SubscribedBy = userDto.SubscribedBy,
SubscribedByExpire = userDto.SubscribedByExpire,
SubscribedByExpireDate = userDto.SubscribedByExpireDate,
SubscribedByAutoprolong = userDto.SubscribedByAutoprolong,
SubscribedIsExpiredNow = userDto.SubscribedIsExpiredNow,
SubscribedOn = userDto.SubscribedOn,
SubscribedOnExpiredNow = userDto.SubscribedOnExpiredNow,
SubscribedOnDuration = userDto.SubscribedOnDuration,
About = userDto.About,
PostsCount = userDto.PostsCount,
ArchivedPostsCount = userDto.ArchivedPostsCount,
PrivateArchivedPostsCount = userDto.PrivateArchivedPostsCount,
PhotosCount = userDto.PhotosCount,
VideosCount = userDto.VideosCount,
AudiosCount = userDto.AudiosCount,
MediasCount = userDto.MediasCount,
};
}
}

View File

@ -0,0 +1,106 @@
using Microsoft.Data.Sqlite;
namespace OF_DL.Services;
public class CajetanDbService(IConfigService configService)
: DbService(configService), ICajetanDbService
{
public async Task InitializeUserInfoTablesAsync()
{
await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, about VARCHAR NULL, expires_on TIMESTAMP NULL, photo_count INT NOT NULL, video_count INT NOT NULL, PRIMARY KEY(user_id));", connection))
{
await cmdInfo.ExecuteNonQueryAsync();
}
using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info_blob (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, blob TEXT NULL, PRIMARY KEY(user_id));", connection))
{
await cmdInfo.ExecuteNonQueryAsync();
}
}
public async Task<Dictionary<string, long>> GetUsersAsync()
{
await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
using SqliteCommand cmd = new("SELECT user_id, username FROM users", connection);
using SqliteDataReader reader = cmd.ExecuteReader();
Dictionary<string, long> result = new(StringComparer.OrdinalIgnoreCase);
while (reader.Read())
{
long userId = reader.GetInt64(0);
string username = reader.GetString(1);
result[username] = userId;
}
return result;
}
public async Task UpdateUserInfoAsync(UserEntities.UserInfo? userInfo)
{
if (userInfo?.Id is null || userInfo?.Username is null)
return;
await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
Log.Debug("Database data source: " + connection.DataSource);
await UpdateAsync();
await UpdateBlobAsync();
async Task UpdateAsync()
{
using SqliteCommand cmdInfo = new(
"INSERT OR REPLACE INTO user_info (user_id, name, about, expires_on, photo_count, video_count) " +
"VALUES (@userId, @name, @about, @expiresOn, @photoCount, @videoCount);",
connection
);
cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
cmdInfo.Parameters.AddWithValue("@about", userInfo.About);
cmdInfo.Parameters.AddWithValue("@expiresOn", userInfo.SubscribedByExpireDate);
cmdInfo.Parameters.AddWithValue("@photoCount", userInfo.PhotosCount ?? 0);
cmdInfo.Parameters.AddWithValue("@videoCount", userInfo.VideosCount ?? 0);
try
{
await cmdInfo.ExecuteNonQueryAsync();
Log.Debug("Inserted or updated creator info: {Username:l}", userInfo.Username);
}
catch (Exception ex)
{
Log.Warning(ex, "Failed to update User Info for: {Username:l}", userInfo.Username);
}
}
async Task UpdateBlobAsync()
{
using SqliteCommand cmdInfo = new(
"INSERT OR REPLACE INTO user_info_blob (user_id, name, blob) " +
"VALUES (@userId, @name, @blob);",
connection
);
cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
cmdInfo.Parameters.AddWithValue("@blob", Newtonsoft.Json.JsonConvert.SerializeObject(userInfo));
try
{
await cmdInfo.ExecuteNonQueryAsync();
Log.Debug("Inserted or updated creator blob: {Username:l}", userInfo.Username);
}
catch (Exception ex)
{
Log.Warning(ex, "Failed to update User Info Blob for: {Username:l}", userInfo.Username);
}
}
}
}

View File

@ -0,0 +1,6 @@
namespace OF_DL.Services;
public class CajetanDownloadOrchestrationService(ICajetanApiService apiService, IConfigService configService, ICajetanDownloadService downloadService, ICajetanDbService dbService)
: DownloadOrchestrationService(apiService, configService, downloadService, dbService), ICajetanDownloadOrchestrationService
{
}

View File

@ -0,0 +1,6 @@
namespace OF_DL.Services;
public class CajetanDownloadService(IAuthService authService, IConfigService configService, ICajetanDbService dbService, IFileNameService fileNameService, ICajetanApiService apiService)
: DownloadService(authService, configService, dbService, fileNameService, apiService), ICajetanDownloadService
{
}

View File

@ -0,0 +1,9 @@
namespace OF_DL.Services;
public interface ICajetanApiService : IApiService
{
Task<UserEntities.UserInfo?> GetDetailedUserInfoAsync(string endpoint);
Task<Dictionary<string, long>> GetUsersWithProgressAsync(string typeDisplay, string endpoint, string? typeParam, bool offsetByCount);
Task<HashSet<long>> GetUsersWithUnreadMessagesAsync();
Task MarkAsUnreadAsync(string endpoint);
}

View File

@ -0,0 +1,9 @@
namespace OF_DL.Services;
public interface ICajetanDbService : IDbService
{
Task InitializeUserInfoTablesAsync();
Task<Dictionary<string, long>> GetUsersAsync();
Task UpdateUserInfoAsync(UserEntities.UserInfo? userInfo);
}

View File

@ -0,0 +1,6 @@
namespace OF_DL.Services;
public interface ICajetanDownloadOrchestrationService : IDownloadOrchestrationService
{
}

View File

@ -0,0 +1,6 @@
namespace OF_DL.Services;
public interface ICajetanDownloadService : IDownloadService
{
}

428
Cajetan.OF-DL/Worker.cs Normal file
View File

@ -0,0 +1,428 @@
using Microsoft.Extensions.DependencyInjection;
namespace OF_DL;
internal class Worker(IServiceProvider serviceProvider)
{
private readonly IConfigService _configService = serviceProvider.GetRequiredService<IConfigService>();
private readonly IAuthService _authService = serviceProvider.GetRequiredService<IAuthService>();
private readonly IStartupService _startupService = serviceProvider.GetRequiredService<IStartupService>();
private readonly IDownloadOrchestrationService _orchestrationService = serviceProvider.GetRequiredService<IDownloadOrchestrationService>();
private readonly ICajetanDbService _dbService = serviceProvider.GetRequiredService<ICajetanDbService>();
private readonly ICajetanApiService _apiService = serviceProvider.GetRequiredService<ICajetanApiService>();
private readonly ExitHelper _exitHelper = serviceProvider.GetRequiredService<ExitHelper>();
private readonly CajetanConfig _cajetanConfig = serviceProvider.GetRequiredService<CajetanConfig>();
private bool _clientIdBlobMissing = true;
private bool _devicePrivateKeyMissing = true;
public async Task RunAsync()
{
try
{
await InitializeAsync();
Task tMode = _cajetanConfig.Mode switch
{
EMode.DownloadCreatorContent => DownloadCreatorContentAsync(),
EMode.OutputBlockedUsers => OutputBlockedUsersAsync(),
EMode.UpdateAllUserInfo => UpdateUserInfoAsync(),
_ => Task.CompletedTask
};
await tMode;
}
catch (ExitCodeException ex)
{
_exitHelper.ExitWithCode(ex.ExitCode);
}
catch (Exception ex)
{
Log.Fatal(ex, "Unhandled Exception! {ExceptionMessage:l}", ex.Message);
}
finally
{
_exitHelper.ExitWithCode(0);
}
}
private async Task InitializeAsync()
{
StartupResult startupResult = await _startupService.ValidateEnvironmentAsync();
if (startupResult.IsWindowsVersionValid is false)
throw new ExitCodeException(1);
if (startupResult.FfmpegFound is false)
throw new ExitCodeException(4);
if (startupResult.ClientIdBlobMissing || startupResult.DevicePrivateKeyMissing)
AnsiConsole.Markup("[yellow]device_client_id_blob and/or device_private_key missing, https://ofdl.tools/ or https://cdrm-project.com/ will be used instead for DRM protected videos\n[/]");
_clientIdBlobMissing = startupResult.ClientIdBlobMissing;
_devicePrivateKeyMissing = startupResult.DevicePrivateKeyMissing;
if (IsRulesJsonValid() is false)
{
AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]");
Console.ReadKey();
throw new ExitCodeException(2);
}
if (await IsAuthorizedAsync() is false)
{
AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]");
Console.ReadKey();
throw new ExitCodeException(2);
}
bool IsRulesJsonValid()
{
if (startupResult.RulesJsonExists is false)
return true;
if (startupResult.RulesJsonValid)
{
AnsiConsole.Markup("[green]rules.json located successfully!\n[/]");
return true;
}
AnsiConsole.MarkupLine("\n[red]rules.json is not valid, check your JSON syntax![/]\n");
Log.Error("processing of rules.json failed: {Error:l}", startupResult.RulesJsonError);
return false;
}
async Task<bool> IsAuthorizedAsync()
{
if (await _authService.LoadFromFileAsync() is false)
{
if (File.Exists("auth.json"))
{
Log.Error("Auth file was found but could not be deserialized!");
AnsiConsole.MarkupLine("\n[red]auth.json could not be deserialized.[/]");
return false;
}
Log.Error("Auth file was not found!");
AnsiConsole.MarkupLine("\n[red]auth.json is missing.");
return false;
}
AnsiConsole.Markup("[green]auth.json located successfully!\n[/]");
// Validate cookie string
_authService.ValidateCookieString();
UserEntities.User? user = await _authService.ValidateAuthAsync();
if (user is null || (user.Name is null && user.Username is null))
{
Log.Error("Auth failed");
_authService.CurrentAuth = null;
AnsiConsole.MarkupLine("\n[red]Auth failed. Please try again or use other authentication methods detailed here:[/]\n");
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth[/]\n");
return false;
}
string displayName = $"{user.Name} {user.Username}".Trim();
AnsiConsole.MarkupLine($"[green]Logged In successfully as {displayName}\n[/]");
return true;
}
}
private async Task DownloadCreatorContentAsync()
{
DateTime startTime = DateTime.Now;
UserListResult allUsersAndLists = await GetAvailableUsersAsync();
Dictionary<string, long> usersToDownload = [];
if (_cajetanConfig.NonInteractiveSpecificLists is not null && _cajetanConfig.NonInteractiveSpecificLists.Length > 0)
usersToDownload = await GetUsersFromSpecificListsAsync(allUsersAndLists, _cajetanConfig.NonInteractiveSpecificLists);
else if (_cajetanConfig.NonInteractiveSpecificUsers is not null && _cajetanConfig.NonInteractiveSpecificUsers.Length > 0)
usersToDownload = GetUsersFromSpecificUsernames(allUsersAndLists, [.. _cajetanConfig.NonInteractiveSpecificUsers]);
if (usersToDownload.Count == 0)
return;
int userNum = 0;
int userCount = usersToDownload.Count;
CajetanDownloadEventHandler eventHandler = new();
CreatorDownloadResult totalResults = new();
LoggerWithConfigContext(_configService.CurrentConfig, _cajetanConfig)
.Information("Scraping Data for {UserCount} user(s)", usersToDownload.Count);
eventHandler.OnMessage($"Scraping Data for {usersToDownload.Count} user(s)\n");
foreach ((string username, long userId) in usersToDownload)
{
try
{
DateTime userStartTime = DateTime.Now;
Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", username, ++userNum, userCount);
eventHandler.OnMessage($"\nScraping Data for {Markup.Escape(username)} ({userNum} of {userCount})\n");
string path = _orchestrationService.ResolveDownloadPath(username);
Log.Debug($"Download path: {path}");
CreatorDownloadResult results = await _orchestrationService.DownloadCreatorContentAsync(
username: username,
userId: userId,
path: path,
users: usersToDownload,
clientIdBlobMissing: _clientIdBlobMissing,
devicePrivateKeyMissing: _devicePrivateKeyMissing,
eventHandler: eventHandler
);
CreatorDownloadResult newResults = results.NewDownloads ?? results;
totalResults.Add(newResults);
DateTime userEndTime = DateTime.Now;
TimeSpan userTotalTime = userEndTime - userStartTime;
Log.ForContext("Paid Posts", newResults.PaidPostCount)
.ForContext("Posts", newResults.PostCount)
.ForContext("Archived", newResults.ArchivedCount)
.ForContext("Streams", newResults.StreamsCount)
.ForContext("Stories", newResults.StoriesCount)
.ForContext("Highlights", newResults.HighlightsCount)
.ForContext("Messages", newResults.MessagesCount)
.ForContext("Paid Messages", newResults.PaidMessagesCount)
.ForContext("Username", username)
.ForContext("TotalMinutes", userTotalTime.TotalMinutes)
.Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes");
}
catch (Exception ex)
{
Log.ForContext("Username", username)
.ForContext("UserNum", userNum)
.ForContext("UserCount", userCount)
.ForContext("ExceptionMessage", ex.Message)
.Error(ex, "Scrape for '{Username:l}' ({UserNum} of {UserCount}) failed! {ExceptionMessage:l}");
}
}
DateTime endTime = DateTime.Now;
TimeSpan totalTime = endTime - startTime;
eventHandler.OnScrapeComplete(totalTime);
Log.ForContext("Paid Posts", totalResults.PaidPostCount)
.ForContext("Posts", totalResults.PostCount)
.ForContext("Archived", totalResults.ArchivedCount)
.ForContext("Streams", totalResults.StreamsCount)
.ForContext("Stories", totalResults.StoriesCount)
.ForContext("Highlights", totalResults.HighlightsCount)
.ForContext("Messages", totalResults.MessagesCount)
.ForContext("Paid Messages", totalResults.PaidMessagesCount)
.ForContext("TotalMinutes", totalTime.TotalMinutes)
.Information("Scrape Completed in {TotalMinutes:0.00} minutes");
await Task.Delay(2000);
}
private async Task OutputBlockedUsersAsync()
{
const string OUTPUT_FILE_BLOCKED = "blocked-users.json";
const string OUTPUT_FILE_EXPIRED = "expired-users.json";
await GetUsersAsync("Blocked", "/users/blocked", OUTPUT_FILE_BLOCKED);
await GetUsersAsync("Expired", "/subscriptions/subscribes", OUTPUT_FILE_EXPIRED, typeParam: "expired", offsetByCount: false);
async Task GetUsersAsync(string typeDisplay, string uri, string outputFile, string? typeParam = null, bool offsetByCount = true)
{
Dictionary<string, long> users = await _apiService.GetUsersWithProgressAsync(typeDisplay, uri, typeParam, offsetByCount);
Console.WriteLine();
if (users is null || users.Count == 0)
{
AnsiConsole.Markup($"[green]No {typeDisplay} Users found.\n[/]");
}
else
{
AnsiConsole.Markup($"[green]Found {users.Count} {typeDisplay} Users, saving to '{outputFile}'\n[/]");
string json = JsonConvert.SerializeObject(users, Formatting.Indented);
await File.WriteAllTextAsync(outputFile, json);
}
}
}
private async Task UpdateUserInfoAsync()
{
await _dbService.CreateUsersDb([]);
await _dbService.InitializeUserInfoTablesAsync();
Dictionary<string, long> users = await _dbService.GetUsersAsync();
Console.WriteLine();
Log.Information("Updating User Info for '{UserCount}' users", users.Count);
AnsiConsole.Markup($"[green]Updating User Info for '{users.Count}' users\n[/]");
await AnsiConsole.Progress()
.Columns(new ProgressBarColumn(), new PercentageColumn(), new TaskDescriptionColumn { Alignment = Justify.Left })
.StartAsync(RunUpdateAsync);
async Task RunUpdateAsync(ProgressContext context)
{
ProgressTask? updateTask = null;
int maxUsernameLength = users.Keys.Max(s => s.Length);
foreach ((string username, long userId) in users)
{
string description = $"Updating '{username}'".PadRight(11 + maxUsernameLength);
double prevValue = updateTask?.Value ?? 0;
updateTask = context.AddTask(description, true, users.Count);
updateTask.Value = prevValue;
using (LogContext.PushProperty("Username", username))
using (LogContext.PushProperty("UserId", userId))
using (LogContext.PushProperty("UserNum", prevValue + 1))
using (LogContext.PushProperty("UserTotal", users.Count))
{
try
{
Log.Information("[{UserNum:0} of {UserTotal}] Updating User Info for for: {Username:l}");
UserEntities.UserInfo? userInfo = await _apiService.GetDetailedUserInfoAsync($"/users/{username}");
await _dbService.UpdateUserInfoAsync(userInfo);
updateTask.Description = $"{description} - COMPLETE";
}
catch (Exception ex)
{
Log.Warning(ex, "[{UserNum:0} of {UserTotal}] Failed to update User Info for: {Username:l}");
AnsiConsole.Markup($"[red]Failed to update User Info for '{username}'\n[/]");
updateTask.Description = $"{description} - FAILED: {ex.Message}";
}
finally
{
updateTask.Increment(1);
updateTask.StopTask();
}
}
}
}
}
private async Task<Dictionary<string, long>> GetUsersFromSpecificListsAsync(UserListResult allUsersAndLists, string[] listNames)
{
Config currentConfig = _configService.CurrentConfig;
Dictionary<string, long> usersFromLists = new(StringComparer.OrdinalIgnoreCase);
foreach (string name in listNames)
{
if (!allUsersAndLists.Lists.TryGetValue(name, out long listId))
continue;
Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", name, currentConfig.IncludeRestrictedSubscriptions);
AnsiConsole.Markup($"[green]Getting Users from list '{name}' (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
List<string> listUsernames = await _apiService.GetListUsers($"/lists/{listId}/users") ?? [];
foreach (string u in listUsernames)
{
if (usersFromLists.ContainsKey(u))
continue;
if (!allUsersAndLists.Users.TryGetValue(u, out long userId))
continue;
usersFromLists[u] = userId;
}
}
return usersFromLists;
}
private static Dictionary<string, long> GetUsersFromSpecificUsernames(UserListResult allUsersAndLists, HashSet<string> usernames)
{
Dictionary<string, long> filteredUsers = allUsersAndLists.Users
.Where(u => usernames.Contains(u.Key))
.ToDictionary(u => u.Key, u => u.Value);
return filteredUsers;
}
private async Task<UserListResult> GetAvailableUsersAsync()
{
Config currentConfig = _configService.CurrentConfig;
UserListResult result = new()
{
Users = new(StringComparer.OrdinalIgnoreCase)
};
await FetchUsersAsync();
await FetchListsAsync();
AnsiConsole.WriteLine();
await _dbService.CreateUsersDb(result.Users);
await _dbService.InitializeUserInfoTablesAsync();
return result;
async Task FetchUsersAsync()
{
Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
Dictionary<string, long>? activeSubs = await _apiService.GetActiveSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
AddToResult(activeSubs);
if (currentConfig.IncludeExpiredSubscriptions)
{
Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
Dictionary<string, long>? expiredSubs = await _apiService.GetExpiredSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
AddToResult(expiredSubs);
}
}
async Task FetchListsAsync()
{
Log.Information("Getting Lists");
AnsiConsole.Markup($"[green]Getting Lists\n[/]");
result.Lists = await _apiService.GetLists("/lists") ?? [];
}
void AddToResult(Dictionary<string, long>? subscriptions)
{
foreach ((string username, long userId) in subscriptions ?? [])
{
if (result.Users.TryAdd(username, userId))
Log.Debug($"Name: {username} ID: {userId}");
}
}
}
static ILogger LoggerWithConfigContext(Config config, CajetanConfig cajetanConfig)
=> Log.Logger
.ForContext(nameof(Config.DownloadPath), config.DownloadPath)
.ForContext(nameof(Config.DownloadPosts), config.DownloadPosts)
.ForContext(nameof(Config.DownloadPaidPosts), config.DownloadPaidPosts)
.ForContext(nameof(Config.DownloadMessages), config.DownloadMessages)
.ForContext(nameof(Config.DownloadPaidMessages), config.DownloadPaidMessages)
.ForContext(nameof(Config.DownloadStories), config.DownloadStories)
.ForContext(nameof(Config.DownloadStreams), config.DownloadStreams)
.ForContext(nameof(Config.DownloadHighlights), config.DownloadHighlights)
.ForContext(nameof(Config.DownloadArchived), config.DownloadArchived)
.ForContext(nameof(Config.DownloadAvatarHeaderPhoto), config.DownloadAvatarHeaderPhoto)
.ForContext(nameof(Config.DownloadImages), config.DownloadImages)
.ForContext(nameof(Config.DownloadVideos), config.DownloadVideos)
.ForContext(nameof(Config.DownloadAudios), config.DownloadAudios)
.ForContext(nameof(Config.IgnoreOwnMessages), config.IgnoreOwnMessages)
.ForContext(nameof(Config.DownloadPostsIncrementally), config.DownloadPostsIncrementally)
.ForContext(nameof(Config.BypassContentForCreatorsWhoNoLongerExist), config.BypassContentForCreatorsWhoNoLongerExist)
.ForContext(nameof(Config.SkipAds), config.SkipAds)
.ForContext(nameof(Config.IncludeExpiredSubscriptions), config.IncludeExpiredSubscriptions)
.ForContext(nameof(Config.IncludeRestrictedSubscriptions), config.IncludeRestrictedSubscriptions)
.ForContext(nameof(CajetanConfig.NonInteractiveSpecificLists), cajetanConfig.NonInteractiveSpecificLists)
.ForContext(nameof(CajetanConfig.NonInteractiveSpecificUsers), cajetanConfig.NonInteractiveSpecificUsers);
}

View File

@ -17,6 +17,8 @@ public class CreatorDownloadResult
public int MessagesCount { get; set; }
public int PaidMessagesCount { get; set; }
public CreatorDownloadResult? NewDownloads { get; set; }
}
public class UserListResult

View File

@ -0,0 +1,36 @@
namespace OF_DL.Models.Entities.Users;
public class UserInfo : User
{
public long? Id { get; set; }
public string? SubscribePrice { get; set; }
public string? CurrentSubscribePrice { get; set; }
public bool? IsPaywallRequired { get; set; }
public bool? IsActive { get; set; }
public bool? IsRestricted { get; set; }
public bool? SubscribedBy { get; set; }
public bool? SubscribedByExpire { get; set; }
public DateTimeOffset? SubscribedByExpireDate { get; set; }
public bool? SubscribedByAutoprolong { get; set; }
public bool? IsPendingAutoprolong { get; set; }
public bool? SubscribedIsExpiredNow { get; set; }
public bool? SubscribedOn { get; set; }
public bool? SubscribedOnExpiredNow { get; set; }
public string? SubscribedOnDuration { get; set; }
public string? About { get; set; }
public int? PostsCount { get; set; }
public int? ArchivedPostsCount { get; set; }
public int? PrivateArchivedPostsCount { get; set; }
public int? PhotosCount { get; set; }
public int? VideosCount { get; set; }
public int? AudiosCount { get; set; }
public int? MediasCount { get; set; }
}

View File

@ -23,4 +23,8 @@
<PackageReference Include="xFFmpeg.NET" Version="7.2.0"/>
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="Cajetan.OF-DL" />
</ItemGroup>
</Project>

View File

@ -43,7 +43,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
{
private const int MaxAttempts = 30;
private const int DelayBetweenAttempts = 3000;
private static readonly JsonSerializerSettings s_mJsonSerializerSettings;
protected static readonly JsonSerializerSettings s_mJsonSerializerSettings;
private static DateTime? s_cachedDynamicRulesExpiration;
private static DynamicRules? s_cachedDynamicRules;
@ -147,7 +147,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
return headers;
}
private bool HasSignedRequestAuth()
protected bool HasSignedRequestAuth()
{
Auth? currentAuth = authService.CurrentAuth;
return currentAuth is { UserId: not null, Cookie: not null, UserAgent: not null, XBc: not null };
@ -2772,12 +2772,12 @@ public class ApiService(IAuthService authService, IConfigService configService,
}
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint,
HttpClient client)
protected async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint,
HttpClient client, HttpMethod? method = null)
{
Log.Debug("Calling BuildHeaderAndExecuteRequests");
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
using HttpResponseMessage response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync();
@ -2788,16 +2788,19 @@ public class ApiService(IAuthService authService, IConfigService configService,
}
private Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
string endpoint)
protected Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
string endpoint, HttpMethod? method = null)
{
Log.Debug("Calling BuildHttpRequestMessage");
string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
string queryParams = "";
if (getParams.Count != 0)
queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.ApiUrl}{endpoint}{queryParams}");
HttpRequestMessage request = new(method ?? HttpMethod.Get, $"{Constants.ApiUrl}{endpoint}{queryParams}");
Log.Debug($"Full request URL: {Constants.ApiUrl}{endpoint}{queryParams}");
@ -2821,7 +2824,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
private static bool IsStringOnlyDigits(string input) => input.All(char.IsDigit);
private HttpClient GetHttpClient()
protected HttpClient GetHttpClient()
{
HttpClient client = new();
if (configService.CurrentConfig.Timeout is > 0)
@ -2832,7 +2835,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
return client;
}
private static T? DeserializeJson<T>(string? body, JsonSerializerSettings? settings = null)
protected static T? DeserializeJson<T>(string? body, JsonSerializerSettings? settings = null)
{
if (string.IsNullOrWhiteSpace(body))
{

View File

@ -160,6 +160,7 @@ public class DownloadOrchestrationService(
{
Config config = configService.CurrentConfig;
CreatorDownloadResult counts = new();
CreatorDownloadResult newCounts = new();
eventHandler.OnUserStarting(username);
Log.Debug($"Scraping Data for {username}");
@ -185,7 +186,8 @@ public class DownloadOrchestrationService(
posts => posts.PaidPosts.Values.ToList(),
async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
eventHandler);
eventHandler,
n => newCounts.PaidPostCount = n);
}
if (config.DownloadPosts)
@ -202,7 +204,8 @@ public class DownloadOrchestrationService(
posts => posts.Posts.Values.ToList(),
async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
eventHandler);
eventHandler,
n => newCounts.PostCount = n);
}
if (config.DownloadArchived)
@ -215,7 +218,8 @@ public class DownloadOrchestrationService(
archived => archived.ArchivedPosts.Values.ToList(),
async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter),
eventHandler);
eventHandler,
n => newCounts.ArchivedCount = n);
}
if (config.DownloadStreams)
@ -228,7 +232,8 @@ public class DownloadOrchestrationService(
streams => streams.Streams.Values.ToList(),
async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter),
eventHandler);
eventHandler,
n => newCounts.StreamsCount = n);
}
if (config.DownloadStories)
@ -252,6 +257,7 @@ public class DownloadOrchestrationService(
eventHandler.OnDownloadComplete("Stories", result);
counts.StoriesCount = result.TotalCount;
newCounts.StoriesCount = result.NewDownloads;
}
else
{
@ -280,6 +286,7 @@ public class DownloadOrchestrationService(
eventHandler.OnDownloadComplete("Highlights", result);
counts.HighlightsCount = result.TotalCount;
newCounts.HighlightsCount = result.NewDownloads;
}
else
{
@ -297,7 +304,8 @@ public class DownloadOrchestrationService(
messages => messages.Messages.Values.ToList(),
async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter),
eventHandler);
eventHandler,
n => newCounts.MessagesCount = n);
}
if (config.DownloadPaidMessages)
@ -310,10 +318,13 @@ public class DownloadOrchestrationService(
paidMessages => paidMessages.PaidMessages.Values.ToList(),
async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users,
clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter),
eventHandler);
eventHandler,
n => newCounts.PaidMessagesCount = n);
}
eventHandler.OnUserComplete(username, counts);
counts.NewDownloads = newCounts;
return counts;
}
@ -612,7 +623,8 @@ public class DownloadOrchestrationService(
Func<T, int> getObjectCount,
Func<T, List<string>?> getUrls,
Func<T, IProgressReporter, Task<DownloadResult>> downloadData,
IDownloadEventHandler eventHandler)
IDownloadEventHandler eventHandler,
Action<int>? newPostAssignmentAction)
{
T data = await eventHandler.WithStatusAsync($"Getting {contentType}",
async statusReporter => await fetchData(statusReporter));
@ -643,6 +655,8 @@ public class DownloadOrchestrationService(
Log.Debug(
$"{contentType} Already Downloaded: {result.ExistingDownloads} New {contentType} Downloaded: {result.NewDownloads}");
newPostAssignmentAction?.Invoke(result.NewDownloads);
return result.TotalCount;
}
}

View File

@ -51,4 +51,8 @@
</None>
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="Cajetan.OF-DL" />
</ItemGroup>
</Project>