using Microsoft.Extensions.DependencyInjection; namespace OF_DL; internal class Worker(IServiceProvider serviceProvider) { private readonly IConfigService _configService = serviceProvider.GetRequiredService(); private readonly IAuthService _authService = serviceProvider.GetRequiredService(); private readonly IStartupService _startupService = serviceProvider.GetRequiredService(); private readonly IDownloadOrchestrationService _orchestrationService = serviceProvider.GetRequiredService(); private readonly ICajetanDbService _dbService = serviceProvider.GetRequiredService(); private readonly ICajetanApiService _apiService = serviceProvider.GetRequiredService(); private readonly ExitHelper _exitHelper = serviceProvider.GetRequiredService(); private readonly CajetanConfig _cajetanConfig = serviceProvider.GetRequiredService(); private bool _clientIdBlobMissing = true; private bool _devicePrivateKeyMissing = true; public async Task RunAsync() { try { await InitializeAsync(); Task tMode = _cajetanConfig.Mode switch { EMode.DownloadCreatorContent => DownloadCreatorContentAsync(), EMode.OutputBlockedUsers => OutputBlockedUsersAsync(), EMode.UpdateAllUserInfo => UpdateUserInfoAsync(), _ => Task.CompletedTask }; await tMode; } catch (ExitCodeException ex) { _exitHelper.ExitWithCode(ex.ExitCode); } catch (Exception ex) { Log.Fatal(ex, "Unhandled Exception! {ExceptionMessage:l}", ex.Message); } finally { _exitHelper.ExitWithCode(0); } } private async Task InitializeAsync() { StartupResult startupResult = await _startupService.ValidateEnvironmentAsync(); if (startupResult.IsWindowsVersionValid is false) throw new ExitCodeException(1); if (startupResult.FfmpegFound is false) throw new ExitCodeException(4); if (startupResult.ClientIdBlobMissing || startupResult.DevicePrivateKeyMissing) AnsiConsole.Markup("[yellow]device_client_id_blob and/or device_private_key missing, https://ofdl.tools/ or https://cdrm-project.com/ will be used instead for DRM protected videos\n[/]"); _clientIdBlobMissing = startupResult.ClientIdBlobMissing; _devicePrivateKeyMissing = startupResult.DevicePrivateKeyMissing; if (IsRulesJsonValid() is false) { AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]"); Console.ReadKey(); throw new ExitCodeException(2); } if (await IsAuthorizedAsync() is false) { AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]"); Console.ReadKey(); throw new ExitCodeException(2); } bool IsRulesJsonValid() { if (startupResult.RulesJsonExists is false) return true; if (startupResult.RulesJsonValid) { AnsiConsole.Markup("[green]rules.json located successfully!\n[/]"); return true; } AnsiConsole.MarkupLine("\n[red]rules.json is not valid, check your JSON syntax![/]\n"); Log.Error("processing of rules.json failed: {Error:l}", startupResult.RulesJsonError); return false; } async Task IsAuthorizedAsync() { if (await _authService.LoadFromFileAsync() is false) { if (File.Exists("auth.json")) { Log.Error("Auth file was found but could not be deserialized!"); AnsiConsole.MarkupLine("\n[red]auth.json could not be deserialized.[/]"); return false; } Log.Error("Auth file was not found!"); AnsiConsole.MarkupLine("\n[red]auth.json is missing."); return false; } AnsiConsole.Markup("[green]auth.json located successfully!\n[/]"); // Validate cookie string _authService.ValidateCookieString(); UserEntities.User? user = await _authService.ValidateAuthAsync(); if (user is null || (user.Name is null && user.Username is null)) { Log.Error("Auth failed"); _authService.CurrentAuth = null; AnsiConsole.MarkupLine("\n[red]Auth failed. Please try again or use other authentication methods detailed here:[/]\n"); AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth[/]\n"); return false; } string displayName = $"{user.Name} {user.Username}".Trim(); AnsiConsole.MarkupLine($"[green]Logged In successfully as {displayName}\n[/]"); return true; } } private async Task DownloadCreatorContentAsync() { DateTime startTime = DateTime.Now; UserListResult allUsersAndLists = await GetAvailableUsersAsync(); Dictionary usersToDownload = []; if (_cajetanConfig.NonInteractiveSpecificLists is not null && _cajetanConfig.NonInteractiveSpecificLists.Length > 0) usersToDownload = await GetUsersFromSpecificListsAsync(allUsersAndLists, _cajetanConfig.NonInteractiveSpecificLists); else if (_cajetanConfig.NonInteractiveSpecificUsers is not null && _cajetanConfig.NonInteractiveSpecificUsers.Length > 0) usersToDownload = GetUsersFromSpecificUsernames(allUsersAndLists, [.. _cajetanConfig.NonInteractiveSpecificUsers]); if (usersToDownload.Count == 0) return; int userNum = 0; int userCount = usersToDownload.Count; CajetanDownloadEventHandler eventHandler = new(); CreatorDownloadResult totalResults = new(); LoggerWithConfigContext(_configService.CurrentConfig, _cajetanConfig) .Information("Scraping Data for {UserCount} user(s)", usersToDownload.Count); eventHandler.OnMessage($"Scraping Data for {usersToDownload.Count} user(s)\n"); foreach ((string username, long userId) in usersToDownload) { try { DateTime userStartTime = DateTime.Now; Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", username, ++userNum, userCount); eventHandler.OnMessage($"\nScraping Data for {Markup.Escape(username)} ({userNum} of {userCount})\n"); string path = _orchestrationService.ResolveDownloadPath(username); Log.Debug($"Download path: {path}"); CreatorDownloadResult results = await _orchestrationService.DownloadCreatorContentAsync( username: username, userId: userId, path: path, users: usersToDownload, clientIdBlobMissing: _clientIdBlobMissing, devicePrivateKeyMissing: _devicePrivateKeyMissing, eventHandler: eventHandler ); CreatorDownloadResult newResults = results.NewDownloads ?? results; totalResults.Add(newResults); DateTime userEndTime = DateTime.Now; TimeSpan userTotalTime = userEndTime - userStartTime; Log.ForContext("Paid Posts", newResults.PaidPostCount) .ForContext("Posts", newResults.PostCount) .ForContext("Archived", newResults.ArchivedCount) .ForContext("Streams", newResults.StreamsCount) .ForContext("Stories", newResults.StoriesCount) .ForContext("Highlights", newResults.HighlightsCount) .ForContext("Messages", newResults.MessagesCount) .ForContext("Paid Messages", newResults.PaidMessagesCount) .ForContext("Username", username) .ForContext("TotalMinutes", userTotalTime.TotalMinutes) .Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes"); } catch (Exception ex) { Log.ForContext("Username", username) .ForContext("UserNum", userNum) .ForContext("UserCount", userCount) .ForContext("ExceptionMessage", ex.Message) .Error(ex, "Scrape for '{Username:l}' ({UserNum} of {UserCount}) failed! {ExceptionMessage:l}"); } } DateTime endTime = DateTime.Now; TimeSpan totalTime = endTime - startTime; eventHandler.OnScrapeComplete(totalTime); Log.ForContext("Paid Posts", totalResults.PaidPostCount) .ForContext("Posts", totalResults.PostCount) .ForContext("Archived", totalResults.ArchivedCount) .ForContext("Streams", totalResults.StreamsCount) .ForContext("Stories", totalResults.StoriesCount) .ForContext("Highlights", totalResults.HighlightsCount) .ForContext("Messages", totalResults.MessagesCount) .ForContext("Paid Messages", totalResults.PaidMessagesCount) .ForContext("TotalMinutes", totalTime.TotalMinutes) .Information("Scrape Completed in {TotalMinutes:0.00} minutes"); await Task.Delay(2000); } private async Task OutputBlockedUsersAsync() { const string OUTPUT_FILE_BLOCKED = "blocked-users.json"; const string OUTPUT_FILE_EXPIRED = "expired-users.json"; await GetUsersAsync("Blocked", "/users/blocked", OUTPUT_FILE_BLOCKED); await GetUsersAsync("Expired", "/subscriptions/subscribes", OUTPUT_FILE_EXPIRED, typeParam: "expired", offsetByCount: false); async Task GetUsersAsync(string typeDisplay, string uri, string outputFile, string? typeParam = null, bool offsetByCount = true) { Dictionary users = await _apiService.GetUsersWithProgressAsync(typeDisplay, uri, typeParam, offsetByCount); Console.WriteLine(); if (users is null || users.Count == 0) { AnsiConsole.Markup($"[green]No {typeDisplay} Users found.\n[/]"); } else { AnsiConsole.Markup($"[green]Found {users.Count} {typeDisplay} Users, saving to '{outputFile}'\n[/]"); string json = JsonConvert.SerializeObject(users, Formatting.Indented); await File.WriteAllTextAsync(outputFile, json); } } } private async Task UpdateUserInfoAsync() { } private async Task> GetUsersFromSpecificListsAsync(UserListResult allUsersAndLists, string[] listNames) { Config currentConfig = _configService.CurrentConfig; Dictionary usersFromLists = new(StringComparer.OrdinalIgnoreCase); foreach (string name in listNames) { if (!allUsersAndLists.Lists.TryGetValue(name, out long listId)) continue; Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", name, currentConfig.IncludeRestrictedSubscriptions); AnsiConsole.Markup($"[green]Getting Users from list '{name}' (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]"); List listUsernames = await _apiService.GetListUsers($"/lists/{listId}/users") ?? []; foreach (string u in listUsernames) { if (usersFromLists.ContainsKey(u)) continue; if (!allUsersAndLists.Users.TryGetValue(u, out long userId)) continue; usersFromLists[u] = userId; } } return usersFromLists; } private static Dictionary GetUsersFromSpecificUsernames(UserListResult allUsersAndLists, HashSet usernames) { Dictionary filteredUsers = allUsersAndLists.Users .Where(u => usernames.Contains(u.Key)) .ToDictionary(u => u.Key, u => u.Value); return filteredUsers; } private async Task GetAvailableUsersAsync() { Config currentConfig = _configService.CurrentConfig; UserListResult result = new() { Users = new(StringComparer.OrdinalIgnoreCase) }; await FetchUsersAsync(); await FetchListsAsync(); AnsiConsole.WriteLine(); await _dbService.CreateUsersDb(result.Users); await _dbService.InitializeUserInfoTablesAsync(); return result; async Task FetchUsersAsync() { Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions); AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]"); Dictionary? activeSubs = await _apiService.GetActiveSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions); AddToResult(activeSubs); if (currentConfig.IncludeExpiredSubscriptions) { Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions); AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]"); Dictionary? expiredSubs = await _apiService.GetExpiredSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions); AddToResult(expiredSubs); } } async Task FetchListsAsync() { Log.Information("Getting Lists"); result.Lists = await _apiService.GetLists("/lists") ?? []; } void AddToResult(Dictionary? subscriptions) { foreach ((string username, long userId) in subscriptions ?? []) { if (result.Users.TryAdd(username, userId)) Log.Debug($"Name: {username} ID: {userId}"); } } } static ILogger LoggerWithConfigContext(Config config, CajetanConfig cajetanConfig) => Log.Logger .ForContext(nameof(Config.DownloadPath), config.DownloadPath) .ForContext(nameof(Config.DownloadPosts), config.DownloadPosts) .ForContext(nameof(Config.DownloadPaidPosts), config.DownloadPaidPosts) .ForContext(nameof(Config.DownloadMessages), config.DownloadMessages) .ForContext(nameof(Config.DownloadPaidMessages), config.DownloadPaidMessages) .ForContext(nameof(Config.DownloadStories), config.DownloadStories) .ForContext(nameof(Config.DownloadStreams), config.DownloadStreams) .ForContext(nameof(Config.DownloadHighlights), config.DownloadHighlights) .ForContext(nameof(Config.DownloadArchived), config.DownloadArchived) .ForContext(nameof(Config.DownloadAvatarHeaderPhoto), config.DownloadAvatarHeaderPhoto) .ForContext(nameof(Config.DownloadImages), config.DownloadImages) .ForContext(nameof(Config.DownloadVideos), config.DownloadVideos) .ForContext(nameof(Config.DownloadAudios), config.DownloadAudios) .ForContext(nameof(Config.IgnoreOwnMessages), config.IgnoreOwnMessages) .ForContext(nameof(Config.DownloadPostsIncrementally), config.DownloadPostsIncrementally) .ForContext(nameof(Config.BypassContentForCreatorsWhoNoLongerExist), config.BypassContentForCreatorsWhoNoLongerExist) .ForContext(nameof(Config.SkipAds), config.SkipAds) .ForContext(nameof(Config.IncludeExpiredSubscriptions), config.IncludeExpiredSubscriptions) .ForContext(nameof(Config.IncludeRestrictedSubscriptions), config.IncludeRestrictedSubscriptions) .ForContext(nameof(CajetanConfig.NonInteractiveSpecificLists), cajetanConfig.NonInteractiveSpecificLists) .ForContext(nameof(CajetanConfig.NonInteractiveSpecificUsers), cajetanConfig.NonInteractiveSpecificUsers); }