forked from sim0n00ps/OF-DL
352 lines
15 KiB
C#
352 lines
15 KiB
C#
using Microsoft.Extensions.DependencyInjection;
|
|
using OF_DL.Models.Downloads;
|
|
using OF_DL.Models.Entities.Users;
|
|
|
|
namespace OF_DL;
|
|
|
|
internal class Worker(IServiceProvider serviceProvider)
|
|
{
|
|
private readonly IConfigService _configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
private readonly IAuthService _authService = serviceProvider.GetRequiredService<IAuthService>();
|
|
private readonly IStartupService _startupService = serviceProvider.GetRequiredService<IStartupService>();
|
|
private readonly IDownloadOrchestrationService _orchestrationService = serviceProvider.GetRequiredService<IDownloadOrchestrationService>();
|
|
private readonly ICajetanDbService _dbService = serviceProvider.GetRequiredService<ICajetanDbService>();
|
|
private readonly ICajetanApiService _apiService = serviceProvider.GetRequiredService<ICajetanApiService>();
|
|
private readonly ExitHelper _exitHelper = serviceProvider.GetRequiredService<ExitHelper>();
|
|
|
|
private readonly CajetanConfig _cajetanConfig = serviceProvider.GetRequiredService<CajetanConfig>();
|
|
private bool _clientIdBlobMissing = true;
|
|
private bool _devicePrivateKeyMissing = true;
|
|
|
|
public async Task RunAsync()
|
|
{
|
|
try
|
|
{
|
|
await InitializeAsync();
|
|
|
|
Task tMode = _cajetanConfig.Mode switch
|
|
{
|
|
EMode.DownloadCreatorContent => DownloadCreatorContentAsync(),
|
|
EMode.OutputBlockedUsers => OutputBlockedUsersAsync(),
|
|
EMode.UpdateAllUserInfo => UpdateUserInfoAsync(),
|
|
_ => Task.CompletedTask
|
|
};
|
|
|
|
await tMode;
|
|
}
|
|
catch (ExitCodeException ex)
|
|
{
|
|
_exitHelper.ExitWithCode(ex.ExitCode);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Fatal(ex, "Unhandled Exception! {ExceptionMessage:l}", ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
_exitHelper.ExitWithCode(0);
|
|
}
|
|
}
|
|
|
|
private async Task InitializeAsync()
|
|
{
|
|
StartupResult startupResult = await _startupService.ValidateEnvironmentAsync();
|
|
|
|
if (startupResult.IsWindowsVersionValid is false)
|
|
throw new ExitCodeException(1);
|
|
|
|
if (startupResult.FfmpegFound is false)
|
|
throw new ExitCodeException(4);
|
|
|
|
if (startupResult.ClientIdBlobMissing || startupResult.DevicePrivateKeyMissing)
|
|
AnsiConsole.Markup("[yellow]device_client_id_blob and/or device_private_key missing, https://ofdl.tools/ or https://cdrm-project.com/ will be used instead for DRM protected videos\n[/]");
|
|
|
|
_clientIdBlobMissing = startupResult.ClientIdBlobMissing;
|
|
_devicePrivateKeyMissing = startupResult.DevicePrivateKeyMissing;
|
|
|
|
if (IsRulesJsonValid() is false)
|
|
{
|
|
AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]");
|
|
Console.ReadKey();
|
|
throw new ExitCodeException(2);
|
|
}
|
|
|
|
if (await IsAuthorizedAsync() is false)
|
|
{
|
|
AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]");
|
|
Console.ReadKey();
|
|
throw new ExitCodeException(2);
|
|
}
|
|
|
|
bool IsRulesJsonValid()
|
|
{
|
|
if (startupResult.RulesJsonExists is false)
|
|
return true;
|
|
|
|
if (startupResult.RulesJsonValid)
|
|
{
|
|
AnsiConsole.Markup("[green]rules.json located successfully!\n[/]");
|
|
return true;
|
|
}
|
|
|
|
AnsiConsole.MarkupLine("\n[red]rules.json is not valid, check your JSON syntax![/]\n");
|
|
Log.Error("processing of rules.json failed: {Error:l}", startupResult.RulesJsonError);
|
|
return false;
|
|
}
|
|
|
|
async Task<bool> IsAuthorizedAsync()
|
|
{
|
|
if (await _authService.LoadFromFileAsync() is false)
|
|
{
|
|
if (File.Exists("auth.json"))
|
|
{
|
|
Log.Error("Auth file was found but could not be deserialized!");
|
|
AnsiConsole.MarkupLine("\n[red]auth.json could not be deserialized.[/]");
|
|
return false;
|
|
}
|
|
|
|
Log.Error("Auth file was not found!");
|
|
AnsiConsole.MarkupLine("\n[red]auth.json is missing.");
|
|
return false;
|
|
}
|
|
|
|
AnsiConsole.Markup("[green]auth.json located successfully!\n[/]");
|
|
|
|
// Validate cookie string
|
|
_authService.ValidateCookieString();
|
|
|
|
User? user = await _authService.ValidateAuthAsync();
|
|
if (user is null || (user.Name is null && user.Username is null))
|
|
{
|
|
Log.Error("Auth failed");
|
|
_authService.CurrentAuth = null;
|
|
AnsiConsole.MarkupLine("\n[red]Auth failed. Please try again or use other authentication methods detailed here:[/]\n");
|
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth[/]\n");
|
|
return false;
|
|
}
|
|
|
|
string displayName = $"{user.Name} {user.Username}".Trim();
|
|
AnsiConsole.MarkupLine($"[green]Logged In successfully as {displayName}\n[/]");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
private async Task DownloadCreatorContentAsync()
|
|
{
|
|
DateTime startTime = DateTime.Now;
|
|
|
|
UserListResult allUsersAndLists = await GetAvailableUsersAsync();
|
|
Dictionary<string, long> usersToDownload = [];
|
|
|
|
if (_cajetanConfig.NonInteractiveSpecificLists is not null && _cajetanConfig.NonInteractiveSpecificLists.Length > 0)
|
|
usersToDownload = await GetUsersFromSpecificListsAsync(allUsersAndLists, _cajetanConfig.NonInteractiveSpecificLists);
|
|
|
|
else if (_cajetanConfig.NonInteractiveSpecificUsers is not null && _cajetanConfig.NonInteractiveSpecificUsers.Length > 0)
|
|
usersToDownload = GetUsersFromSpecificUsernames(allUsersAndLists, [.. _cajetanConfig.NonInteractiveSpecificUsers]);
|
|
|
|
if (usersToDownload.Count == 0)
|
|
return;
|
|
|
|
int userNum = 0;
|
|
int userCount = usersToDownload.Count;
|
|
CajetanDownloadEventHandler eventHandler = new();
|
|
CreatorDownloadResult totalResults = new();
|
|
|
|
LoggerWithConfigContext(_configService.CurrentConfig, _cajetanConfig)
|
|
.Information("Scraping Data for {UserCount} user(s)", usersToDownload.Count);
|
|
eventHandler.OnMessage($"Scraping Data for {usersToDownload.Count} user(s)\n");
|
|
|
|
foreach ((string username, long userId) in usersToDownload)
|
|
{
|
|
try
|
|
{
|
|
DateTime userStartTime = DateTime.Now;
|
|
|
|
Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", username, ++userNum, userCount);
|
|
eventHandler.OnMessage($"\nScraping Data for {Markup.Escape(username)} ({userNum} of {userCount})\n");
|
|
|
|
string path = _orchestrationService.ResolveDownloadPath(username);
|
|
Log.Debug($"Download path: {path}");
|
|
|
|
CreatorDownloadResult results = await _orchestrationService.DownloadCreatorContentAsync(
|
|
username: username,
|
|
userId: userId,
|
|
path: path,
|
|
users: usersToDownload,
|
|
clientIdBlobMissing: _clientIdBlobMissing,
|
|
devicePrivateKeyMissing: _devicePrivateKeyMissing,
|
|
eventHandler: eventHandler
|
|
);
|
|
|
|
totalResults.Add(results);
|
|
|
|
DateTime userEndTime = DateTime.Now;
|
|
TimeSpan userTotalTime = userEndTime - userStartTime;
|
|
|
|
Log.ForContext("Paid Posts", results.PaidPostCount)
|
|
.ForContext("Posts", results.PostCount)
|
|
.ForContext("Archived", results.ArchivedCount)
|
|
.ForContext("Streams", results.StreamsCount)
|
|
.ForContext("Stories", results.StoriesCount)
|
|
.ForContext("Highlights", results.HighlightsCount)
|
|
.ForContext("Messages", results.MessagesCount)
|
|
.ForContext("Paid Messages", results.PaidMessagesCount)
|
|
.ForContext("Username", username)
|
|
.ForContext("TotalMinutes", userTotalTime.TotalMinutes)
|
|
.Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext("Username", username)
|
|
.ForContext("UserNum", userNum)
|
|
.ForContext("UserCount", userCount)
|
|
.ForContext("ExceptionMessage", ex.Message)
|
|
.Error(ex, "Scrape for '{Username:l}' ({UserNum} of {UserCount}) failed! {ExceptionMessage:l}");
|
|
}
|
|
}
|
|
|
|
DateTime endTime = DateTime.Now;
|
|
TimeSpan totalTime = endTime - startTime;
|
|
|
|
eventHandler.OnScrapeComplete(totalTime);
|
|
|
|
Log.ForContext("Paid Posts", totalResults.PaidPostCount)
|
|
.ForContext("Posts", totalResults.PostCount)
|
|
.ForContext("Archived", totalResults.ArchivedCount)
|
|
.ForContext("Streams", totalResults.StreamsCount)
|
|
.ForContext("Stories", totalResults.StoriesCount)
|
|
.ForContext("Highlights", totalResults.HighlightsCount)
|
|
.ForContext("Messages", totalResults.MessagesCount)
|
|
.ForContext("Paid Messages", totalResults.PaidMessagesCount)
|
|
.ForContext("TotalMinutes", totalTime.TotalMinutes)
|
|
.Information("Scrape Completed in {TotalMinutes:0.00} minutes");
|
|
}
|
|
|
|
private async Task OutputBlockedUsersAsync()
|
|
{
|
|
|
|
}
|
|
|
|
private async Task UpdateUserInfoAsync()
|
|
{
|
|
|
|
}
|
|
|
|
private async Task<Dictionary<string, long>> GetUsersFromSpecificListsAsync(UserListResult allUsersAndLists, string[] listNames)
|
|
{
|
|
Config currentConfig = _configService.CurrentConfig;
|
|
Dictionary<string, long> usersFromLists = new(StringComparer.OrdinalIgnoreCase);
|
|
|
|
foreach (string name in listNames)
|
|
{
|
|
if (!allUsersAndLists.Lists.TryGetValue(name, out long listId))
|
|
continue;
|
|
|
|
Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", name, currentConfig.IncludeRestrictedSubscriptions);
|
|
AnsiConsole.Markup($"[green]Getting Users from list '{name}' (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
|
|
|
|
List<string> listUsernames = await _apiService.GetListUsers($"/lists/{listId}/users") ?? [];
|
|
|
|
foreach (string u in listUsernames)
|
|
{
|
|
if (usersFromLists.ContainsKey(u))
|
|
continue;
|
|
|
|
if (!allUsersAndLists.Users.TryGetValue(u, out long userId))
|
|
continue;
|
|
|
|
usersFromLists[u] = userId;
|
|
}
|
|
}
|
|
|
|
return usersFromLists;
|
|
}
|
|
|
|
private static Dictionary<string, long> GetUsersFromSpecificUsernames(UserListResult allUsersAndLists, HashSet<string> usernames)
|
|
{
|
|
Dictionary<string, long> filteredUsers = allUsersAndLists.Users
|
|
.Where(u => usernames.Contains(u.Key))
|
|
.ToDictionary(u => u.Key, u => u.Value);
|
|
|
|
return filteredUsers;
|
|
}
|
|
|
|
private async Task<UserListResult> GetAvailableUsersAsync()
|
|
{
|
|
Config currentConfig = _configService.CurrentConfig;
|
|
UserListResult result = new()
|
|
{
|
|
Users = new(StringComparer.OrdinalIgnoreCase)
|
|
};
|
|
|
|
await FetchUsersAsync();
|
|
await FetchListsAsync();
|
|
|
|
AnsiConsole.WriteLine();
|
|
|
|
await _dbService.CreateUsersDb(result.Users);
|
|
await _dbService.InitializeUserInfoTablesAsync();
|
|
|
|
return result;
|
|
|
|
async Task FetchUsersAsync()
|
|
{
|
|
Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
|
|
AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
|
|
|
|
Dictionary<string, long>? activeSubs = await _apiService.GetActiveSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
|
|
AddToResult(activeSubs);
|
|
|
|
if (currentConfig.IncludeExpiredSubscriptions)
|
|
{
|
|
Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
|
|
AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
|
|
|
|
Dictionary<string, long>? expiredSubs = await _apiService.GetExpiredSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
|
|
AddToResult(expiredSubs);
|
|
}
|
|
}
|
|
|
|
async Task FetchListsAsync()
|
|
{
|
|
Log.Information("Getting Lists");
|
|
AnsiConsole.Markup($"[green]Getting Lists\n[/]");
|
|
|
|
result.Lists = await _apiService.GetLists("/lists") ?? [];
|
|
}
|
|
|
|
void AddToResult(Dictionary<string, long>? subscriptions)
|
|
{
|
|
foreach ((string username, long userId) in subscriptions ?? [])
|
|
{
|
|
if (result.Users.TryAdd(username, userId))
|
|
Log.Debug($"Name: {username} ID: {userId}");
|
|
}
|
|
}
|
|
}
|
|
|
|
static ILogger LoggerWithConfigContext(Config config, CajetanConfig cajetanConfig)
|
|
=> Log.Logger
|
|
.ForContext(nameof(Config.DownloadPath), config.DownloadPath)
|
|
.ForContext(nameof(Config.DownloadPosts), config.DownloadPosts)
|
|
.ForContext(nameof(Config.DownloadPaidPosts), config.DownloadPaidPosts)
|
|
.ForContext(nameof(Config.DownloadMessages), config.DownloadMessages)
|
|
.ForContext(nameof(Config.DownloadPaidMessages), config.DownloadPaidMessages)
|
|
.ForContext(nameof(Config.DownloadStories), config.DownloadStories)
|
|
.ForContext(nameof(Config.DownloadStreams), config.DownloadStreams)
|
|
.ForContext(nameof(Config.DownloadHighlights), config.DownloadHighlights)
|
|
.ForContext(nameof(Config.DownloadArchived), config.DownloadArchived)
|
|
.ForContext(nameof(Config.DownloadAvatarHeaderPhoto), config.DownloadAvatarHeaderPhoto)
|
|
.ForContext(nameof(Config.DownloadImages), config.DownloadImages)
|
|
.ForContext(nameof(Config.DownloadVideos), config.DownloadVideos)
|
|
.ForContext(nameof(Config.DownloadAudios), config.DownloadAudios)
|
|
.ForContext(nameof(Config.IgnoreOwnMessages), config.IgnoreOwnMessages)
|
|
.ForContext(nameof(Config.DownloadPostsIncrementally), config.DownloadPostsIncrementally)
|
|
.ForContext(nameof(Config.BypassContentForCreatorsWhoNoLongerExist), config.BypassContentForCreatorsWhoNoLongerExist)
|
|
.ForContext(nameof(Config.SkipAds), config.SkipAds)
|
|
.ForContext(nameof(Config.IncludeExpiredSubscriptions), config.IncludeExpiredSubscriptions)
|
|
.ForContext(nameof(Config.IncludeRestrictedSubscriptions), config.IncludeRestrictedSubscriptions)
|
|
.ForContext(nameof(CajetanConfig.NonInteractiveSpecificLists), cajetanConfig.NonInteractiveSpecificLists)
|
|
.ForContext(nameof(CajetanConfig.NonInteractiveSpecificUsers), cajetanConfig.NonInteractiveSpecificUsers);
|
|
}
|