forked from sim0n00ps/OF-DL
453 lines
20 KiB
C#
453 lines
20 KiB
C#
using Microsoft.Extensions.DependencyInjection;
|
|
|
|
namespace OF_DL;
|
|
|
|
internal class Worker(IServiceProvider serviceProvider)
|
|
{
|
|
private readonly IConfigService _configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
private readonly IAuthService _authService = serviceProvider.GetRequiredService<IAuthService>();
|
|
private readonly IStartupService _startupService = serviceProvider.GetRequiredService<IStartupService>();
|
|
private readonly IDownloadOrchestrationService _orchestrationService = serviceProvider.GetRequiredService<IDownloadOrchestrationService>();
|
|
private readonly ICajetanDbService _dbService = serviceProvider.GetRequiredService<ICajetanDbService>();
|
|
private readonly ICajetanApiService _apiService = serviceProvider.GetRequiredService<ICajetanApiService>();
|
|
private readonly ExitHelper _exitHelper = serviceProvider.GetRequiredService<ExitHelper>();
|
|
|
|
private readonly CajetanConfig _cajetanConfig = serviceProvider.GetRequiredService<CajetanConfig>();
|
|
private bool _clientIdBlobMissing = true;
|
|
private bool _devicePrivateKeyMissing = true;
|
|
|
|
public async Task RunAsync()
|
|
{
|
|
try
|
|
{
|
|
await InitializeAsync();
|
|
|
|
Task tMode = _cajetanConfig.Mode switch
|
|
{
|
|
EMode.DownloadCreatorContent => DownloadCreatorContentAsync(),
|
|
EMode.OutputBlockedUsers => OutputBlockedUsersAsync(),
|
|
EMode.UpdateAllUserInfo => UpdateUserInfoAsync(),
|
|
_ => Task.CompletedTask
|
|
};
|
|
|
|
await tMode;
|
|
}
|
|
catch (ExitCodeException ex)
|
|
{
|
|
_exitHelper.ExitWithCode(ex.ExitCode);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Fatal(ex, "Unhandled Exception! {ExceptionMessage:l}", ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
_exitHelper.ExitWithCode(0);
|
|
}
|
|
}
|
|
|
|
private async Task InitializeAsync()
|
|
{
|
|
StartupResult startupResult = await _startupService.ValidateEnvironmentAsync();
|
|
|
|
if (startupResult.IsWindowsVersionValid is false)
|
|
throw new ExitCodeException(1);
|
|
|
|
if (startupResult.FfmpegFound is false)
|
|
throw new ExitCodeException(4);
|
|
|
|
if (startupResult.ClientIdBlobMissing || startupResult.DevicePrivateKeyMissing)
|
|
AnsiConsole.Markup("[yellow]device_client_id_blob and/or device_private_key missing, https://ofdl.tools/ or https://cdrm-project.com/ will be used instead for DRM protected videos\n[/]");
|
|
|
|
_clientIdBlobMissing = startupResult.ClientIdBlobMissing;
|
|
_devicePrivateKeyMissing = startupResult.DevicePrivateKeyMissing;
|
|
|
|
if (IsRulesJsonValid() is false)
|
|
{
|
|
AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]");
|
|
Console.ReadKey();
|
|
throw new ExitCodeException(2);
|
|
}
|
|
|
|
if (await IsAuthorizedAsync() is false)
|
|
{
|
|
AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]");
|
|
Console.ReadKey();
|
|
throw new ExitCodeException(2);
|
|
}
|
|
|
|
bool IsRulesJsonValid()
|
|
{
|
|
if (startupResult.RulesJsonExists is false)
|
|
return true;
|
|
|
|
if (startupResult.RulesJsonValid)
|
|
{
|
|
AnsiConsole.Markup("[green]rules.json located successfully!\n[/]");
|
|
return true;
|
|
}
|
|
|
|
AnsiConsole.MarkupLine("\n[red]rules.json is not valid, check your JSON syntax![/]\n");
|
|
Log.Error("processing of rules.json failed: {Error:l}", startupResult.RulesJsonError);
|
|
return false;
|
|
}
|
|
|
|
async Task<bool> IsAuthorizedAsync()
|
|
{
|
|
if (await _authService.LoadFromFileAsync() is false)
|
|
{
|
|
if (File.Exists("auth.json"))
|
|
{
|
|
Log.Error("Auth file was found but could not be deserialized!");
|
|
AnsiConsole.MarkupLine("\n[red]auth.json could not be deserialized.[/]");
|
|
return false;
|
|
}
|
|
|
|
Log.Error("Auth file was not found!");
|
|
AnsiConsole.MarkupLine("\n[red]auth.json is missing.");
|
|
return false;
|
|
}
|
|
|
|
AnsiConsole.Markup("[green]auth.json located successfully!\n[/]");
|
|
|
|
// Validate cookie string
|
|
_authService.ValidateCookieString();
|
|
|
|
UserEntities.User? user = await _authService.ValidateAuthAsync();
|
|
if (user is null || (user.Name is null && user.Username is null))
|
|
{
|
|
Log.Error("Auth failed");
|
|
_authService.CurrentAuth = null;
|
|
AnsiConsole.MarkupLine("\n[red]Auth failed. Please try again or use other authentication methods detailed here:[/]\n");
|
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth[/]\n");
|
|
return false;
|
|
}
|
|
|
|
string displayName = $"{user.Name} {user.Username}".Trim();
|
|
AnsiConsole.MarkupLine($"[green]Logged In successfully as {displayName}\n[/]");
|
|
return true;
|
|
}
|
|
}
|
|
|
|
private async Task DownloadCreatorContentAsync()
|
|
{
|
|
DateTime startTime = DateTime.Now;
|
|
|
|
UserListResult allUsersAndLists = await GetAvailableUsersAsync();
|
|
Dictionary<string, long> usersToDownload = [];
|
|
|
|
if (_cajetanConfig.NonInteractiveSpecificLists is not null && _cajetanConfig.NonInteractiveSpecificLists.Length > 0)
|
|
usersToDownload = await GetUsersFromSpecificListsAsync(allUsersAndLists, _cajetanConfig.NonInteractiveSpecificLists);
|
|
|
|
else if (_cajetanConfig.NonInteractiveSpecificUsers is not null && _cajetanConfig.NonInteractiveSpecificUsers.Length > 0)
|
|
usersToDownload = GetUsersFromSpecificUsernames(allUsersAndLists, [.. _cajetanConfig.NonInteractiveSpecificUsers]);
|
|
|
|
if (usersToDownload.Count == 0)
|
|
return;
|
|
|
|
int userNum = 0;
|
|
int userCount = usersToDownload.Count;
|
|
CajetanDownloadEventHandler eventHandler = new();
|
|
CreatorDownloadResult totalResults = new();
|
|
|
|
LoggerWithConfigContext(_configService.CurrentConfig, _cajetanConfig)
|
|
.Information("Scraping Data for {UserCount} user(s)", usersToDownload.Count);
|
|
eventHandler.OnMessage($"Scraping Data for {usersToDownload.Count} user(s)\n");
|
|
|
|
foreach ((string username, long userId) in usersToDownload)
|
|
{
|
|
try
|
|
{
|
|
DateTime userStartTime = DateTime.Now;
|
|
|
|
Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", username, ++userNum, userCount);
|
|
eventHandler.OnMessage($"\nScraping Data for {Markup.Escape(username)} ({userNum} of {userCount})\n");
|
|
|
|
string path = _orchestrationService.ResolveDownloadPath(username);
|
|
Log.Debug($"Download path: {path}");
|
|
|
|
CreatorDownloadResult results = await _orchestrationService.DownloadCreatorContentAsync(
|
|
username: username,
|
|
userId: userId,
|
|
path: path,
|
|
users: usersToDownload,
|
|
clientIdBlobMissing: _clientIdBlobMissing,
|
|
devicePrivateKeyMissing: _devicePrivateKeyMissing,
|
|
eventHandler: eventHandler
|
|
);
|
|
CreatorDownloadResult newResults = results.NewDownloads ?? results;
|
|
|
|
totalResults.Add(newResults);
|
|
|
|
DateTime userEndTime = DateTime.Now;
|
|
TimeSpan userTotalTime = userEndTime - userStartTime;
|
|
|
|
Log.ForContext("Paid Posts", newResults.PaidPostCount)
|
|
.ForContext("Posts", newResults.PostCount)
|
|
.ForContext("Archived", newResults.ArchivedCount)
|
|
.ForContext("Streams", newResults.StreamsCount)
|
|
.ForContext("Stories", newResults.StoriesCount)
|
|
.ForContext("Highlights", newResults.HighlightsCount)
|
|
.ForContext("Messages", newResults.MessagesCount)
|
|
.ForContext("Paid Messages", newResults.PaidMessagesCount)
|
|
.ForContext("Username", username)
|
|
.ForContext("TotalMinutes", userTotalTime.TotalMinutes)
|
|
.Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.ForContext("Username", username)
|
|
.ForContext("UserNum", userNum)
|
|
.ForContext("UserCount", userCount)
|
|
.ForContext("ExceptionMessage", ex.Message)
|
|
.Error(ex, "Scrape for '{Username:l}' ({UserNum} of {UserCount}) failed! {ExceptionMessage:l}");
|
|
}
|
|
}
|
|
|
|
DateTime endTime = DateTime.Now;
|
|
TimeSpan totalTime = endTime - startTime;
|
|
|
|
eventHandler.OnScrapeComplete(totalTime);
|
|
|
|
Log.ForContext("Paid Posts", totalResults.PaidPostCount)
|
|
.ForContext("Posts", totalResults.PostCount)
|
|
.ForContext("Archived", totalResults.ArchivedCount)
|
|
.ForContext("Streams", totalResults.StreamsCount)
|
|
.ForContext("Stories", totalResults.StoriesCount)
|
|
.ForContext("Highlights", totalResults.HighlightsCount)
|
|
.ForContext("Messages", totalResults.MessagesCount)
|
|
.ForContext("Paid Messages", totalResults.PaidMessagesCount)
|
|
.ForContext("TotalMinutes", totalTime.TotalMinutes)
|
|
.Information("Scrape Completed in {TotalMinutes:0.00} minutes");
|
|
|
|
await Task.Delay(2000);
|
|
}
|
|
|
|
private async Task OutputBlockedUsersAsync()
|
|
{
|
|
const string OUTPUT_FILE_BLOCKED = "blocked-users.json";
|
|
const string OUTPUT_FILE_EXPIRED = "expired-users.json";
|
|
|
|
await _apiService.SortBlockedAsync("/lists/blocked/sort");
|
|
await GetUsersAsync("Blocked", "/users/blocked", OUTPUT_FILE_BLOCKED);
|
|
|
|
await GetUsersAsync("Expired", "/subscriptions/subscribes", OUTPUT_FILE_EXPIRED, typeParam: "expired", offsetByCount: false);
|
|
|
|
async Task GetUsersAsync(string typeDisplay, string uri, string outputFile, string? typeParam = null, bool offsetByCount = true)
|
|
{
|
|
Dictionary<string, long> users = await _apiService.GetUsersWithProgressAsync(typeDisplay, uri, typeParam, offsetByCount);
|
|
|
|
Console.WriteLine();
|
|
|
|
if (users is null || users.Count == 0)
|
|
{
|
|
AnsiConsole.Markup($"[green]No {typeDisplay} Users found.\n[/]");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup($"[green]Found {users.Count} {typeDisplay} Users, saving to '{outputFile}'\n[/]");
|
|
string json = JsonConvert.SerializeObject(users, Formatting.Indented);
|
|
await File.WriteAllTextAsync(outputFile, json);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task UpdateUserInfoAsync()
|
|
{
|
|
await _dbService.CreateUsersDb([]);
|
|
await _dbService.InitializeUserInfoTablesAsync();
|
|
|
|
Dictionary<string, long> users = await _dbService.GetUsersAsync();
|
|
Console.WriteLine();
|
|
|
|
Log.Information("Updating User Info for '{UserCount}' users", users.Count);
|
|
AnsiConsole.Markup($"[green]Updating User Info for '{users.Count}' users\n[/]");
|
|
|
|
await AnsiConsole.Progress()
|
|
.Columns(new ProgressBarColumn(), new PercentageColumn(), new TaskDescriptionColumn { Alignment = Justify.Left })
|
|
.StartAsync(RunUpdateAsync);
|
|
|
|
async Task RunUpdateAsync(ProgressContext context)
|
|
{
|
|
ProgressTask? updateTask = null;
|
|
|
|
int maxUsernameLength = users.Keys.Max(s => s.Length);
|
|
|
|
foreach ((string username, long userId) in users)
|
|
{
|
|
string description = $"Updating '{username}'".PadRight(11 + maxUsernameLength);
|
|
double prevValue = updateTask?.Value ?? 0;
|
|
|
|
updateTask = context.AddTask(description, true, users.Count);
|
|
updateTask.Value = prevValue;
|
|
|
|
using (LogContext.PushProperty("Username", username))
|
|
using (LogContext.PushProperty("UserId", userId))
|
|
using (LogContext.PushProperty("UserNum", prevValue + 1))
|
|
using (LogContext.PushProperty("UserTotal", users.Count))
|
|
{
|
|
try
|
|
{
|
|
Log.Information("[{UserNum:0} of {UserTotal}] Updating User Info for for: {Username:l}");
|
|
UserEntities.UserInfo? userInfo = await _apiService.GetDetailedUserInfoAsync($"/users/{username}");
|
|
await _dbService.UpdateUserInfoAsync(userInfo);
|
|
|
|
updateTask.Description = $"{description} - COMPLETE";
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning(ex, "[{UserNum:0} of {UserTotal}] Failed to update User Info for: {Username:l}");
|
|
AnsiConsole.Markup($"[red]Failed to update User Info for '{username}'\n[/]");
|
|
|
|
updateTask.Description = $"{description} - FAILED: {ex.Message}";
|
|
}
|
|
finally
|
|
{
|
|
updateTask.Increment(1);
|
|
updateTask.StopTask();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private async Task<Dictionary<string, long>> GetUsersFromSpecificListsAsync(UserListResult allUsersAndLists, string[] listNames)
|
|
{
|
|
Config currentConfig = _configService.CurrentConfig;
|
|
Dictionary<string, long> usersFromLists = new(StringComparer.OrdinalIgnoreCase);
|
|
|
|
foreach (string name in listNames)
|
|
{
|
|
if (!allUsersAndLists.Lists.TryGetValue(name, out long listId))
|
|
continue;
|
|
|
|
Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", name, currentConfig.IncludeRestrictedSubscriptions);
|
|
AnsiConsole.Markup($"[green]Getting Users from list '{name}' (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
|
|
|
|
List<string> listUsernames = await _apiService.GetListUsers($"/lists/{listId}/users") ?? [];
|
|
|
|
foreach (string u in listUsernames)
|
|
{
|
|
if (usersFromLists.ContainsKey(u))
|
|
continue;
|
|
|
|
if (!allUsersAndLists.Users.TryGetValue(u, out long userId))
|
|
continue;
|
|
|
|
usersFromLists[u] = userId;
|
|
}
|
|
}
|
|
|
|
return usersFromLists;
|
|
}
|
|
|
|
private static Dictionary<string, long> GetUsersFromSpecificUsernames(UserListResult allUsersAndLists, HashSet<string> usernames)
|
|
{
|
|
Dictionary<string, long> filteredUsers = allUsersAndLists.Users
|
|
.Where(u => usernames.Contains(u.Key))
|
|
.ToDictionary(u => u.Key, u => u.Value);
|
|
|
|
return filteredUsers;
|
|
}
|
|
|
|
private async Task<UserListResult> GetAvailableUsersAsync()
|
|
{
|
|
Config currentConfig = _configService.CurrentConfig;
|
|
UserListResult result = new()
|
|
{
|
|
Users = new(StringComparer.OrdinalIgnoreCase)
|
|
};
|
|
|
|
await FetchUsersAsync();
|
|
await FetchListsAsync();
|
|
|
|
AnsiConsole.WriteLine();
|
|
|
|
await _dbService.CreateUsersDb(result.Users);
|
|
await _dbService.InitializeUserInfoTablesAsync();
|
|
|
|
await UpdateNonNudeListAsync();
|
|
|
|
return result;
|
|
|
|
async Task FetchUsersAsync()
|
|
{
|
|
Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
|
|
AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
|
|
|
|
Dictionary<string, long>? activeSubs = await _apiService.GetActiveSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
|
|
AddToResult(activeSubs);
|
|
|
|
if (currentConfig.IncludeExpiredSubscriptions)
|
|
{
|
|
Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
|
|
AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
|
|
|
|
Dictionary<string, long>? expiredSubs = await _apiService.GetExpiredSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
|
|
AddToResult(expiredSubs);
|
|
}
|
|
}
|
|
|
|
async Task FetchListsAsync()
|
|
{
|
|
Log.Information("Getting Lists");
|
|
result.Lists = await _apiService.GetLists("/lists") ?? [];
|
|
}
|
|
|
|
async Task UpdateNonNudeListAsync()
|
|
{
|
|
const string LIST_NAME = "NonNude";
|
|
const long LIST_ID = 1220021758;
|
|
|
|
HashSet<string> listNames = new(StringComparer.OrdinalIgnoreCase);
|
|
|
|
if (result.Lists.ContainsKey(LIST_NAME))
|
|
listNames.Add(LIST_NAME);
|
|
|
|
string? nameById = result.Lists.FirstOrDefault(l => l.Value == LIST_ID).Key;
|
|
if (!string.IsNullOrWhiteSpace(nameById))
|
|
listNames.Add(nameById);
|
|
|
|
Dictionary<string, long> usersInNonNudeLists = await GetUsersFromSpecificListsAsync(result, [.. listNames]);
|
|
|
|
AnsiConsole.Markup($"[green]Updating Non-Nude collection with {usersInNonNudeLists.Count} Users[/]");
|
|
await _dbService.UpdateNonNudeCollectionAsync(usersInNonNudeLists);
|
|
|
|
AnsiConsole.WriteLine();
|
|
}
|
|
|
|
void AddToResult(Dictionary<string, long>? subscriptions)
|
|
{
|
|
foreach ((string username, long userId) in subscriptions ?? [])
|
|
{
|
|
if (result.Users.TryAdd(username, userId))
|
|
Log.Debug($"Name: {username} ID: {userId}");
|
|
}
|
|
}
|
|
}
|
|
|
|
static ILogger LoggerWithConfigContext(Config config, CajetanConfig cajetanConfig)
|
|
=> Log.Logger
|
|
.ForContext(nameof(Config.DownloadPath), config.DownloadPath)
|
|
.ForContext(nameof(Config.DownloadPosts), config.DownloadPosts)
|
|
.ForContext(nameof(Config.DownloadPaidPosts), config.DownloadPaidPosts)
|
|
.ForContext(nameof(Config.DownloadMessages), config.DownloadMessages)
|
|
.ForContext(nameof(Config.DownloadPaidMessages), config.DownloadPaidMessages)
|
|
.ForContext(nameof(Config.DownloadStories), config.DownloadStories)
|
|
.ForContext(nameof(Config.DownloadStreams), config.DownloadStreams)
|
|
.ForContext(nameof(Config.DownloadHighlights), config.DownloadHighlights)
|
|
.ForContext(nameof(Config.DownloadArchived), config.DownloadArchived)
|
|
.ForContext(nameof(Config.DownloadAvatarHeaderPhoto), config.DownloadAvatarHeaderPhoto)
|
|
.ForContext(nameof(Config.DownloadImages), config.DownloadImages)
|
|
.ForContext(nameof(Config.DownloadVideos), config.DownloadVideos)
|
|
.ForContext(nameof(Config.DownloadAudios), config.DownloadAudios)
|
|
.ForContext(nameof(Config.IgnoreOwnMessages), config.IgnoreOwnMessages)
|
|
.ForContext(nameof(Config.DownloadPostsIncrementally), config.DownloadPostsIncrementally)
|
|
.ForContext(nameof(Config.BypassContentForCreatorsWhoNoLongerExist), config.BypassContentForCreatorsWhoNoLongerExist)
|
|
.ForContext(nameof(Config.SkipAds), config.SkipAds)
|
|
.ForContext(nameof(Config.IncludeExpiredSubscriptions), config.IncludeExpiredSubscriptions)
|
|
.ForContext(nameof(Config.IncludeRestrictedSubscriptions), config.IncludeRestrictedSubscriptions)
|
|
.ForContext(nameof(CajetanConfig.NonInteractiveSpecificLists), cajetanConfig.NonInteractiveSpecificLists)
|
|
.ForContext(nameof(CajetanConfig.NonInteractiveSpecificUsers), cajetanConfig.NonInteractiveSpecificUsers);
|
|
}
|