diff --git a/OF DL/CLI/SpectreDownloadEventHandler.cs b/OF DL/CLI/SpectreDownloadEventHandler.cs new file mode 100644 index 0000000..a34cd8a --- /dev/null +++ b/OF DL/CLI/SpectreDownloadEventHandler.cs @@ -0,0 +1,127 @@ +using OF_DL.Models; +using OF_DL.Services; +using Spectre.Console; + +namespace OF_DL.CLI; + +/// +/// Spectre.Console implementation of IDownloadEventHandler. +/// Handles all CLI-specific display logic for downloads. +/// +public class SpectreDownloadEventHandler : IDownloadEventHandler +{ + public async Task WithStatusAsync(string statusMessage, Func> work) + { + T result = default!; + await AnsiConsole.Status() + .StartAsync($"[red]{Markup.Escape(statusMessage)}[/]", + async ctx => + { + SpectreStatusReporter reporter = new(ctx); + result = await work(reporter); + }); + return result; + } + + public async Task WithProgressAsync(string description, long maxValue, bool showSize, + Func> work) + { + T result = default!; + await AnsiConsole.Progress() + .Columns(GetProgressColumns(showSize)) + .StartAsync(async ctx => + { + ProgressTask task = ctx.AddTask($"[red]{Markup.Escape(description)}[/]", false); + task.MaxValue = maxValue; + task.StartTask(); + + SpectreProgressReporter progressReporter = new(task); + result = await work(progressReporter); + + task.StopTask(); + }); + return result; + } + + public void OnContentFound(string contentType, int mediaCount, int objectCount) + { + AnsiConsole.Markup($"[red]Found {mediaCount} Media from {objectCount} {Markup.Escape(contentType)}\n[/]"); + } + + public void OnNoContentFound(string contentType) + { + AnsiConsole.Markup($"[red]Found 0 {Markup.Escape(contentType)}\n[/]"); + } + + public void OnDownloadComplete(string contentType, DownloadResult result) + { + AnsiConsole.Markup( + $"[red]{Markup.Escape(contentType)} Already Downloaded: {result.ExistingDownloads} New {Markup.Escape(contentType)} Downloaded: {result.NewDownloads}[/]\n"); + } + + public void OnUserStarting(string username) + { + AnsiConsole.Markup($"[red]\nScraping Data for {Markup.Escape(username)}\n[/]"); + } + + public void OnUserComplete(string username, CreatorDownloadResult result) + { + AnsiConsole.Markup("\n"); + AnsiConsole.Write(new BreakdownChart() + .FullSize() + .AddItem("Paid Posts", result.PaidPostCount, Color.Red) + .AddItem("Posts", result.PostCount, Color.Blue) + .AddItem("Archived", result.ArchivedCount, Color.Green) + .AddItem("Streams", result.StreamsCount, Color.Purple) + .AddItem("Stories", result.StoriesCount, Color.Yellow) + .AddItem("Highlights", result.HighlightsCount, Color.Orange1) + .AddItem("Messages", result.MessagesCount, Color.LightGreen) + .AddItem("Paid Messages", result.PaidMessagesCount, Color.Aqua)); + AnsiConsole.Markup("\n"); + } + + public void OnPurchasedTabUserComplete(string username, int paidPostCount, int paidMessagesCount) + { + AnsiConsole.Markup("\n"); + AnsiConsole.Write(new BreakdownChart() + .FullSize() + .AddItem("Paid Posts", paidPostCount, Color.Red) + .AddItem("Paid Messages", paidMessagesCount, Color.Aqua)); + AnsiConsole.Markup("\n"); + } + + public void OnScrapeComplete(TimeSpan elapsed) + { + AnsiConsole.Markup($"[green]Scrape Completed in {elapsed.TotalMinutes:0.00} minutes\n[/]"); + } + + public void OnMessage(string message) + { + AnsiConsole.Markup($"[red]{Markup.Escape(message)}\n[/]"); + } + + private static ProgressColumn[] GetProgressColumns(bool showScrapeSize) + { + List progressColumns; + if (showScrapeSize) + { + progressColumns = + [ + new TaskDescriptionColumn(), + new ProgressBarColumn(), + new PercentageColumn(), + new DownloadedColumn(), + new RemainingTimeColumn() + ]; + } + else + { + progressColumns = + [ + new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn() + ]; + } + + return progressColumns.ToArray(); + } +} diff --git a/OF DL/CLI/SpectreProgressReporter.cs b/OF DL/CLI/SpectreProgressReporter.cs index 49cf12b..a9fbd4a 100644 --- a/OF DL/CLI/SpectreProgressReporter.cs +++ b/OF DL/CLI/SpectreProgressReporter.cs @@ -6,11 +6,9 @@ namespace OF_DL.CLI; /// /// Implementation of IProgressReporter that uses Spectre.Console's ProgressTask for CLI output. /// -public class SpectreProgressReporter : IProgressReporter +public class SpectreProgressReporter(ProgressTask task) : IProgressReporter { - private readonly ProgressTask _task; - - public SpectreProgressReporter(ProgressTask task) => _task = task ?? throw new ArgumentNullException(nameof(task)); + private readonly ProgressTask _task = task ?? throw new ArgumentNullException(nameof(task)); public void ReportProgress(long increment) => _task.Increment(increment); diff --git a/OF DL/CLI/SpectreStatusReporter.cs b/OF DL/CLI/SpectreStatusReporter.cs new file mode 100644 index 0000000..7c965fd --- /dev/null +++ b/OF DL/CLI/SpectreStatusReporter.cs @@ -0,0 +1,17 @@ +using OF_DL.Services; +using Spectre.Console; + +namespace OF_DL.CLI; + +/// +/// Implementation of IStatusReporter that uses Spectre.Console's StatusContext for CLI output. +/// +public class SpectreStatusReporter(StatusContext ctx) : IStatusReporter +{ + public void ReportStatus(string message) + { + ctx.Status($"[red]{message}[/]"); + ctx.Spinner(Spinner.Known.Dots); + ctx.SpinnerStyle(Style.Parse("blue")); + } +} diff --git a/OF DL/Models/CreatorDownloadResult.cs b/OF DL/Models/CreatorDownloadResult.cs new file mode 100644 index 0000000..16de4cb --- /dev/null +++ b/OF DL/Models/CreatorDownloadResult.cs @@ -0,0 +1,29 @@ +namespace OF_DL.Models; + +public class CreatorDownloadResult +{ + public int PaidPostCount { get; set; } + + public int PostCount { get; set; } + + public int ArchivedCount { get; set; } + + public int StreamsCount { get; set; } + + public int StoriesCount { get; set; } + + public int HighlightsCount { get; set; } + + public int MessagesCount { get; set; } + + public int PaidMessagesCount { get; set; } +} + +public class UserListResult +{ + public Dictionary Users { get; set; } = new(); + + public Dictionary Lists { get; set; } = new(); + + public string? IgnoredListError { get; set; } +} diff --git a/OF DL/Models/StartupResult.cs b/OF DL/Models/StartupResult.cs new file mode 100644 index 0000000..6b67cd0 --- /dev/null +++ b/OF DL/Models/StartupResult.cs @@ -0,0 +1,39 @@ +namespace OF_DL.Models; + +public class StartupResult +{ + public bool IsWindowsVersionValid { get; set; } = true; + + public string? OsVersionString { get; set; } + + public bool FfmpegFound { get; set; } + + public bool FfmpegPathAutoDetected { get; set; } + + public string? FfmpegPath { get; set; } + + public string? FfmpegVersion { get; set; } + + public bool ClientIdBlobMissing { get; set; } + + public bool DevicePrivateKeyMissing { get; set; } + + public bool RulesJsonValid { get; set; } + + public bool RulesJsonExists { get; set; } + + public string? RulesJsonError { get; set; } +} + +public class VersionCheckResult +{ + public Version? LocalVersion { get; set; } + + public Version? LatestVersion { get; set; } + + public bool IsUpToDate { get; set; } + + public bool CheckFailed { get; set; } + + public bool TimedOut { get; set; } +} diff --git a/OF DL/Program.cs b/OF DL/Program.cs index 19dd492..550ab6f 100644 --- a/OF DL/Program.cs +++ b/OF DL/Program.cs @@ -1,34 +1,17 @@ -using System.Diagnostics; -using System.Reflection; -using System.Runtime.InteropServices; using System.Text.RegularExpressions; using Microsoft.Extensions.DependencyInjection; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using OF_DL.CLI; using OF_DL.Models; using OF_DL.Enumerations; -using OF_DL.Helpers; -using ArchivedEntities = OF_DL.Models.Entities.Archived; -using MessageEntities = OF_DL.Models.Entities.Messages; -using PostEntities = OF_DL.Models.Entities.Posts; -using PurchasedEntities = OF_DL.Models.Entities.Purchased; -using StreamEntities = OF_DL.Models.Entities.Streams; -using UserEntities = OF_DL.Models.Entities.Users; +using OF_DL.Models.Entities.Users; using OF_DL.Services; using Serilog; using Spectre.Console; -using WidevineConstants = OF_DL.Widevine.Constants; namespace OF_DL; public class Program(IServiceProvider serviceProvider) { - public static List paid_post_ids = new(); - - private static bool clientIdBlobMissing; - private static bool devicePrivateKeyMissing; - private async Task LoadAuthFromBrowser() { IAuthService authService = serviceProvider.GetRequiredService(); @@ -86,7 +69,6 @@ public class Program(IServiceProvider serviceProvider) AnsiConsole.Markup("Documentation: [link]https://docs.ofdl.tools/[/]\n"); AnsiConsole.Markup("Discord server: [link]https://discord.com/invite/6bUW8EJ53j[/]\n\n"); - ServiceCollection services = await ConfigureServices(args); ServiceProvider serviceProvider = services.BuildServiceProvider(); @@ -126,10 +108,12 @@ public class Program(IServiceProvider serviceProvider) services.AddSingleton(loggingService); services.AddSingleton(configService); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); return services; @@ -139,362 +123,41 @@ public class Program(IServiceProvider serviceProvider) { IConfigService configService = serviceProvider.GetRequiredService(); IAuthService authService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); + IStartupService startupService = serviceProvider.GetRequiredService(); + IDownloadOrchestrationService orchestrationService = + serviceProvider.GetRequiredService(); try { - OperatingSystem os = Environment.OSVersion; + // Version check + VersionCheckResult versionResult = await startupService.CheckVersionAsync(); + DisplayVersionResult(versionResult); - Log.Debug($"Operating system information: {os.VersionString}"); + // Environment validation + StartupResult startupResult = await startupService.ValidateEnvironmentAsync(); + DisplayStartupResult(startupResult); - if (os.Platform == PlatformID.Win32NT) + if (!startupResult.IsWindowsVersionValid) { - // check if this is windows 10+ - if (os.Version.Major < 10) + Console.Write( + "This appears to be running on an older version of Windows which is not supported.\n\n"); + Console.Write( + "OF-DL requires Windows 10 or higher when being run on Windows. Your reported version is: {0}\n\n", + startupResult.OsVersionString); + Console.Write("Press any key to continue.\n"); + + if (!configService.CurrentConfig.NonInteractiveMode) { - Console.Write( - "This appears to be running on an older version of Windows which is not supported.\n\n"); - Console.Write( - "OF-DL requires Windows 10 or higher when being run on Windows. Your reported version is: {0}\n\n", - os.VersionString); - Console.Write("Press any key to continue.\n"); - Log.Error("Windows version prior to 10.x: {0}", os.VersionString); - - if (!configService.CurrentConfig.NonInteractiveMode) - { - Console.ReadKey(); - } - - Environment.Exit(1); - } - else - { - AnsiConsole.Markup("[green]Valid version of Windows found.\n[/]"); - } - } - - try - { - // Only run the version check if not in DEBUG mode -#if !DEBUG - Version localVersion = - Assembly.GetEntryAssembly()?.GetName().Version; //Only tested with numeric values. - - // Create a cancellation token with 30 second timeout - using CancellationTokenSource cts = new(TimeSpan.FromSeconds(30)); - string? latestReleaseTag = null; - - try - { - latestReleaseTag = await VersionHelper.GetLatestReleaseTag(cts.Token); - } - catch (OperationCanceledException) - { - AnsiConsole.Markup("[yellow]Version check timed out after 30 seconds.\n[/]"); - Log.Warning("Version check timed out after 30 seconds"); - latestReleaseTag = null; - } - - if (latestReleaseTag == null) - { - AnsiConsole.Markup("[yellow]Failed to verify that OF-DL is up-to-date.\n[/]"); - Log.Error("Failed to get the latest release tag."); - } - else - { - Version latestGiteaRelease = new(latestReleaseTag.Replace("OFDLV", "")); - - // Compare the Versions - int versionComparison = localVersion.CompareTo(latestGiteaRelease); - if (versionComparison < 0) - { - // The version on GitHub is more up to date than this local release. - AnsiConsole.Markup("[red]You are running OF-DL version " + - $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}\n[/]"); - AnsiConsole.Markup("[red]Please update to the current release, " + - $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}: [link=https://git.ofdl.tools/sim0n00ps/OF-DL/releases]https://git.ofdl.tools/sim0n00ps/OF-DL/releases[/]\n[/]"); - Log.Debug("Detected outdated client running version " + - $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}"); - Log.Debug("Latest release version " + - $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}"); - } - else - { - // This local version is greater than the release version on GitHub. - AnsiConsole.Markup("[green]You are running OF-DL version " + - $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}\n[/]"); - AnsiConsole.Markup("[green]Latest Release version: " + - $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}\n[/]"); - Log.Debug("Detected client running version " + - $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}"); - Log.Debug("Latest release version " + - $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}"); - } - } - -#else - AnsiConsole.Markup("[yellow]Running in Debug/Local mode. Version check skipped.\n[/]"); - Log.Debug("Running in Debug/Local mode. Version check skipped."); -#endif - } - catch (Exception e) - { - AnsiConsole.Markup("[red]Error checking latest release on GitHub:\n[/]"); - Console.WriteLine(e); - Log.Error("Error checking latest release on GitHub.", e.Message); - } - - - if (await authService.LoadFromFileAsync()) - { - AnsiConsole.Markup("[green]auth.json located successfully!\n[/]"); - } - else if (File.Exists("auth.json")) - { - // File exists but failed to load - Log.Information("Auth file found but could not be deserialized"); - if (!configService.CurrentConfig!.DisableBrowserAuth) - { - Log.Debug("Deleting auth.json"); - File.Delete("auth.json"); - } - - if (configService.CurrentConfig.NonInteractiveMode) - { - AnsiConsole.MarkupLine( - "\n[red]auth.json has invalid JSON syntax. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n"); - AnsiConsole.MarkupLine( - "[red]You may also want to try using the browser extension which is documented here:[/]\n"); - AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]"); - AnsiConsole.MarkupLine("[red]Press any key to exit.[/]"); - Console.ReadKey(); - Environment.Exit(2); } - - if (!configService.CurrentConfig!.DisableBrowserAuth) - { - await LoadAuthFromBrowser(); - } - else - { - AnsiConsole.MarkupLine( - "\n[red]auth.json is missing. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n"); - AnsiConsole.MarkupLine( - "[red]You may also want to try using the browser extension which is documented here:[/]\n"); - AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]"); - AnsiConsole.MarkupLine("[red]Press any key to exit.[/]"); - - Console.ReadKey(); - Environment.Exit(2); - } - } - else - { - if (configService.CurrentConfig.NonInteractiveMode) - { - AnsiConsole.MarkupLine( - "\n[red]auth.json is missing. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n"); - AnsiConsole.MarkupLine( - "[red]You may also want to try using the browser extension which is documented here:[/]\n"); - AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]"); - AnsiConsole.MarkupLine("[red]Press any key to exit.[/]"); - - Console.ReadKey(); - Environment.Exit(2); - } - - if (!configService.CurrentConfig!.DisableBrowserAuth) - { - await LoadAuthFromBrowser(); - } - else - { - AnsiConsole.MarkupLine( - "\n[red]auth.json is missing. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n"); - AnsiConsole.MarkupLine( - "[red]You may also want to try using the browser extension which is documented here:[/]\n"); - AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]"); - AnsiConsole.MarkupLine("[red]Press any key to exit.[/]"); - - Console.ReadKey(); - Environment.Exit(2); - } + Environment.Exit(1); } - //Added to stop cookie being filled with un-needed headers - ValidateCookieString(authService.CurrentAuth!); - - if (File.Exists("rules.json")) - { - AnsiConsole.Markup("[green]rules.json located successfully!\n[/]"); - try - { - JsonConvert.DeserializeObject(File.ReadAllText("rules.json")); - Log.Debug("Rules.json: "); - Log.Debug(JsonConvert.SerializeObject(File.ReadAllText("rules.json"), Formatting.Indented)); - } - catch (Exception e) - { - Console.WriteLine(e); - AnsiConsole.MarkupLine("\n[red]rules.json is not valid, check your JSON syntax![/]\n"); - AnsiConsole.MarkupLine("[red]Please ensure you are using the latest version of the software.[/]\n"); - AnsiConsole.MarkupLine("[red]Press any key to exit.[/]"); - Log.Error("rules.json processing failed.", e.Message); - - if (!configService.CurrentConfig.NonInteractiveMode) - { - Console.ReadKey(); - } - - Environment.Exit(2); - } - } - - if (configService.CurrentConfig.NonInteractiveMode) - { - // CLI argument overrides configuration - configService.CurrentConfig!.NonInteractiveMode = true; - Log.Debug("NonInteractiveMode = true"); - } - - if (configService.CurrentConfig!.NonInteractiveMode) - { - configService.CurrentConfig.NonInteractiveMode = - true; // If it was set in the config, reset the cli value so exception handling works - Log.Debug("NonInteractiveMode = true (set via config)"); - } - - bool ffmpegFound = false; - bool pathAutoDetected = false; - if (!string.IsNullOrEmpty(configService.CurrentConfig!.FFmpegPath) && - ValidateFilePath(configService.CurrentConfig.FFmpegPath)) - { - // FFmpeg path is set in config.json and is valid - ffmpegFound = true; - Log.Debug($"FFMPEG found: {configService.CurrentConfig.FFmpegPath}"); - Log.Debug("FFMPEG path set in config.conf"); - } - else if (!string.IsNullOrEmpty(authService.CurrentAuth!.FfmpegPath) && - ValidateFilePath(authService.CurrentAuth.FfmpegPath)) - { - // FFmpeg path is set in auth.json and is valid (config.conf takes precedence and auth.json is only available for backward compatibility) - ffmpegFound = true; - configService.CurrentConfig.FFmpegPath = authService.CurrentAuth.FfmpegPath; - Log.Debug($"FFMPEG found: {configService.CurrentConfig.FFmpegPath}"); - Log.Debug("FFMPEG path set in auth.json"); - } - else if (string.IsNullOrEmpty(configService.CurrentConfig.FFmpegPath)) - { - // FFmpeg path is not set in config.conf, so we will try to locate it in the PATH or current directory - string? ffmpegPath = GetFullPath("ffmpeg"); - if (ffmpegPath != null) - { - // FFmpeg is found in the PATH or current directory - ffmpegFound = true; - pathAutoDetected = true; - configService.CurrentConfig.FFmpegPath = ffmpegPath; - Log.Debug($"FFMPEG found: {ffmpegPath}"); - Log.Debug("FFMPEG path found via PATH or current directory"); - } - else - { - // FFmpeg is not found in the PATH or current directory, so we will try to locate the windows executable - ffmpegPath = GetFullPath("ffmpeg.exe"); - if (ffmpegPath != null) - { - // FFmpeg windows executable is found in the PATH or current directory - ffmpegFound = true; - pathAutoDetected = true; - configService.CurrentConfig.FFmpegPath = ffmpegPath; - Log.Debug($"FFMPEG found: {ffmpegPath}"); - Log.Debug("FFMPEG path found in windows excutable directory"); - } - } - } - - if (ffmpegFound) - { - if (pathAutoDetected) - { - AnsiConsole.Markup( - $"[green]FFmpeg located successfully. Path auto-detected: {configService.CurrentConfig.FFmpegPath}\n[/]"); - } - else - { - AnsiConsole.Markup("[green]FFmpeg located successfully\n[/]"); - } - - // Escape backslashes in the path for Windows - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && - configService.CurrentConfig.FFmpegPath!.Contains(@":\") && - !configService.CurrentConfig.FFmpegPath.Contains(@":\\")) - { - configService.CurrentConfig.FFmpegPath = - configService.CurrentConfig.FFmpegPath.Replace(@"\", @"\\"); - } - - // Get FFmpeg version - try - { - ProcessStartInfo processStartInfo = new() - { - FileName = configService.CurrentConfig.FFmpegPath, - Arguments = "-version", - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - CreateNoWindow = true - }; - - using (Process? process = Process.Start(processStartInfo)) - { - if (process != null) - { - string output = await process.StandardOutput.ReadToEndAsync(); - await process.WaitForExitAsync(); - - // Log full output - Log.Information("FFmpeg version output:\n{Output}", output); - - // Parse first line for console output - string firstLine = output.Split('\n')[0].Trim(); - if (firstLine.StartsWith("ffmpeg version")) - { - // Extract version string (text between "ffmpeg version " and " Copyright") - int versionStart = "ffmpeg version ".Length; - int copyrightIndex = firstLine.IndexOf(" Copyright"); - if (copyrightIndex > versionStart) - { - string version = firstLine.Substring(versionStart, copyrightIndex - versionStart); - AnsiConsole.Markup($"[green]ffmpeg version detected as {version}[/]\n"); - } - else - { - // Fallback if Copyright not found - string version = firstLine.Substring(versionStart); - AnsiConsole.Markup($"[green]ffmpeg version detected as {version}[/]\n"); - } - } - else - { - AnsiConsole.Markup("[yellow]ffmpeg version could not be parsed[/]\n"); - } - } - } - } - catch (Exception ex) - { - Log.Warning(ex, "Failed to get FFmpeg version"); - AnsiConsole.Markup("[yellow]Could not retrieve ffmpeg version[/]\n"); - } - } - else + if (!startupResult.FfmpegFound) { AnsiConsole.Markup( "[red]Cannot locate FFmpeg; please modify config.conf with the correct path. Press any key to exit.[/]"); - Log.Error($"Cannot locate FFmpeg with path: {configService.CurrentConfig.FFmpegPath}"); if (!configService.CurrentConfig.NonInteractiveMode) { Console.ReadKey(); @@ -503,45 +166,30 @@ public class Program(IServiceProvider serviceProvider) Environment.Exit(4); } - if (!File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER, WidevineConstants.DEVICE_NAME, - "device_client_id_blob"))) + // Auth flow + await HandleAuthFlow(authService, configService); + + // Validate cookie string + authService.ValidateCookieString(); + + // rules.json validation + DisplayRulesJsonResult(startupResult, configService); + + // NonInteractiveMode + if (configService.CurrentConfig.NonInteractiveMode) { - clientIdBlobMissing = true; - Log.Debug("clientIdBlobMissing missing"); - } - else - { - AnsiConsole.Markup("[green]device_client_id_blob located successfully![/]\n"); - Log.Debug("clientIdBlobMissing found: " + File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER, - WidevineConstants.DEVICE_NAME, "device_client_id_blob"))); + configService.CurrentConfig.NonInteractiveMode = true; + Log.Debug("NonInteractiveMode = true"); } - if (!File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER, WidevineConstants.DEVICE_NAME, - "device_private_key"))) - { - devicePrivateKeyMissing = true; - Log.Debug("devicePrivateKeyMissing missing"); - } - else - { - AnsiConsole.Markup("[green]device_private_key located successfully![/]\n"); - Log.Debug("devicePrivateKeyMissing found: " + File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER, - WidevineConstants.DEVICE_NAME, "device_private_key"))); - } - - if (clientIdBlobMissing || devicePrivateKeyMissing) - { - AnsiConsole.Markup( - "[yellow]device_client_id_blob and/or device_private_key missing, https://ofdl.tools/ or https://cdrm-project.com/ will be used instead for DRM protected videos\n[/]"); - } - - UserEntities.User? validate = await apiService.GetUserInfo("/users/me"); - if (validate == null || (validate?.Name == null && validate?.Username == null)) + // Validate auth via API + User? validate = await authService.ValidateAuthAsync(); + if (validate == null || (validate.Name == null && validate.Username == null)) { Log.Error("Auth failed"); - authService.CurrentAuth = null; - if (!configService.CurrentConfig!.DisableBrowserAuth) + + if (!configService.CurrentConfig.DisableBrowserAuth) { if (File.Exists("auth.json")) { @@ -549,7 +197,8 @@ public class Program(IServiceProvider serviceProvider) } } - if (!configService.CurrentConfig.NonInteractiveMode && !configService.CurrentConfig!.DisableBrowserAuth) + if (!configService.CurrentConfig.NonInteractiveMode && + !configService.CurrentConfig.DisableBrowserAuth) { await LoadAuthFromBrowser(); } @@ -564,8 +213,11 @@ public class Program(IServiceProvider serviceProvider) } } - AnsiConsole.Markup($"[green]Logged In successfully as {validate.Name} {validate.Username}\n[/]"); - await DownloadAllData(); + AnsiConsole.Markup( + $"[green]Logged In successfully as {(!string.IsNullOrEmpty(validate?.Name) ? validate.Name : "Unknown Name")} {(!string.IsNullOrEmpty(validate?.Username) ? validate.Username : "Unknown Username")}\n[/]"); + + // Main download loop + await DownloadAllData(orchestrationService, configService, startupResult); } catch (Exception ex) { @@ -590,1426 +242,233 @@ public class Program(IServiceProvider serviceProvider) } } - private async Task DownloadAllData() + private async Task DownloadAllData( + IDownloadOrchestrationService orchestrationService, + IConfigService configService, + StartupResult startupResult) { - IDBService dbService = serviceProvider.GetRequiredService(); - IConfigService configService = serviceProvider.GetRequiredService(); - IAuthService authService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - Config Config = configService.CurrentConfig!; + Config config = configService.CurrentConfig; + SpectreDownloadEventHandler eventHandler = new(); Log.Debug("Calling DownloadAllData"); do { DateTime startTime = DateTime.Now; - Dictionary users = new(); - Dictionary activeSubs = - await apiService.GetActiveSubscriptions("/subscriptions/subscribes", - Config.IncludeRestrictedSubscriptions); - Log.Debug("Subscriptions: "); + UserListResult userListResult = await orchestrationService.GetAvailableUsersAsync(); + Dictionary users = userListResult.Users; + Dictionary lists = userListResult.Lists; - foreach (KeyValuePair activeSub in activeSubs) + if (userListResult.IgnoredListError != null) { - if (!users.ContainsKey(activeSub.Key)) - { - users.Add(activeSub.Key, activeSub.Value); - Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}"); - } + AnsiConsole.Markup($"[red]{Markup.Escape(userListResult.IgnoredListError)}\n[/]"); } - if (Config!.IncludeExpiredSubscriptions) - { - Log.Debug("Inactive Subscriptions: "); - - Dictionary expiredSubs = - await apiService.GetExpiredSubscriptions("/subscriptions/subscribes", - Config.IncludeRestrictedSubscriptions); - foreach (KeyValuePair expiredSub in expiredSubs) - { - if (!users.ContainsKey(expiredSub.Key)) - { - users.Add(expiredSub.Key, expiredSub.Value); - Log.Debug($"Name: {expiredSub.Key} ID: {expiredSub.Value}"); - } - } - } - - Dictionary lists = await apiService.GetLists("/lists"); - - // Remove users from the list if they are in the ignored list - if (!string.IsNullOrEmpty(Config.IgnoredUsersListName)) - { - if (!lists.TryGetValue(Config.IgnoredUsersListName, out long ignoredUsersListId)) - { - AnsiConsole.Markup($"[red]Ignored users list '{Config.IgnoredUsersListName}' not found\n[/]"); - Log.Error($"Ignored users list '{Config.IgnoredUsersListName}' not found"); - } - else - { - List ignoredUsernames = - await apiService.GetListUsers($"/lists/{ignoredUsersListId}/users") ?? []; - users = users.Where(x => !ignoredUsernames.Contains(x.Key)).ToDictionary(x => x.Key, x => x.Value); - } - } - - await dbService.CreateUsersDB(users); KeyValuePair> hasSelectedUsersKVP; - if (Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab) + if (config.NonInteractiveMode && config.NonInteractiveModePurchasedTab) { hasSelectedUsersKVP = new KeyValuePair>(true, new Dictionary { { "PurchasedTab", 0 } }); } - else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName)) + else if (config.NonInteractiveMode && string.IsNullOrEmpty(config.NonInteractiveModeListName)) { hasSelectedUsersKVP = new KeyValuePair>(true, users); } - else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName)) + else if (config.NonInteractiveMode && !string.IsNullOrEmpty(config.NonInteractiveModeListName)) { - long listId = lists[Config.NonInteractiveModeListName]; - List listUsernames = await apiService.GetListUsers($"/lists/{listId}/users") ?? []; - Dictionary selectedUsers = users.Where(x => listUsernames.Contains(x.Key)).Distinct() - .ToDictionary(x => x.Key, x => x.Value); + Dictionary selectedUsers = + await orchestrationService.GetUsersForListAsync(config.NonInteractiveModeListName, users, lists); hasSelectedUsersKVP = new KeyValuePair>(true, selectedUsers); } else { - ILoggingService loggingService = serviceProvider.GetRequiredService(); (bool IsExit, Dictionary? selectedUsers) userSelectionResult = await HandleUserSelection(users, lists); - Config = configService.CurrentConfig!; + config = configService.CurrentConfig; hasSelectedUsersKVP = new KeyValuePair>(userSelectionResult.IsExit, - userSelectionResult.selectedUsers); + userSelectionResult.selectedUsers ?? []); } - if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && + if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value.ContainsKey("SinglePost")) { - AnsiConsole.Markup( - "[red]To find an individual post URL, click on the ... at the top right corner of the post and select 'Copy link to post'.\n\nTo return to the main menu, enter 'back' or 'exit' when prompted for the URL.\n\n[/]"); - string postUrl = AnsiConsole.Prompt( - new TextPrompt("[red]Please enter a post URL: [/]") - .ValidationErrorMessage("[red]Please enter a valid post URL[/]") - .Validate(url => - { - Log.Debug($"Single Post URL: {url}"); - Regex regex = new("https://onlyfans\\.com/[0-9]+/[A-Za-z0-9]+", RegexOptions.IgnoreCase); - if (regex.IsMatch(url)) - { - return ValidationResult.Success(); - } - - if (url == "" || url == "exit" || url == "back") - { - return ValidationResult.Success(); - } - - Log.Error("Post URL invalid"); - return ValidationResult.Error("[red]Please enter a valid post URL[/]"); - })); - - if (postUrl != "" && postUrl != "exit" && postUrl != "back") - { - long post_id = Convert.ToInt64(postUrl.Split("/")[3]); - string username = postUrl.Split("/")[4]; - - Log.Debug($"Single Post ID: {post_id.ToString()}"); - Log.Debug($"Single Post Creator: {username}"); - - if (users.ContainsKey(username)) - { - string path = ""; - if (!string.IsNullOrEmpty(Config.DownloadPath)) - { - path = Path.Combine(Config.DownloadPath, username); - } - else - { - path = $"__user_data__/sites/OnlyFans/{username}"; - } - - Log.Debug($"Download path: {path}"); - - if (!Directory.Exists(path)) - { - Directory.CreateDirectory(path); - AnsiConsole.Markup($"[red]Created folder for {username}\n[/]"); - Log.Debug($"Created folder for {username}"); - } - else - { - AnsiConsole.Markup($"[red]Folder for {username} already created\n[/]"); - } - - await dbService.CreateDB(path); - - await DownloadSinglePost(username, post_id, path, users); - } - } + await HandleSinglePostDownload(orchestrationService, users, startupResult, eventHandler); } - else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && + else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value.ContainsKey("PurchasedTab")) { - Dictionary purchasedTabUsers = - await apiService.GetPurchasedTabUsers("/posts/paid/all", users); - AnsiConsole.Markup("[red]Checking folders for Users in Purchased Tab\n[/]"); - foreach (KeyValuePair user in purchasedTabUsers) - { - string path = ""; - if (!string.IsNullOrEmpty(Config.DownloadPath)) - { - path = Path.Combine(Config.DownloadPath, user.Key); - } - else - { - path = $"__user_data__/sites/OnlyFans/{user.Key}"; - } - - Log.Debug($"Download path: {path}"); - - await dbService.CheckUsername(user, path); - - if (!Directory.Exists(path)) - { - Directory.CreateDirectory(path); - AnsiConsole.Markup($"[red]Created folder for {user.Key}\n[/]"); - Log.Debug($"Created folder for {user.Key}"); - } - else - { - AnsiConsole.Markup($"[red]Folder for {user.Key} already created\n[/]"); - Log.Debug($"Folder for {user.Key} already created"); - } - - UserEntities.User user_info = await apiService.GetUserInfo($"/users/{user.Key}"); - - await dbService.CreateDB(path); - } - - string p = ""; - if (!string.IsNullOrEmpty(Config.DownloadPath)) - { - p = Config.DownloadPath; - } - else - { - p = "__user_data__/sites/OnlyFans/"; - } - - Log.Debug($"Download path: {p}"); - - List purchasedTabCollections = - await apiService.GetPurchasedTab("/posts/paid/all", p, users); - foreach (PurchasedEntities.PurchasedTabCollection purchasedTabCollection in purchasedTabCollections) - { - AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]"); - string path = ""; - if (!string.IsNullOrEmpty(Config.DownloadPath)) - { - path = Path.Combine(Config.DownloadPath, purchasedTabCollection.Username); - } - else - { - path = $"__user_data__/sites/OnlyFans/{purchasedTabCollection.Username}"; - } - - - Log.Debug($"Download path: {path}"); - - int paidPostCount = 0; - int paidMessagesCount = 0; - paidPostCount = await DownloadPaidPostsPurchasedTab(purchasedTabCollection.Username, - purchasedTabCollection.PaidPosts, - users.FirstOrDefault(u => u.Value == purchasedTabCollection.UserId), paidPostCount, path, - users); - paidMessagesCount = await DownloadPaidMessagesPurchasedTab(purchasedTabCollection.Username, - purchasedTabCollection.PaidMessages, - users.FirstOrDefault(u => u.Value == purchasedTabCollection.UserId), paidMessagesCount, path, - users); - - AnsiConsole.Markup("\n"); - AnsiConsole.Write(new BreakdownChart() - .FullSize() - .AddItem("Paid Posts", paidPostCount, Color.Red) - .AddItem("Paid Messages", paidMessagesCount, Color.Aqua)); - AnsiConsole.Markup("\n"); - } + await orchestrationService.DownloadPurchasedTabAsync(users, + startupResult.ClientIdBlobMissing, startupResult.DevicePrivateKeyMissing, eventHandler); DateTime endTime = DateTime.Now; - TimeSpan totalTime = endTime - startTime; - AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]"); - Log.Debug($"Scrape Completed in {totalTime.TotalMinutes:0.00} minutes"); + eventHandler.OnScrapeComplete(endTime - startTime); } - else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && + else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value.ContainsKey("SingleMessage")) { - AnsiConsole.Markup( - "[red]To find an individual message URL, note that you can only do so for PPV messages that you have unlocked. Go the main OnlyFans timeline, click on the Purchased tab, find the relevant message, click on the ... at the top right corner of the message, and select 'Copy link to message'. For all other messages, you cannot scrape them individually, you must scrape all messages from that creator.\n\nTo return to the main menu, enter 'back' or 'exit' when prompted for the URL.\n\n[/]"); - string messageUrl = AnsiConsole.Prompt( - new TextPrompt("[red]Please enter a message URL: [/]") - .ValidationErrorMessage("[red]Please enter a valid message URL[/]") - .Validate(url => - { - Log.Debug($"Single Paid Message URL: {url}"); - Regex regex = new("https://onlyfans\\.com/my/chats/chat/[0-9]+/\\?firstId=[0-9]+$", - RegexOptions.IgnoreCase); - if (regex.IsMatch(url)) - { - return ValidationResult.Success(); - } - - if (url == "" || url == "back" || url == "exit") - { - return ValidationResult.Success(); - } - - Log.Error("Message URL invalid"); - return ValidationResult.Error("[red]Please enter a valid message URL[/]"); - })); - - if (messageUrl != "" && messageUrl != "exit" && messageUrl != "back") - { - long message_id = Convert.ToInt64(messageUrl.Split("?firstId=")[1]); - long user_id = Convert.ToInt64(messageUrl.Split("/")[6]); - JObject user = await apiService.GetUserInfoById($"/users/list?x[]={user_id.ToString()}"); - string username = ""; - - Log.Debug($"Message ID: {message_id}"); - Log.Debug($"User ID: {user_id}"); - - if (user is null) - { - username = $"Deleted User - {user_id.ToString()}"; - Log.Debug("Content creator not longer exists - ", user_id.ToString()); - } - else if (!string.IsNullOrEmpty(user[user_id.ToString()]["username"].ToString())) - { - username = user[user_id.ToString()]["username"].ToString(); - Log.Debug("Content creator: ", username); - } - - string path = ""; - if (!string.IsNullOrEmpty(Config.DownloadPath)) - { - path = Path.Combine(Config.DownloadPath, username); - } - else - { - path = $"__user_data__/sites/OnlyFans/{username}"; - } - - Log.Debug("Download path: ", path); - - if (!Directory.Exists(path)) - { - Directory.CreateDirectory(path); - AnsiConsole.Markup($"[red]Created folder for {username}\n[/]"); - Log.Debug($"Created folder for {username}"); - } - else - { - AnsiConsole.Markup($"[red]Folder for {username} already created\n[/]"); - Log.Debug($"Folder for {username} already created"); - } - - await dbService.CreateDB(path); - - await DownloadPaidMessage(username, hasSelectedUsersKVP, 1, path, message_id); - } + await HandleSingleMessageDownload(orchestrationService, users, startupResult, eventHandler); } - else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged")) + else if (hasSelectedUsersKVP.Key && + !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged")) { - //Iterate over each user in the list of users foreach (KeyValuePair user in hasSelectedUsersKVP.Value) { - int paidPostCount = 0; - int postCount = 0; - int archivedCount = 0; - int streamsCount = 0; - int storiesCount = 0; - int highlightsCount = 0; - int messagesCount = 0; - int paidMessagesCount = 0; - AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]"); + string path = orchestrationService.ResolveDownloadPath(user.Key); + Log.Debug($"Download path: {path}"); - Log.Debug($"Scraping Data for {user.Key}"); - - string path = ""; - if (!string.IsNullOrEmpty(Config.DownloadPath)) - { - path = Path.Combine(Config.DownloadPath, user.Key); - } - else - { - path = $"__user_data__/sites/OnlyFans/{user.Key}"; - } - - Log.Debug("Download path: ", path); - - await dbService.CheckUsername(user, path); - - if (!Directory.Exists(path)) - { - Directory.CreateDirectory(path); - AnsiConsole.Markup($"[red]Created folder for {user.Key}\n[/]"); - Log.Debug($"Created folder for {user.Key}"); - } - else - { - AnsiConsole.Markup($"[red]Folder for {user.Key} already created\n[/]"); - Log.Debug($"Folder for {user.Key} already created"); - } - - await dbService.CreateDB(path); - - if (Config.DownloadAvatarHeaderPhoto) - { - UserEntities.User? user_info = await apiService.GetUserInfo($"/users/{user.Key}"); - if (user_info != null) - { - await downloadService.DownloadAvatarHeader(user_info.Avatar, user_info.Header, path, - user.Key); - } - } - - if (Config.DownloadPaidPosts) - { - paidPostCount = - await DownloadPaidPosts(user.Key, hasSelectedUsersKVP, user, paidPostCount, path); - } - - if (Config.DownloadPosts) - { - postCount = await DownloadFreePosts(user.Key, hasSelectedUsersKVP, user, postCount, path); - } - - if (Config.DownloadArchived) - { - archivedCount = - await DownloadArchived(user.Key, hasSelectedUsersKVP, user, archivedCount, path); - } - - if (Config.DownloadStreams) - { - streamsCount = await DownloadStreams(user.Key, hasSelectedUsersKVP, user, streamsCount, path); - } - - if (Config.DownloadStories) - { - storiesCount = await DownloadStories(user.Key, user, storiesCount, path); - } - - if (Config.DownloadHighlights) - { - highlightsCount = await DownloadHighlights(user.Key, user, highlightsCount, path); - } - - if (Config.DownloadMessages) - { - messagesCount = - await DownloadMessages(user.Key, hasSelectedUsersKVP, user, messagesCount, path); - } - - if (Config.DownloadPaidMessages) - { - paidMessagesCount = await DownloadPaidMessages(user.Key, hasSelectedUsersKVP, user, - paidMessagesCount, path); - } - - AnsiConsole.Markup("\n"); - AnsiConsole.Write(new BreakdownChart() - .FullSize() - .AddItem("Paid Posts", paidPostCount, Color.Red) - .AddItem("Posts", postCount, Color.Blue) - .AddItem("Archived", archivedCount, Color.Green) - .AddItem("Streams", streamsCount, Color.Purple) - .AddItem("Stories", storiesCount, Color.Yellow) - .AddItem("Highlights", highlightsCount, Color.Orange1) - .AddItem("Messages", messagesCount, Color.LightGreen) - .AddItem("Paid Messages", paidMessagesCount, Color.Aqua)); - AnsiConsole.Markup("\n"); + await orchestrationService.DownloadCreatorContentAsync( + user.Key, user.Value, path, users, + startupResult.ClientIdBlobMissing, startupResult.DevicePrivateKeyMissing, + eventHandler); } DateTime endTime = DateTime.Now; - TimeSpan totalTime = endTime - startTime; - AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]"); + eventHandler.OnScrapeComplete(endTime - startTime); } - else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && + else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged")) { + // Config was changed, loop will re-read } else { break; } - } while (!Config.NonInteractiveMode); + } while (!config.NonInteractiveMode); } - private async Task DownloadPaidMessages(string username, - KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, - int paidMessagesCount, string path) + private async Task HandleSinglePostDownload( + IDownloadOrchestrationService orchestrationService, + Dictionary users, + StartupResult startupResult, + IDownloadEventHandler eventHandler) { - IConfigService configService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - PurchasedEntities.PaidMessageCollection paidMessageCollection = new(); - - await AnsiConsole.Status() - .StartAsync("[red]Getting Paid Messages[/]", - async ctx => - { - paidMessageCollection = await apiService.GetPaidMessages("/posts/paid/chat", path, user.Key, ctx); - }); - - if (paidMessageCollection != null && paidMessageCollection.PaidMessages.Count > 0) - { - AnsiConsole.Markup( - $"[red]Found {paidMessageCollection.PaidMessages.Count} Media from {paidMessageCollection.PaidMessageObjects.Count} Paid Messages\n[/]"); - - long totalSize = 0; - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(paidMessageCollection.PaidMessages.Values - .ToList()); - } - else - { - totalSize = paidMessageCollection.PaidMessages.Count; - } - - DownloadResult result = null; - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - ProgressTask task = - ctx.AddTask($"[red]Downloading {paidMessageCollection.PaidMessages.Count} Paid Messages[/]", - false); - task.MaxValue = totalSize; - task.StartTask(); - - SpectreProgressReporter progressReporter = new(task); - result = await downloadService.DownloadPaidMessages(username, path, hasSelectedUsersKVP.Value, - clientIdBlobMissing, devicePrivateKeyMissing, paidMessageCollection, progressReporter); - - task.StopTask(); - }); - - AnsiConsole.Markup( - $"[red]Paid Messages Already Downloaded: {result.ExistingDownloads} New Paid Messages Downloaded: {result.NewDownloads}[/]\n"); - return result.TotalCount; - } - - AnsiConsole.Markup("[red]Found 0 Paid Messages\n[/]"); - return 0; - } - - private async Task DownloadMessages(string username, - KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, - int messagesCount, string path) - { - IConfigService configService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - MessageEntities.MessageCollection messages = new(); - - await AnsiConsole.Status() - .StartAsync("[red]Getting Messages[/]", - async ctx => { messages = await apiService.GetMessages($"/chats/{user.Value}/messages", path, ctx); }); - - if (messages != null && messages.Messages.Count > 0) - { - AnsiConsole.Markup( - $"[red]Found {messages.Messages.Count} Media from {messages.MessageObjects.Count} Messages\n[/]"); - - long totalSize = 0; - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(messages.Messages.Values.ToList()); - } - else - { - totalSize = messages.Messages.Count; - } - - DownloadResult result = null; - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - ProgressTask task = ctx.AddTask($"[red]Downloading {messages.Messages.Count} Messages[/]", false); - task.MaxValue = totalSize; - task.StartTask(); - - SpectreProgressReporter progressReporter = new(task); - result = await downloadService.DownloadMessages(username, user.Value, path, - hasSelectedUsersKVP.Value, clientIdBlobMissing, devicePrivateKeyMissing, messages, - progressReporter); - - task.StopTask(); - }); - - AnsiConsole.Markup( - $"[red]Messages Already Downloaded: {result.ExistingDownloads} New Messages Downloaded: {result.NewDownloads}[/]\n"); - return result.TotalCount; - } - - AnsiConsole.Markup("[red]Found 0 Messages\n[/]"); - return 0; - } - - private async Task DownloadHighlights(string username, KeyValuePair user, int highlightsCount, - string path) - { - IConfigService configService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - - AnsiConsole.Markup("[red]Getting Highlights\n[/]"); - - // Calculate total size for progress bar - long totalSize = 0; - Dictionary? tempHighlights = await apiService.GetMedia(MediaType.Highlights, - $"/users/{user.Value}/stories/highlights", null, path, paid_post_ids); - if (tempHighlights != null && tempHighlights.Count > 0) - { - AnsiConsole.Markup($"[red]Found {tempHighlights.Count} Highlights\n[/]"); - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(tempHighlights.Values.ToList()); - } - else - { - totalSize = tempHighlights.Count; - } - } - else - { - AnsiConsole.Markup("[red]Found 0 Highlights\n[/]"); - return 0; - } - - DownloadResult result = null; - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - ProgressTask task = ctx.AddTask($"[red]Downloading {tempHighlights.Count} Highlights[/]", false); - task.MaxValue = totalSize; - task.StartTask(); - - SpectreProgressReporter progressReporter = new(task); - result = await downloadService.DownloadHighlights(username, user.Value, path, paid_post_ids.ToHashSet(), - progressReporter); - - task.StopTask(); - }); - AnsiConsole.Markup( - $"[red]Highlights Already Downloaded: {result.ExistingDownloads} New Highlights Downloaded: {result.NewDownloads}[/]\n"); - return result.TotalCount; - } - - private async Task DownloadStories(string username, KeyValuePair user, int storiesCount, - string path) - { - IConfigService configService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - AnsiConsole.Markup("[red]Getting Stories\n[/]"); - - // Calculate total size for progress bar - long totalSize = 0; - Dictionary? tempStories = await serviceProvider.GetRequiredService() - .GetMedia(MediaType.Stories, $"/users/{user.Value}/stories", null, path, paid_post_ids); - if (tempStories != null && tempStories.Count > 0) - { - AnsiConsole.Markup($"[red]Found {tempStories.Count} Stories\n[/]"); - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(tempStories.Values.ToList()); - } - else - { - totalSize = tempStories.Count; - } - } - else - { - AnsiConsole.Markup("[red]Found 0 Stories\n[/]"); - return 0; - } - - DownloadResult result = null; - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - ProgressTask task = ctx.AddTask($"[red]Downloading {tempStories.Count} Stories[/]", false); - task.MaxValue = totalSize; - task.StartTask(); - - SpectreProgressReporter progressReporter = new(task); - result = await downloadService.DownloadStories(username, user.Value, path, paid_post_ids.ToHashSet(), - progressReporter); - - task.StopTask(); - }); - - AnsiConsole.Markup( - $"[red]Stories Already Downloaded: {result.ExistingDownloads} New Stories Downloaded: {result.NewDownloads}[/]\n"); - return result.TotalCount; - } - - private async Task DownloadArchived(string username, - KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, - int archivedCount, string path) - { - IConfigService configService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - ArchivedEntities.ArchivedCollection archived = new(); - - await AnsiConsole.Status() - .StartAsync("[red]Getting Archived Posts[/]", - async ctx => { archived = await apiService.GetArchived($"/users/{user.Value}/posts", path, ctx); }); - - if (archived != null && archived.ArchivedPosts.Count > 0) - { - AnsiConsole.Markup( - $"[red]Found {archived.ArchivedPosts.Count} Media from {archived.ArchivedPostObjects.Count} Archived Posts\n[/]"); - - long totalSize = 0; - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(archived.ArchivedPosts.Values.ToList()); - } - else - { - totalSize = archived.ArchivedPosts.Count; - } - - DownloadResult result = null; - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => + "[red]To find an individual post URL, click on the ... at the top right corner of the post and select 'Copy link to post'.\n\nTo return to the main menu, enter 'back' or 'exit' when prompted for the URL.\n\n[/]"); + string postUrl = AnsiConsole.Prompt( + new TextPrompt("[red]Please enter a post URL: [/]") + .ValidationErrorMessage("[red]Please enter a valid post URL[/]") + .Validate(url => { - ProgressTask task = - ctx.AddTask($"[red]Downloading {archived.ArchivedPosts.Count} Archived Posts[/]", false); - task.MaxValue = totalSize; - task.StartTask(); - - SpectreProgressReporter progressReporter = new(task); - result = await downloadService.DownloadArchived(username, user.Value, path, - hasSelectedUsersKVP.Value, clientIdBlobMissing, devicePrivateKeyMissing, archived, - progressReporter); - - task.StopTask(); - }); - - AnsiConsole.Markup( - $"[red]Archived Posts Already Downloaded: {result.ExistingDownloads} New Archived Posts Downloaded: {result.NewDownloads}[/]\n"); - return result.TotalCount; - } - - AnsiConsole.Markup("[red]Found 0 Archived Posts\n[/]"); - return 0; - } - - private async Task DownloadFreePosts(string username, - KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, - int postCount, string path) - { - IConfigService configService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - AnsiConsole.Markup( - "[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/]"); - Log.Debug($"Calling DownloadFreePosts - {user.Key}"); - - PostEntities.PostCollection posts = new(); - - await AnsiConsole.Status() - .StartAsync("[red]Getting Posts[/]", - async ctx => - { - posts = await apiService.GetPosts($"/users/{user.Value}/posts", path, paid_post_ids, ctx); - }); - - if (posts == null || posts.Posts.Count <= 0) - { - AnsiConsole.Markup("[red]Found 0 Posts\n[/]"); - Log.Debug("Found 0 Posts"); - return 0; - } - - AnsiConsole.Markup($"[red]Found {posts.Posts.Count} Media from {posts.PostObjects.Count} Posts\n[/]"); - Log.Debug($"Found {posts.Posts.Count} Media from {posts.PostObjects.Count} Posts"); - - long totalSize = configService.CurrentConfig.ShowScrapeSize - ? await downloadService.CalculateTotalFileSize(posts.Posts.Values.ToList()) - : posts.Posts.Count; - - DownloadResult result = null; - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - ProgressTask task = ctx.AddTask($"[red]Downloading {posts.Posts.Count} Posts[/]", false); - task.MaxValue = totalSize; - task.StartTask(); - - SpectreProgressReporter progressReporter = new(task); - result = await downloadService.DownloadFreePosts(username, user.Value, path, hasSelectedUsersKVP.Value, - clientIdBlobMissing, devicePrivateKeyMissing, posts, progressReporter); - - task.StopTask(); - }); - - AnsiConsole.Markup( - $"[red]Posts Already Downloaded: {result.ExistingDownloads} New Posts Downloaded: {result.NewDownloads}[/]\n"); - Log.Debug($"Posts Already Downloaded: {result.ExistingDownloads} New Posts Downloaded: {result.NewDownloads}"); - - return result.TotalCount; - } - - private async Task DownloadPaidPosts(string username, - KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, - int paidPostCount, string path) - { - IConfigService configService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - AnsiConsole.Markup("[red]Getting Paid Posts\n[/]"); - Log.Debug($"Calling DownloadPaidPosts - {user.Key}"); - - PurchasedEntities.PaidPostCollection purchasedPosts = new(); - - await AnsiConsole.Status() - .StartAsync("[red]Getting Paid Posts[/]", - async ctx => - { - purchasedPosts = - await apiService.GetPaidPosts("/posts/paid/post", path, user.Key, paid_post_ids, ctx); - }); - - if (purchasedPosts == null || purchasedPosts.PaidPosts.Count <= 0) - { - AnsiConsole.Markup("[red]Found 0 Paid Posts\n[/]"); - Log.Debug("Found 0 Paid Posts"); - return 0; - } - - AnsiConsole.Markup( - $"[red]Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts\n[/]"); - Log.Debug( - $"Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts"); - - long totalSize = configService.CurrentConfig.ShowScrapeSize - ? await downloadService.CalculateTotalFileSize(purchasedPosts.PaidPosts.Values.ToList()) - : purchasedPosts.PaidPosts.Count; - - DownloadResult result = null; - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - ProgressTask task = ctx.AddTask($"[red]Downloading {purchasedPosts.PaidPosts.Count} Paid Posts[/]", - false); - task.MaxValue = totalSize; - task.StartTask(); - - SpectreProgressReporter progressReporter = new(task); - result = await downloadService.DownloadPaidPosts(username, user.Value, path, hasSelectedUsersKVP.Value, - clientIdBlobMissing, devicePrivateKeyMissing, purchasedPosts, progressReporter); - - task.StopTask(); - }); - - AnsiConsole.Markup( - $"[red]Paid Posts Already Downloaded: {result.ExistingDownloads} New Paid Posts Downloaded: {result.NewDownloads}[/]\n"); - Log.Debug( - $"Paid Posts Already Downloaded: {result.ExistingDownloads} New Paid Posts Downloaded: {result.NewDownloads}"); - - return result.TotalCount; - } - - private async Task DownloadPaidPostsPurchasedTab(string username, - PurchasedEntities.PaidPostCollection purchasedPosts, - KeyValuePair user, int paidPostCount, string path, Dictionary users) - { - IDBService dbService = serviceProvider.GetRequiredService(); - IConfigService configService = serviceProvider.GetRequiredService(); - IAuthService authService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - int oldPaidPostCount = 0; - int newPaidPostCount = 0; - if (purchasedPosts == null || purchasedPosts.PaidPosts.Count <= 0) - { - AnsiConsole.Markup("[red]Found 0 Paid Posts\n[/]"); - Log.Debug("Found 0 Paid Posts"); - return 0; - } - - AnsiConsole.Markup( - $"[red]Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts\n[/]"); - Log.Debug( - $"Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts"); - - paidPostCount = purchasedPosts.PaidPosts.Count; - long totalSize = 0; - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(purchasedPosts.PaidPosts.Values.ToList()); - } - else - { - totalSize = paidPostCount; - } - - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - // Define tasks - ProgressTask task = ctx.AddTask($"[red]Downloading {purchasedPosts.PaidPosts.Count} Paid Posts[/]", - false); - Log.Debug($"Downloading {purchasedPosts.PaidPosts.Count} Paid Posts"); - task.MaxValue = totalSize; - task.StartTask(); - foreach (KeyValuePair purchasedPostKVP in purchasedPosts.PaidPosts) - { - bool isNew; - MessageEntities.Medium? mediaInfo = - purchasedPosts?.PaidPostMedia?.FirstOrDefault(m => m.Id == purchasedPostKVP.Key); - PurchasedEntities.ListItem? postInfo = mediaInfo != null - ? purchasedPosts?.PaidPostObjects?.FirstOrDefault(p => - p?.Media?.Any(m => m.Id == purchasedPostKVP.Key) == true) - : null; - string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) - .PaidPostFileNameFormat ?? ""; - string paidPostPath = configService.CurrentConfig.FolderPerPaidPost && postInfo != null && - postInfo?.Id is not null && postInfo?.PostedAt is not null - ? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}" - : "/Posts/Paid"; - - if (purchasedPostKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) + Log.Debug($"Single Post URL: {url}"); + Regex regex = new("https://onlyfans\\.com/[0-9]+/[A-Za-z0-9]+", RegexOptions.IgnoreCase); + if (regex.IsMatch(url)) { - string[] parsed = purchasedPostKVP.Value.Split(','); - (string decryptionKey, DateTime lastModified)? drmInfo = - await downloadService.GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], - parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); - if (drmInfo == null) - { - continue; - } - - isNew = await downloadService.DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], - drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, purchasedPostKVP.Key, - "Posts", - new SpectreProgressReporter(task), paidPostPath + "/Videos", filenameFormat, - postInfo, mediaInfo, postInfo?.FromUser, users); - } - else - { - isNew = await downloadService.DownloadMedia(purchasedPostKVP.Value, path, - purchasedPostKVP.Key, "Posts", new SpectreProgressReporter(task), - paidPostPath, filenameFormat, postInfo, mediaInfo, postInfo?.FromUser, users); + return ValidationResult.Success(); } - if (isNew) + if (url == "" || url == "exit" || url == "back") { - newPaidPostCount++; - } - else - { - oldPaidPostCount++; + return ValidationResult.Success(); } + + Log.Error("Post URL invalid"); + return ValidationResult.Error("[red]Please enter a valid post URL[/]"); + })); + + if (postUrl != "" && postUrl != "exit" && postUrl != "back") + { + long postId = Convert.ToInt64(postUrl.Split("/")[3]); + string username = postUrl.Split("/")[4]; + + Log.Debug($"Single Post ID: {postId}"); + Log.Debug($"Single Post Creator: {username}"); + + if (users.ContainsKey(username)) + { + string path = orchestrationService.ResolveDownloadPath(username); + Log.Debug($"Download path: {path}"); + + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + AnsiConsole.Markup($"[red]Created folder for {Markup.Escape(username)}\n[/]"); + Log.Debug($"Created folder for {username}"); + } + else + { + AnsiConsole.Markup($"[red]Folder for {Markup.Escape(username)} already created\n[/]"); } - task.StopTask(); - }); + IDBService dbService = serviceProvider.GetRequiredService(); + await dbService.CreateDB(path); + + await orchestrationService.DownloadSinglePostAsync(username, postId, path, users, + startupResult.ClientIdBlobMissing, startupResult.DevicePrivateKeyMissing, eventHandler); + } + } + } + + private async Task HandleSingleMessageDownload( + IDownloadOrchestrationService orchestrationService, + Dictionary users, + StartupResult startupResult, + IDownloadEventHandler eventHandler) + { AnsiConsole.Markup( - $"[red]Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}[/]\n"); - Log.Debug($"Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}"); - return paidPostCount; - } + "[red]To find an individual message URL, note that you can only do so for PPV messages that you have unlocked. Go the main OnlyFans timeline, click on the Purchased tab, find the relevant message, click on the ... at the top right corner of the message, and select 'Copy link to message'. For all other messages, you cannot scrape them individually, you must scrape all messages from that creator.\n\nTo return to the main menu, enter 'back' or 'exit' when prompted for the URL.\n\n[/]"); + string messageUrl = AnsiConsole.Prompt( + new TextPrompt("[red]Please enter a message URL: [/]") + .ValidationErrorMessage("[red]Please enter a valid message URL[/]") + .Validate(url => + { + Log.Debug($"Single Paid Message URL: {url}"); + Regex regex = new("https://onlyfans\\.com/my/chats/chat/[0-9]+/\\?firstId=[0-9]+$", + RegexOptions.IgnoreCase); + if (regex.IsMatch(url)) + { + return ValidationResult.Success(); + } - private async Task DownloadPaidMessagesPurchasedTab(string username, - PurchasedEntities.PaidMessageCollection paidMessageCollection, KeyValuePair user, - int paidMessagesCount, - string path, Dictionary users) - { - IDBService dbService = serviceProvider.GetRequiredService(); - IConfigService configService = serviceProvider.GetRequiredService(); - IAuthService authService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); + if (url == "" || url == "back" || url == "exit") + { + return ValidationResult.Success(); + } - int oldPaidMessagesCount = 0; - int newPaidMessagesCount = 0; - if (paidMessageCollection != null && paidMessageCollection.PaidMessages.Count > 0) + Log.Error("Message URL invalid"); + return ValidationResult.Error("[red]Please enter a valid message URL[/]"); + })); + + if (messageUrl != "" && messageUrl != "exit" && messageUrl != "back") { - AnsiConsole.Markup( - $"[red]Found {paidMessageCollection.PaidMessages.Count} Media from {paidMessageCollection.PaidMessageObjects.Count} Paid Messages\n[/]"); - Log.Debug( - $"Found {paidMessageCollection.PaidMessages.Count} Media from {paidMessageCollection.PaidMessageObjects.Count} Paid Messages"); - paidMessagesCount = paidMessageCollection.PaidMessages.Count; - long totalSize = 0; - if (configService.CurrentConfig.ShowScrapeSize) + long messageId = Convert.ToInt64(messageUrl.Split("?firstId=")[1]); + long userId = Convert.ToInt64(messageUrl.Split("/")[6]); + + Log.Debug($"Message ID: {messageId}"); + Log.Debug($"User ID: {userId}"); + + string? username = await orchestrationService.ResolveUsernameAsync(userId); + Log.Debug("Content creator: {Username}", username); + + if (username == null) { - totalSize = await downloadService.CalculateTotalFileSize(paidMessageCollection.PaidMessages.Values - .ToList()); + Log.Error("Could not resolve username for user ID: {userId}", userId); + AnsiConsole.MarkupLine("[red]Could not resolve username for user ID[/]"); + return; + } + + string path = orchestrationService.ResolveDownloadPath(username); + Log.Debug($"Download path: {path}"); + + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + AnsiConsole.Markup($"[red]Created folder for {Markup.Escape(username)}\n[/]"); + Log.Debug($"Created folder for {username}"); } else { - totalSize = paidMessagesCount; + AnsiConsole.Markup($"[red]Folder for {Markup.Escape(username)} already created\n[/]"); + Log.Debug($"Folder for {username} already created"); } - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - // Define tasks - ProgressTask task = - ctx.AddTask($"[red]Downloading {paidMessageCollection.PaidMessages.Count} Paid Messages[/]", - false); - Log.Debug($"Downloading {paidMessageCollection.PaidMessages.Count} Paid Messages"); - task.MaxValue = totalSize; - task.StartTask(); - foreach (KeyValuePair paidMessageKVP in paidMessageCollection.PaidMessages) - { - bool isNew; - MessageEntities.Medium? mediaInfo = - paidMessageCollection.PaidMessageMedia.FirstOrDefault(m => m.Id == paidMessageKVP.Key); - PurchasedEntities.ListItem? messageInfo = - paidMessageCollection.PaidMessageObjects.FirstOrDefault(p => - p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true); - string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) - .PaidMessageFileNameFormat ?? ""; - string paidMsgPath = configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null && - messageInfo?.Id is not null && messageInfo?.CreatedAt is not null - ? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" - : "/Messages/Paid"; + IDBService dbService = serviceProvider.GetRequiredService(); + await dbService.CreateDB(path); - if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) - { - string[] parsed = paidMessageKVP.Value.Split(','); - (string decryptionKey, DateTime lastModified)? drmInfo = - await downloadService.GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], - parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); - if (drmInfo == null) - { - continue; - } - - isNew = await downloadService.DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], - drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKVP.Key, - "Messages", - new SpectreProgressReporter(task), paidMsgPath + "/Videos", filenameFormat, - messageInfo, mediaInfo, messageInfo?.FromUser, users); - } - else - { - isNew = await downloadService.DownloadMedia(paidMessageKVP.Value, path, - paidMessageKVP.Key, "Messages", new SpectreProgressReporter(task), - paidMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users); - } - - if (isNew) - { - newPaidMessagesCount++; - } - else - { - oldPaidMessagesCount++; - } - } - - task.StopTask(); - }); - AnsiConsole.Markup( - $"[red]Paid Messages Already Downloaded: {oldPaidMessagesCount} New Paid Messages Downloaded: {newPaidMessagesCount}[/]\n"); - Log.Debug( - $"[red]Paid Messages Already Downloaded: {oldPaidMessagesCount} New Paid Messages Downloaded: {newPaidMessagesCount}"); - } - else - { - AnsiConsole.Markup("[red]Found 0 Paid Messages\n[/]"); - Log.Debug("Found 0 Paid Messages"); - } - - return paidMessagesCount; - } - - private async Task DownloadStreams(string username, - KeyValuePair> hasSelectedUsersKVP, KeyValuePair user, - int streamsCount, string path) - { - IConfigService configService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - StreamEntities.StreamsCollection streams = new(); - - await AnsiConsole.Status() - .StartAsync("[red]Getting Streams[/]", - async ctx => - { - streams = await apiService.GetStreams($"/users/{user.Value}/posts/streams", path, paid_post_ids, - ctx); - }); - - if (streams != null && streams.Streams.Count > 0) - { - AnsiConsole.Markup( - $"[red]Found {streams.Streams.Count} Media from {streams.StreamObjects.Count} Streams\n[/]"); - - long totalSize = 0; - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(streams.Streams.Values.ToList()); - } - else - { - totalSize = streams.Streams.Count; - } - - DownloadResult result = null; - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - ProgressTask task = ctx.AddTask($"[red]Downloading {streams.Streams.Count} Streams[/]", false); - task.MaxValue = totalSize; - task.StartTask(); - - SpectreProgressReporter progressReporter = new(task); - result = await downloadService.DownloadStreams(username, user.Value, path, - hasSelectedUsersKVP.Value, clientIdBlobMissing, devicePrivateKeyMissing, streams, - progressReporter); - - task.StopTask(); - }); - - AnsiConsole.Markup( - $"[red]Streams Already Downloaded: {result.ExistingDownloads} New Streams Downloaded: {result.NewDownloads}[/]\n"); - return result.TotalCount; - } - - AnsiConsole.Markup("[red]Found 0 Streams\n[/]"); - return 0; - } - - private async Task DownloadPaidMessage(string username, - KeyValuePair> hasSelectedUsersKVP, int paidMessagesCount, string path, - long message_id) - { - IDBService dbService = serviceProvider.GetRequiredService(); - IConfigService configService = serviceProvider.GetRequiredService(); - IAuthService authService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) - .PaidMessageFileNameFormat ?? ""; - - Log.Debug($"Calling DownloadPaidMessage - {username}"); - - AnsiConsole.Markup("[red]Getting Paid Message\n[/]"); - - PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection = - await apiService.GetPaidMessage($"/messages/{message_id.ToString()}", path); - int oldPreviewPaidMessagesCount = 0; - int newPreviewPaidMessagesCount = 0; - int oldPaidMessagesCount = 0; - int newPaidMessagesCount = 0; - if (singlePaidMessageCollection != null && singlePaidMessageCollection.PreviewSingleMessages.Count > 0) - { - AnsiConsole.Markup( - $"[red]Found {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages\n[/]"); - Log.Debug( - $"Found {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages"); - paidMessagesCount = singlePaidMessageCollection.SingleMessages.Count; - long totalSize = 0; - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(singlePaidMessageCollection - .PreviewSingleMessages.Values.ToList()); - } - else - { - totalSize = paidMessagesCount; - } - - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - // Define tasks - ProgressTask task = - ctx.AddTask( - $"[red]Downloading {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Paid Messages[/]", - false); - Log.Debug($"Downloading {singlePaidMessageCollection.PreviewSingleMessages.Count} Paid Messages"); - task.MaxValue = totalSize; - task.StartTask(); - foreach (KeyValuePair paidMessageKVP in singlePaidMessageCollection - .PreviewSingleMessages) - { - bool isNew; - MessageEntities.Medium? mediaInfo = - singlePaidMessageCollection.PreviewSingleMessageMedia.FirstOrDefault(m => - m.Id == paidMessageKVP.Key); - MessageEntities.SingleMessage? messageInfo = - singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p => - p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true); - - string previewMsgPath = configService.CurrentConfig.FolderPerMessage && messageInfo != null && - messageInfo?.Id is not null && messageInfo?.CreatedAt is not null - ? $"/Messages/Free/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" - : "/Messages/Free"; - - if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) - { - string[] parsed = paidMessageKVP.Value.Split(','); - (string decryptionKey, DateTime lastModified)? drmInfo = - await downloadService.GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], - parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); - if (drmInfo == null) - { - continue; - } - - isNew = await downloadService.DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], - drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKVP.Key, - "Messages", - new SpectreProgressReporter(task), previewMsgPath + "/Videos", filenameFormat, - messageInfo, mediaInfo, messageInfo?.FromUser, hasSelectedUsersKVP.Value); - } - else - { - isNew = await downloadService.DownloadMedia(paidMessageKVP.Value, path, - paidMessageKVP.Key, "Messages", new SpectreProgressReporter(task), - previewMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, - hasSelectedUsersKVP.Value); - } - - if (isNew) - { - newPreviewPaidMessagesCount++; - } - else - { - oldPreviewPaidMessagesCount++; - } - } - - task.StopTask(); - }); - AnsiConsole.Markup( - $"[red]Preview Paid Messages Already Downloaded: {oldPreviewPaidMessagesCount} New Preview Paid Messages Downloaded: {newPreviewPaidMessagesCount}[/]\n"); - Log.Debug( - $"Preview Paid Messages Already Downloaded: {oldPreviewPaidMessagesCount} New Preview Paid Messages Downloaded: {newPreviewPaidMessagesCount}"); - } - - if (singlePaidMessageCollection != null && singlePaidMessageCollection.SingleMessages.Count > 0) - { - AnsiConsole.Markup( - $"[red]Found {singlePaidMessageCollection.SingleMessages.Count} Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages\n[/]"); - Log.Debug( - $"Found {singlePaidMessageCollection.SingleMessages.Count} Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages"); - paidMessagesCount = singlePaidMessageCollection.SingleMessages.Count; - long totalSize = 0; - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(singlePaidMessageCollection.SingleMessages - .Values.ToList()); - } - else - { - totalSize = paidMessagesCount; - } - - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - // Define tasks - ProgressTask task = - ctx.AddTask( - $"[red]Downloading {singlePaidMessageCollection.SingleMessages.Count} Paid Messages[/]", - false); - Log.Debug($"Downloading {singlePaidMessageCollection.SingleMessages.Count} Paid Messages"); - task.MaxValue = totalSize; - task.StartTask(); - foreach (KeyValuePair paidMessageKVP in singlePaidMessageCollection.SingleMessages) - { - bool isNew; - MessageEntities.Medium? mediaInfo = - singlePaidMessageCollection.SingleMessageMedia.FirstOrDefault(m => - m.Id == paidMessageKVP.Key); - MessageEntities.SingleMessage? messageInfo = - singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p => - p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true); - string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) - .PaidMessageFileNameFormat ?? ""; - string singlePaidMsgPath = configService.CurrentConfig.FolderPerPaidMessage && - messageInfo != null && messageInfo?.Id is not null && - messageInfo?.CreatedAt is not null - ? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" - : "/Messages/Paid"; - - if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) - { - string[] parsed = paidMessageKVP.Value.Split(','); - (string decryptionKey, DateTime lastModified)? drmInfo = - await downloadService.GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], - parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); - if (drmInfo == null) - { - continue; - } - - isNew = await downloadService.DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], - drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKVP.Key, - "Messages", - new SpectreProgressReporter(task), singlePaidMsgPath + "/Videos", filenameFormat, - messageInfo, mediaInfo, messageInfo?.FromUser, hasSelectedUsersKVP.Value); - } - else - { - isNew = await downloadService.DownloadMedia(paidMessageKVP.Value, path, - paidMessageKVP.Key, "Messages", new SpectreProgressReporter(task), - singlePaidMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, - hasSelectedUsersKVP.Value); - if (isNew) - { - newPaidMessagesCount++; - } - else - { - oldPaidMessagesCount++; - } - } - } - - task.StopTask(); - }); - AnsiConsole.Markup( - $"[red]Paid Messages Already Downloaded: {oldPaidMessagesCount} New Paid Messages Downloaded: {newPaidMessagesCount}[/]\n"); - Log.Debug( - $"Paid Messages Already Downloaded: {oldPaidMessagesCount} New Paid Messages Downloaded: {newPaidMessagesCount}"); - } - else - { - AnsiConsole.Markup("[red]Found 0 Paid Messages\n[/]"); - Log.Debug("Found 0 Paid Messages"); - } - - return paidMessagesCount; - } - - private async Task DownloadSinglePost(string username, long post_id, string path, Dictionary users) - { - IConfigService configService = serviceProvider.GetRequiredService(); - IAPIService apiService = serviceProvider.GetRequiredService(); - IDownloadService downloadService = serviceProvider.GetRequiredService(); - - Log.Debug($"Calling DownloadSinglePost - {post_id.ToString()}"); - - AnsiConsole.Markup("[red]Getting Post\n[/]"); - PostEntities.SinglePostCollection post = await apiService.GetPost($"/posts/{post_id.ToString()}", path); - if (post == null) - { - AnsiConsole.Markup("[red]Couldn't find post\n[/]"); - Log.Debug("Couldn't find post"); - return; - } - - long totalSize = 0; - if (configService.CurrentConfig.ShowScrapeSize) - { - totalSize = await downloadService.CalculateTotalFileSize(post.SinglePosts.Values.ToList()); - } - else - { - totalSize = post.SinglePosts.Count; - } - - bool isNew = false; - await AnsiConsole.Progress() - .Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize)) - .StartAsync(async ctx => - { - ProgressTask task = ctx.AddTask("[red]Downloading Post[/]", false); - task.MaxValue = totalSize; - task.StartTask(); - foreach (KeyValuePair postKVP in post.SinglePosts) - { - PostEntities.Medium? mediaInfo = post.SinglePostMedia.FirstOrDefault(m => m.Id == postKVP.Key); - PostEntities.SinglePost? postInfo = - post.SinglePostObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); - string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) - .PostFileNameFormat ?? ""; - string postPath = configService.CurrentConfig.FolderPerPost && postInfo != null && - postInfo?.Id is not null && postInfo?.PostedAt is not null - ? $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}" - : "/Posts/Free"; - - if (postKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) - { - string[] parsed = postKVP.Value.Split(','); - (string decryptionKey, DateTime lastModified)? drmInfo = - await downloadService.GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], - parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); - if (drmInfo == null) - { - continue; - } - - isNew = await downloadService.DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], - drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKVP.Key, "Posts", - new SpectreProgressReporter(task), postPath + "/Videos", filenameFormat, - postInfo, mediaInfo, postInfo?.Author, users); - } - else - { - try - { - isNew = await downloadService.DownloadMedia(postKVP.Value, path, - postKVP.Key, "Posts", new SpectreProgressReporter(task), - postPath, filenameFormat, postInfo, mediaInfo, postInfo?.Author, users); - } - catch - { - Console.WriteLine("Media was null"); - } - } - } - - task.StopTask(); - }); - if (isNew) - { - AnsiConsole.Markup($"[red]Post {post_id} downloaded\n[/]"); - Log.Debug($"Post {post_id} downloaded"); - } - else - { - AnsiConsole.Markup($"[red]Post {post_id} already downloaded\n[/]"); - Log.Debug($"Post {post_id} already downloaded"); + await orchestrationService.DownloadSinglePaidMessageAsync(username, messageId, path, users, + startupResult.ClientIdBlobMissing, startupResult.DevicePrivateKeyMissing, eventHandler); } } @@ -2023,7 +482,7 @@ public class Program(IServiceProvider serviceProvider) bool hasSelectedUsers = false; Dictionary selectedUsers = new(); - Config currentConfig = configService.CurrentConfig!; + Config currentConfig = configService.CurrentConfig; while (!hasSelectedUsers) { @@ -2058,7 +517,7 @@ public class Program(IServiceProvider serviceProvider) if (listSelection.Contains("[red]Go Back[/]")) { - break; // Go back to the main menu + break; } hasSelectedUsers = true; @@ -2066,7 +525,7 @@ public class Program(IServiceProvider serviceProvider) foreach (string item in listSelection) { long listId = lists[item.Replace("[red]", "").Replace("[/]", "")]; - List usernames = await apiService.GetListUsers($"/lists/{listId}/users"); + List usernames = await apiService.GetListUsers($"/lists/{listId}/users") ?? []; foreach (string user in usernames) { listUsernames.Add(user); @@ -2099,7 +558,7 @@ public class Program(IServiceProvider serviceProvider) List userSelection = AnsiConsole.Prompt(selectedNamesPrompt); if (userSelection.Contains("[red]Go Back[/]")) { - break; // Go back to the main menu + break; } hasSelectedUsers = true; @@ -2118,22 +577,11 @@ public class Program(IServiceProvider serviceProvider) case "[red]Edit config.conf[/]": while (true) { - if (currentConfig == null) - { - currentConfig = new Config(); - } - + List<(string Name, bool Value)> toggleableProps = configService.GetToggleableProperties(); List<(string choice, bool isSelected)> choices = new() { ("[red]Go Back[/]", false) }; - - foreach (PropertyInfo propInfo in typeof(Config).GetProperties()) + foreach ((string Name, bool Value) prop in toggleableProps) { - ToggleableConfigAttribute? attr = propInfo.GetCustomAttribute(); - if (attr != null) - { - string itemLabel = $"[red]{propInfo.Name}[/]"; - choices.Add(new ValueTuple(itemLabel, - (bool)propInfo.GetValue(currentConfig)!)); - } + choices.Add(($"[red]{prop.Name}[/]", prop.Value)); } MultiSelectionPrompt multiSelectionPrompt = new MultiSelectionPrompt() @@ -2158,41 +606,15 @@ public class Program(IServiceProvider serviceProvider) break; } - bool configChanged = false; + // Extract plain names from selections + List selectedNames = configOptions + .Select(o => o.Replace("[red]", "").Replace("[/]", "")) + .ToList(); - Config newConfig = new(); - foreach (PropertyInfo propInfo in typeof(Config).GetProperties()) - { - ToggleableConfigAttribute? attr = propInfo.GetCustomAttribute(); - if (attr != null) - { - // - // Get the new choice from the selection - // - string itemLabel = $"[red]{propInfo.Name}[/]"; - bool newValue = configOptions.Contains(itemLabel); - bool oldValue = choices.Where(c => c.choice == itemLabel).Select(c => c.isSelected) - .First(); - propInfo.SetValue(newConfig, newValue); - - if (newValue != oldValue) - { - configChanged = true; - } - } - else - { - // - // Reassign any non toggleable values - // - propInfo.SetValue(newConfig, propInfo.GetValue(currentConfig)); - } - } - - configService.UpdateConfig(newConfig); + bool configChanged = configService.ApplyToggleableSelections(selectedNames); await configService.SaveConfigurationAsync(); + currentConfig = configService.CurrentConfig; - currentConfig = newConfig; if (configChanged) { return (true, new Dictionary { { "ConfigChanged", 0 } }); @@ -2205,7 +627,7 @@ public class Program(IServiceProvider serviceProvider) case "[red]Change logging level[/]": while (true) { - List<(string choice, bool isSelected)> choices = new() { ("[red]Go Back[/]", false) }; + List<(string choice, bool isSelected)> choices = [("[red]Go Back[/]", false)]; foreach (string name in typeof(LoggingLevel).GetEnumNames()) { @@ -2231,52 +653,31 @@ public class Program(IServiceProvider serviceProvider) } levelOption = levelOption.Replace("[red]", "").Replace("[/]", ""); - LoggingLevel newLogLevel = (LoggingLevel)Enum.Parse(typeof(LoggingLevel), levelOption, true); + LoggingLevel newLogLevel = + (LoggingLevel)Enum.Parse(typeof(LoggingLevel), levelOption, true); Log.Debug($"Logging level changed to: {levelOption}"); - bool configChanged = false; - - Config newConfig = new(); - - newConfig = currentConfig; - + Config newConfig = currentConfig; newConfig.LoggingLevel = newLogLevel; - currentConfig = newConfig; configService.UpdateConfig(newConfig); await configService.SaveConfigurationAsync(); - if (configChanged) - { - return (true, new Dictionary { { "ConfigChanged", 0 } }); - } - break; } break; case "[red]Logout and Exit[/]": - if (Directory.Exists("chrome-data")) - { - Log.Information("Deleting chrome-data folder"); - Directory.Delete("chrome-data", true); - } - - if (File.Exists("auth.json")) - { - Log.Information("Deleting auth.json"); - File.Delete("auth.json"); - } - - return (false, null); // Return false to indicate exit + authService.Logout(); + return (false, null); case "[red]Exit[/]": - return (false, null); // Return false to indicate exit + return (false, null); } } - return (true, selectedUsers); // Return true to indicate selected users + return (true, selectedUsers); } public static List GetMainMenuOptions(Dictionary users, Dictionary lists) @@ -2312,98 +713,182 @@ public class Program(IServiceProvider serviceProvider) }; } - private static bool ValidateFilePath(string path) + private async Task HandleAuthFlow(IAuthService authService, IConfigService configService) { - char[] invalidChars = Path.GetInvalidPathChars(); - char[] foundInvalidChars = path.Where(c => invalidChars.Contains(c)).ToArray(); - - if (foundInvalidChars.Any()) + if (await authService.LoadFromFileAsync()) { - AnsiConsole.Markup( - $"[red]Invalid characters found in path {path}:[/] {string.Join(", ", foundInvalidChars)}\n"); - return false; + AnsiConsole.Markup("[green]auth.json located successfully!\n[/]"); } - - if (!File.Exists(path)) + else if (File.Exists("auth.json")) { - if (Directory.Exists(path)) + Log.Information("Auth file found but could not be deserialized"); + if (!configService.CurrentConfig.DisableBrowserAuth) { - AnsiConsole.Markup( - $"[red]The provided path {path} improperly points to a directory and not a file.[/]\n"); + Log.Debug("Deleting auth.json"); + File.Delete("auth.json"); + } + + if (configService.CurrentConfig.NonInteractiveMode) + { + AnsiConsole.MarkupLine( + "\n[red]auth.json has invalid JSON syntax. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n"); + AnsiConsole.MarkupLine( + "[red]You may also want to try using the browser extension which is documented here:[/]\n"); + AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]"); + AnsiConsole.MarkupLine("[red]Press any key to exit.[/]"); + + Console.ReadKey(); + Environment.Exit(2); + } + + if (!configService.CurrentConfig.DisableBrowserAuth) + { + await LoadAuthFromBrowser(); } else { - AnsiConsole.Markup($"[red]The provided path {path} does not exist or is not accessible.[/]\n"); + ShowAuthMissingError(); } - - return false; - } - - return true; - } - - private static ProgressColumn[] GetProgressColumns(bool showScrapeSize) - { - List progressColumns; - if (showScrapeSize) - { - progressColumns = new List - { - new TaskDescriptionColumn(), - new ProgressBarColumn(), - new PercentageColumn(), - new DownloadedColumn(), - new RemainingTimeColumn() - }; } else { - progressColumns = new List + if (configService.CurrentConfig.NonInteractiveMode) { - new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn() - }; + ShowAuthMissingError(); + } + else if (!configService.CurrentConfig.DisableBrowserAuth) + { + await LoadAuthFromBrowser(); + } + else + { + ShowAuthMissingError(); + } } - - return progressColumns.ToArray(); } - public static string? GetFullPath(string filename) + private static void ShowAuthMissingError() { - if (File.Exists(filename)) + AnsiConsole.MarkupLine( + "\n[red]auth.json is missing. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n"); + AnsiConsole.MarkupLine( + "[red]You may also want to try using the browser extension which is documented here:[/]\n"); + AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]"); + AnsiConsole.MarkupLine("[red]Press any key to exit.[/]"); + + Console.ReadKey(); + Environment.Exit(2); + } + + private static void DisplayVersionResult(VersionCheckResult result) + { + if (result.TimedOut) { - return Path.GetFullPath(filename); + AnsiConsole.Markup("[yellow]Version check timed out after 30 seconds.\n[/]"); + return; } - string pathEnv = Environment.GetEnvironmentVariable("PATH") ?? ""; - foreach (string path in pathEnv.Split(Path.PathSeparator)) + if (result.CheckFailed) { - string fullPath = Path.Combine(path, filename); - if (File.Exists(fullPath)) + AnsiConsole.Markup("[yellow]Failed to verify that OF-DL is up-to-date.\n[/]"); + return; + } + + if (result.LocalVersion == null || result.LatestVersion == null) + { + // Debug mode or no version info + AnsiConsole.Markup("[yellow]Running in Debug/Local mode. Version check skipped.\n[/]"); + return; + } + + if (result.IsUpToDate) + { + AnsiConsole.Markup("[green]You are running OF-DL version " + + $"{result.LocalVersion.Major}.{result.LocalVersion.Minor}.{result.LocalVersion.Build}\n[/]"); + AnsiConsole.Markup("[green]Latest Release version: " + + $"{result.LatestVersion.Major}.{result.LatestVersion.Minor}.{result.LatestVersion.Build}\n[/]"); + } + else + { + AnsiConsole.Markup("[red]You are running OF-DL version " + + $"{result.LocalVersion.Major}.{result.LocalVersion.Minor}.{result.LocalVersion.Build}\n[/]"); + AnsiConsole.Markup("[red]Please update to the current release, " + + $"{result.LatestVersion.Major}.{result.LatestVersion.Minor}.{result.LatestVersion.Build}: [link=https://git.ofdl.tools/sim0n00ps/OF-DL/releases]https://git.ofdl.tools/sim0n00ps/OF-DL/releases[/]\n[/]"); + } + } + + private static void DisplayStartupResult(StartupResult result) + { + // OS + if (result.IsWindowsVersionValid && result.OsVersionString != null && + Environment.OSVersion.Platform == PlatformID.Win32NT) + { + AnsiConsole.Markup("[green]Valid version of Windows found.\n[/]"); + } + + // FFmpeg + if (result.FfmpegFound) + { + if (result.FfmpegPathAutoDetected && result.FfmpegPath != null) { - return fullPath; + AnsiConsole.Markup( + $"[green]FFmpeg located successfully. Path auto-detected: {Markup.Escape(result.FfmpegPath)}\n[/]"); + } + else + { + AnsiConsole.Markup("[green]FFmpeg located successfully\n[/]"); + } + + if (result.FfmpegVersion != null) + { + AnsiConsole.Markup($"[green]ffmpeg version detected as {Markup.Escape(result.FfmpegVersion)}[/]\n"); + } + else + { + AnsiConsole.Markup("[yellow]ffmpeg version could not be parsed[/]\n"); } } - return null; - } - - public static void ValidateCookieString(Auth auth) - { - string pattern = @"(auth_id=\d+)|(sess=[^;]+)"; - MatchCollection matches = Regex.Matches(auth.Cookie, pattern); - - string output = string.Join("; ", matches); - - if (!output.EndsWith(";")) + // Widevine + if (!result.ClientIdBlobMissing) { - output += ";"; + AnsiConsole.Markup("[green]device_client_id_blob located successfully![/]\n"); } - if (auth.Cookie.Trim() != output.Trim()) + if (!result.DevicePrivateKeyMissing) { - auth.Cookie = output; - string newAuthString = JsonConvert.SerializeObject(auth, Formatting.Indented); - File.WriteAllText("auth.json", newAuthString); + AnsiConsole.Markup("[green]device_private_key located successfully![/]\n"); + } + + if (result.ClientIdBlobMissing || result.DevicePrivateKeyMissing) + { + AnsiConsole.Markup( + "[yellow]device_client_id_blob and/or device_private_key missing, https://ofdl.tools/ or https://cdrm-project.com/ will be used instead for DRM protected videos\n[/]"); + } + } + + private static void DisplayRulesJsonResult(StartupResult result, IConfigService configService) + { + if (result.RulesJsonExists) + { + if (result.RulesJsonValid) + { + AnsiConsole.Markup("[green]rules.json located successfully!\n[/]"); + } + else + { + AnsiConsole.MarkupLine("\n[red]rules.json is not valid, check your JSON syntax![/]\n"); + AnsiConsole.MarkupLine("[red]Please ensure you are using the latest version of the software.[/]\n"); + AnsiConsole.MarkupLine("[red]Press any key to exit.[/]"); + Log.Error("rules.json processing failed: {Error}", result.RulesJsonError); + + if (!configService.CurrentConfig.NonInteractiveMode) + { + Console.ReadKey(); + } + + Environment.Exit(2); + } } } } diff --git a/OF DL/Services/APIService.cs b/OF DL/Services/ApiService.cs similarity index 92% rename from OF DL/Services/APIService.cs rename to OF DL/Services/ApiService.cs index 9c64c1d..c460b17 100644 --- a/OF DL/Services/APIService.cs +++ b/OF DL/Services/ApiService.cs @@ -30,53 +30,52 @@ using UserEntities = OF_DL.Models.Entities.Users; using OF_DL.Models.Mappers; using OF_DL.Widevine; using Serilog; -using Spectre.Console; using static OF_DL.Utils.HttpUtil; using Constants = OF_DL.Helpers.Constants; using SinglePostCollection = OF_DL.Models.Entities.Posts.SinglePostCollection; namespace OF_DL.Services; -public class APIService(IAuthService authService, IConfigService configService, IDBService dbService) +public class ApiService(IAuthService authService, IConfigService configService, IDBService dbService) : IAPIService { private const int MaxAttempts = 30; private const int DelayBetweenAttempts = 3000; - private static readonly JsonSerializerSettings m_JsonSerializerSettings; - private static DateTime? cachedDynamicRulesExpiration; - private static DynamicRules? cachedDynamicRules; + private static readonly JsonSerializerSettings s_mJsonSerializerSettings; + private static DateTime? s_cachedDynamicRulesExpiration; + private static DynamicRules? s_cachedDynamicRules; - static APIService() => - m_JsonSerializerSettings = new JsonSerializerSettings { MissingMemberHandling = MissingMemberHandling.Ignore }; + static ApiService() => + s_mJsonSerializerSettings = new JsonSerializerSettings { MissingMemberHandling = MissingMemberHandling.Ignore }; public Dictionary GetDynamicHeaders(string path, string queryParams) { Log.Debug("Calling GetDynamicHeaders"); - Log.Debug($"Path: {path}"); - Log.Debug($"Query Params: {queryParams}"); + Log.Debug("Path: {Path}", path); + Log.Debug("Query Params: {QueryParams}", queryParams); DynamicRules? root; //Check if we have a cached version of the dynamic rules - if (cachedDynamicRules != null && cachedDynamicRulesExpiration.HasValue && - DateTime.UtcNow < cachedDynamicRulesExpiration) + if (s_cachedDynamicRules != null && s_cachedDynamicRulesExpiration.HasValue && + DateTime.UtcNow < s_cachedDynamicRulesExpiration) { Log.Debug("Using cached dynamic rules"); - root = cachedDynamicRules; + root = s_cachedDynamicRules; } else { //Get rules from GitHub and fallback to local file - string? dynamicRulesJSON = GetDynamicRules(); - if (!string.IsNullOrEmpty(dynamicRulesJSON)) + string? dynamicRulesJson = GetDynamicRules(); + if (!string.IsNullOrEmpty(dynamicRulesJson)) { Log.Debug("Using dynamic rules from GitHub"); - root = JsonConvert.DeserializeObject(dynamicRulesJSON); + root = JsonConvert.DeserializeObject(dynamicRulesJson); // Cache the GitHub response for 15 minutes - cachedDynamicRules = root; - cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(15); + s_cachedDynamicRules = root; + s_cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(15); } else { @@ -87,33 +86,55 @@ public class APIService(IAuthService authService, IConfigService configService, // operations and frequent call to GitHub. Since the GitHub dynamic rules // are preferred to the local file, the cache time is shorter than when dynamic rules // are successfully retrieved from GitHub. - cachedDynamicRules = root; - cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(5); + s_cachedDynamicRules = root; + s_cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(5); } } + if (root == null) + { + throw new Exception("Unable to parse dynamic rules. Root is null"); + } + + if (root.ChecksumConstant == null || root.ChecksumIndexes.Count == 0 || root.Prefix == null || + root.Suffix == null || root.AppToken == null) + { + throw new Exception("Invalid dynamic rules. Missing required fields"); + } + + if (authService.CurrentAuth == null) + { + throw new Exception("Auth service is null"); + } + + if (authService.CurrentAuth.UserId == null || authService.CurrentAuth.Cookie == null || + authService.CurrentAuth.UserAgent == null || authService.CurrentAuth.XBc == null) + { + throw new Exception("Auth service is missing required fields"); + } + DateTimeOffset dto = DateTime.UtcNow; long timestamp = dto.ToUnixTimeMilliseconds(); - string input = $"{root!.StaticParam}\n{timestamp}\n{path + queryParams}\n{authService.CurrentAuth.UserId}"; + string input = $"{root.StaticParam}\n{timestamp}\n{path + queryParams}\n{authService.CurrentAuth.UserId}"; byte[] inputBytes = Encoding.UTF8.GetBytes(input); byte[] hashBytes = SHA1.HashData(inputBytes); string hashString = BitConverter.ToString(hashBytes).Replace("-", "").ToLower(); int checksum = root.ChecksumIndexes.Aggregate(0, (current, number) => current + hashString[number]) + - root.ChecksumConstant!.Value; + root.ChecksumConstant.Value; string sign = $"{root.Prefix}:{hashString}:{checksum.ToString("X").ToLower()}:{root.Suffix}"; Dictionary headers = new() { { "accept", "application/json, text/plain" }, - { "app-token", root.AppToken! }, - { "cookie", authService.CurrentAuth!.Cookie! }, + { "app-token", root.AppToken }, + { "cookie", authService.CurrentAuth.Cookie }, { "sign", sign }, { "time", timestamp.ToString() }, - { "user-id", authService.CurrentAuth!.UserId! }, - { "user-agent", authService.CurrentAuth!.UserAgent! }, - { "x-bc", authService.CurrentAuth!.XBc! } + { "user-id", authService.CurrentAuth.UserId }, + { "user-agent", authService.CurrentAuth.UserAgent }, + { "x-bc", authService.CurrentAuth.XBc } }; return headers; } @@ -126,10 +147,10 @@ public class APIService(IAuthService authService, IConfigService configService, try { UserEntities.User user = new(); - int post_limit = 50; + const int postLimit = 50; Dictionary getParams = new() { - { "limit", post_limit.ToString() }, { "order", "publish_date_asc" } + { "limit", postLimit.ToString() }, { "order", "publish_date_asc" } }; HttpClient client = new(); @@ -145,7 +166,7 @@ public class APIService(IAuthService authService, IConfigService configService, response.EnsureSuccessStatusCode(); string body = await response.Content.ReadAsStringAsync(); UserDtos.UserDto? userDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); user = UserMapper.FromDto(userDto) ?? new UserEntities.User(); return user; } @@ -369,7 +390,7 @@ public class APIService(IAuthService authService, IConfigService configService, try { Dictionary return_urls = new(); - int post_limit = 50; + const int postLimit = 50; int limit = 5; int offset = 0; @@ -380,9 +401,7 @@ public class APIService(IAuthService authService, IConfigService configService, case MediaType.Stories: getParams = new Dictionary { - { "limit", post_limit.ToString() }, - { "order", "publish_date_desc" }, - { "skip_users", "all" } + { "limit", postLimit.ToString() }, { "order", "publish_date_desc" }, { "skip_users", "all" } }; break; @@ -402,7 +421,7 @@ public class APIService(IAuthService authService, IConfigService configService, Log.Debug("Media Stories - " + endpoint); List? storiesDto = - JsonConvert.DeserializeObject>(body, m_JsonSerializerSettings); + JsonConvert.DeserializeObject>(body, s_mJsonSerializerSettings); List stories = StoriesMapper.FromDto(storiesDto); foreach (StoryEntities.Stories story in stories) @@ -422,15 +441,27 @@ public class APIService(IAuthService authService, IConfigService configService, await dbService.AddStory(folder, story.Id, "", "0", false, false, DateTime.Now); } - if (story.Media != null && story.Media.Count > 0) + if (story.Media.Count > 0) { foreach (StoryEntities.Medium medium in story.Media) { + if (medium.Files.Full == null || medium.Files.Full.Url == null) + { + continue; + } + + string? mediaType = medium.Type == "photo" ? "Images" : + medium.Type == "video" || medium.Type == "gif" ? "Videos" : + medium.Type == "audio" ? "Audios" : null; + if (mediaType == null) + { + continue; + } + await dbService.AddMedia(folder, medium.Id, story.Id, medium.Files.Full.Url, null, null, null, "Stories", - medium.Type == "photo" ? "Images" : - medium.Type == "video" || medium.Type == "gif" ? "Videos" : - medium.Type == "audio" ? "Audios" : null, false, false, null); + mediaType, false, false, null); + if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; @@ -464,9 +495,9 @@ public class APIService(IAuthService authService, IConfigService configService, } else if (mediatype == MediaType.Highlights) { - List highlight_ids = new(); + List highlightIds = []; HighlightDtos.HighlightsDto? highlightsDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); HighlightEntities.Highlights highlights = HighlightsMapper.FromDto(highlightsDto); if (highlights.HasMore) @@ -480,7 +511,7 @@ public class APIService(IAuthService authService, IConfigService configService, string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); HighlightDtos.HighlightsDto? newHighlightsDto = JsonConvert.DeserializeObject(loopbody, - m_JsonSerializerSettings); + s_mJsonSerializerSettings); HighlightEntities.Highlights newHighlights = HighlightsMapper.FromDto(newHighlightsDto); highlights.List.AddRange(newHighlights.List); @@ -496,13 +527,13 @@ public class APIService(IAuthService authService, IConfigService configService, foreach (HighlightEntities.ListItem list in highlights.List) { - if (!highlight_ids.Contains(list.Id.ToString())) + if (!highlightIds.Contains(list.Id.ToString())) { - highlight_ids.Add(list.Id.ToString()); + highlightIds.Add(list.Id.ToString()); } } - foreach (string highlight_id in highlight_ids) + foreach (string highlight_id in highlightIds) { Dictionary highlight_headers = GetDynamicHeaders("/api2/v2/stories/highlights/" + highlight_id, ""); @@ -522,15 +553,20 @@ public class APIService(IAuthService authService, IConfigService configService, string highlightBody = await highlightResponse.Content.ReadAsStringAsync(); HighlightDtos.HighlightMediaDto? highlightMediaDto = JsonConvert.DeserializeObject(highlightBody, - m_JsonSerializerSettings); + s_mJsonSerializerSettings); HighlightEntities.HighlightMedia highlightMedia = HighlightsMapper.FromDto(highlightMediaDto); foreach (HighlightEntities.Story item in highlightMedia.Stories) { - if (item.Media != null && item.Media.Count > 0 && item.Media[0].CreatedAt.HasValue) + if (item.Media != null && item.Media.Count > 0 && item.Media[0].CreatedAt.HasValue && + item.Media[0].CreatedAt != null) { - await dbService.AddStory(folder, item.Id, "", "0", false, false, - item.Media[0].CreatedAt.Value); + DateTime? createdAt = item.Media[0].CreatedAt; + if (createdAt != null) + { + await dbService.AddStory(folder, item.Id, "", "0", false, false, + createdAt.Value); + } } else if (item.CreatedAt.HasValue) { @@ -605,18 +641,17 @@ public class APIService(IAuthService authService, IConfigService configService, public async Task GetPaidPosts(string endpoint, string folder, string username, - List paid_post_ids, StatusContext ctx) + List paid_post_ids, IStatusReporter statusReporter) { Log.Debug($"Calling GetPaidPosts - {username}"); try { - PurchasedEntities.Purchased paidPosts = new(); PurchasedEntities.PaidPostCollection paidPostCollection = new(); - int post_limit = 50; + const int postLimit = 50; Dictionary getParams = new() { - { "limit", post_limit.ToString() }, + { "limit", postLimit.ToString() }, { "skip_users", "all" }, { "order", "publish_date_desc" }, { "format", "infinite" }, @@ -625,11 +660,9 @@ public class APIService(IAuthService authService, IConfigService configService, string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); PurchasedDtos.PurchasedDto? paidPostsDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - paidPosts = PurchasedMapper.FromDto(paidPostsDto); - ctx.Status($"[red]Getting Paid Posts\n[/] [red]Found {paidPosts.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); + PurchasedEntities.Purchased paidPosts = PurchasedMapper.FromDto(paidPostsDto); + statusReporter.ReportStatus($"Getting Paid Posts - Found {paidPosts.List.Count}"); if (paidPosts != null && paidPosts.HasMore) { getParams["offset"] = paidPosts.List.Count.ToString(); @@ -639,19 +672,17 @@ public class APIService(IAuthService authService, IConfigService configService, string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); PurchasedDtos.PurchasedDto? newPaidPostsDto = - JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); + JsonConvert.DeserializeObject(loopbody, s_mJsonSerializerSettings); newPaidPosts = PurchasedMapper.FromDto(newPaidPostsDto); paidPosts.List.AddRange(newPaidPosts.List); - ctx.Status($"[red]Getting Paid Posts\n[/] [red]Found {paidPosts.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + statusReporter.ReportStatus($"Getting Paid Posts - Found {paidPosts.List.Count}"); if (!newPaidPosts.HasMore) { break; } - getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); + getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + postLimit); } } @@ -812,18 +843,17 @@ public class APIService(IAuthService authService, IConfigService configService, public async Task GetPosts(string endpoint, string folder, List paid_post_ids, - StatusContext ctx) + IStatusReporter statusReporter) { Log.Debug($"Calling GetPosts - {endpoint}"); try { - PostEntities.Post posts = new(); PostEntities.PostCollection postCollection = new(); - int post_limit = 50; + const int postLimit = 50; Dictionary getParams = new() { - { "limit", post_limit.ToString() }, + { "limit", postLimit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "skip_users", "all" } @@ -855,12 +885,9 @@ public class APIService(IAuthService authService, IConfigService configService, string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); PostDtos.PostDto? postsDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - posts = PostMapper.FromDto(postsDto); - ctx.Status( - $"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); + PostEntities.Post posts = PostMapper.FromDto(postsDto); + statusReporter.ReportStatus($"Getting Posts - Found {posts.List.Count}"); if (posts != null && posts.HasMore) { UpdateGetParamsForDateSelection( @@ -874,14 +901,11 @@ public class APIService(IAuthService authService, IConfigService configService, string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); PostDtos.PostDto? newPostsDto = - JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); + JsonConvert.DeserializeObject(loopbody, s_mJsonSerializerSettings); newposts = PostMapper.FromDto(newPostsDto); posts.List.AddRange(newposts.List); - ctx.Status( - $"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + statusReporter.ReportStatus($"Getting Posts - Found {posts.List.Count}"); if (!newposts.HasMore) { break; @@ -1046,7 +1070,7 @@ public class APIService(IAuthService authService, IConfigService configService, string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); PostDtos.SinglePostDto? singlePostDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); singlePost = PostMapper.FromDto(singlePostDto); if (singlePostDto != null) @@ -1217,18 +1241,17 @@ public class APIService(IAuthService authService, IConfigService configService, public async Task GetStreams(string endpoint, string folder, List paid_post_ids, - StatusContext ctx) + IStatusReporter statusReporter) { Log.Debug($"Calling GetStreams - {endpoint}"); try { - StreamEntities.Streams streams = new(); StreamEntities.StreamsCollection streamsCollection = new(); - int post_limit = 50; + const int postLimit = 50; Dictionary getParams = new() { - { "limit", post_limit.ToString() }, + { "limit", postLimit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "skip_users", "all" } @@ -1248,11 +1271,9 @@ public class APIService(IAuthService authService, IConfigService configService, string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); StreamsDtos.StreamsDto? streamsDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - streams = StreamsMapper.FromDto(streamsDto); - ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); + StreamEntities.Streams streams = StreamsMapper.FromDto(streamsDto); + statusReporter.ReportStatus($"Getting Streams - Found {streams.List.Count}"); if (streams != null && streams.HasMore) { UpdateGetParamsForDateSelection( @@ -1266,13 +1287,11 @@ public class APIService(IAuthService authService, IConfigService configService, string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); StreamsDtos.StreamsDto? newStreamsDto = - JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); + JsonConvert.DeserializeObject(loopbody, s_mJsonSerializerSettings); newstreams = StreamsMapper.FromDto(newStreamsDto); streams.List.AddRange(newstreams.List); - ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + statusReporter.ReportStatus($"Getting Streams - Found {streams.List.Count}"); if (!newstreams.HasMore) { break; @@ -1393,18 +1412,17 @@ public class APIService(IAuthService authService, IConfigService configService, public async Task GetArchived(string endpoint, string folder, - StatusContext ctx) + IStatusReporter statusReporter) { Log.Debug($"Calling GetArchived - {endpoint}"); try { - ArchivedEntities.Archived archived = new(); ArchivedEntities.ArchivedCollection archivedCollection = new(); - int post_limit = 50; + const int postLimit = 50; Dictionary getParams = new() { - { "limit", post_limit.ToString() }, + { "limit", postLimit.ToString() }, { "order", "publish_date_desc" }, { "skip_users", "all" }, { "format", "infinite" }, @@ -1425,13 +1443,16 @@ public class APIService(IAuthService authService, IConfigService configService, configService.CurrentConfig.CustomDate); string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); - ArchivedDtos.ArchivedDto archivedDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - archived = ArchivedMapper.FromDto(archivedDto); - ctx.Status($"[red]Getting Archived Posts\n[/] [red]Found {archived.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); - if (archived != null && archived.HasMore) + if (body == null) + { + throw new Exception("Failed to retrieve archived posts. Received null response."); + } + + ArchivedDtos.ArchivedDto? archivedDto = + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); + ArchivedEntities.Archived archived = ArchivedMapper.FromDto(archivedDto); + statusReporter.ReportStatus($"Getting Archived Posts - Found {archived.List.Count}"); + if (archived.HasMore) { UpdateGetParamsForDateSelection( downloadDateSelection, @@ -1439,17 +1460,18 @@ public class APIService(IAuthService authService, IConfigService configService, archived.TailMarker); while (true) { - ArchivedEntities.Archived newarchived = new(); - string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); - ArchivedDtos.ArchivedDto newarchivedDto = - JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); - newarchived = ArchivedMapper.FromDto(newarchivedDto); + if (loopbody == null) + { + throw new Exception("Failed to retrieve archived posts. Received null response."); + } + + ArchivedDtos.ArchivedDto? newarchivedDto = + JsonConvert.DeserializeObject(loopbody, s_mJsonSerializerSettings); + ArchivedEntities.Archived newarchived = ArchivedMapper.FromDto(newarchivedDto); archived.List.AddRange(newarchived.List); - ctx.Status($"[red]Getting Archived Posts\n[/] [red]Found {archived.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + statusReporter.ReportStatus($"Getting Archived Posts - Found {archived.List.Count}"); if (!newarchived.HasMore) { break; @@ -1561,27 +1583,25 @@ public class APIService(IAuthService authService, IConfigService configService, } - public async Task GetMessages(string endpoint, string folder, StatusContext ctx) + public async Task GetMessages(string endpoint, string folder, + IStatusReporter statusReporter) { Log.Debug($"Calling GetMessages - {endpoint}"); try { - MessageEntities.Messages messages = new(); MessageEntities.MessageCollection messageCollection = new(); - int post_limit = 50; + const int postLimit = 50; Dictionary getParams = new() { - { "limit", post_limit.ToString() }, { "order", "desc" }, { "skip_users", "all" } + { "limit", postLimit.ToString() }, { "order", "desc" }, { "skip_users", "all" } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); MessageDtos.MessagesDto? messagesDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - messages = MessagesMapper.FromDto(messagesDto); - ctx.Status($"[red]Getting Messages\n[/] [red]Found {messages.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); + MessageEntities.Messages messages = MessagesMapper.FromDto(messagesDto); + statusReporter.ReportStatus($"Getting Messages - Found {messages.List.Count}"); if (messages.HasMore) { getParams["id"] = messages.List[^1].Id.ToString(); @@ -1591,13 +1611,11 @@ public class APIService(IAuthService authService, IConfigService configService, string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); MessageDtos.MessagesDto? newMessagesDto = - JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); + JsonConvert.DeserializeObject(loopbody, s_mJsonSerializerSettings); newMessages = MessagesMapper.FromDto(newMessagesDto); messages.List.AddRange(newMessages.List); - ctx.Status($"[red]Getting Messages\n[/] [red]Found {messages.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + statusReporter.ReportStatus($"Getting Messages - Found {messages.List.Count}"); if (!newMessages.HasMore) { break; @@ -1823,15 +1841,14 @@ public class APIService(IAuthService authService, IConfigService configService, try { - MessageEntities.SingleMessage message = new(); PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection = new(); - int post_limit = 50; - Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "desc" } }; + const int postLimit = 50; + Dictionary getParams = new() { { "limit", postLimit.ToString() }, { "order", "desc" } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); MessageDtos.SingleMessageDto? messageDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - message = MessagesMapper.FromDto(messageDto); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); + MessageEntities.SingleMessage message = MessagesMapper.FromDto(messageDto); if (!configService.CurrentConfig.IgnoreOwnMessages || message.FromUser?.Id != Convert.ToInt32(authService.CurrentAuth.UserId)) @@ -2028,18 +2045,17 @@ public class APIService(IAuthService authService, IConfigService configService, public async Task GetPaidMessages(string endpoint, string folder, string username, - StatusContext ctx) + IStatusReporter statusReporter) { Log.Debug($"Calling GetPaidMessages - {username}"); try { - PurchasedEntities.Purchased paidMessages = new(); PurchasedEntities.PaidMessageCollection paidMessageCollection = new(); - int post_limit = 50; + const int postLimit = 50; Dictionary getParams = new() { - { "limit", post_limit.ToString() }, + { "limit", postLimit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "author", username }, @@ -2048,11 +2064,9 @@ public class APIService(IAuthService authService, IConfigService configService, string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); PurchasedDtos.PurchasedDto? paidMessagesDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - paidMessages = PurchasedMapper.FromDto(paidMessagesDto); - ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); + PurchasedEntities.Purchased paidMessages = PurchasedMapper.FromDto(paidMessagesDto); + statusReporter.ReportStatus($"Getting Paid Messages - Found {paidMessages.List.Count}"); if (paidMessages != null && paidMessages.HasMore) { getParams["offset"] = paidMessages.List.Count.ToString(); @@ -2077,20 +2091,18 @@ public class APIService(IAuthService authService, IConfigService configService, string loopbody = await loopresponse.Content.ReadAsStringAsync(); PurchasedDtos.PurchasedDto? newPaidMessagesDto = JsonConvert.DeserializeObject(loopbody, - m_JsonSerializerSettings); + s_mJsonSerializerSettings); newpaidMessages = PurchasedMapper.FromDto(newPaidMessagesDto); } paidMessages.List.AddRange(newpaidMessages.List); - ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.List.Count}[/]"); - ctx.Spinner(Spinner.Known.Dots); - ctx.SpinnerStyle(Style.Parse("blue")); + statusReporter.ReportStatus($"Getting Paid Messages - Found {paidMessages.List.Count}"); if (!newpaidMessages.HasMore) { break; } - getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); + getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + postLimit); } } @@ -2330,27 +2342,31 @@ public class APIService(IAuthService authService, IConfigService configService, try { Dictionary purchasedTabUsers = new(); - PurchasedEntities.Purchased purchased = new(); - int post_limit = 50; + const int postLimit = 50; Dictionary getParams = new() { - { "limit", post_limit.ToString() }, + { "limit", postLimit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "skip_users", "all" } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); + if (body == null) + { + throw new Exception("Failed to get purchased tab users. null body returned."); + } + PurchasedDtos.PurchasedDto? purchasedDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - purchased = PurchasedMapper.FromDto(purchasedDto); - if (purchased != null && purchased.HasMore) + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); + PurchasedEntities.Purchased purchased = PurchasedMapper.FromDto(purchasedDto); + if (purchased.HasMore) { getParams["offset"] = purchased.List.Count.ToString(); while (true) { string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); - PurchasedEntities.Purchased newPurchased = new(); + PurchasedEntities.Purchased newPurchased; Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams); HttpClient loopclient = GetHttpClient(); @@ -2368,7 +2384,7 @@ public class APIService(IAuthService authService, IConfigService configService, string loopbody = await loopresponse.Content.ReadAsStringAsync(); PurchasedDtos.PurchasedDto? newPurchasedDto = JsonConvert.DeserializeObject(loopbody, - m_JsonSerializerSettings); + s_mJsonSerializerSettings); newPurchased = PurchasedMapper.FromDto(newPurchasedDto); } @@ -2378,16 +2394,17 @@ public class APIService(IAuthService authService, IConfigService configService, break; } - getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); + getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + postLimit); } } - if (purchased.List != null && purchased.List.Count > 0) + if (purchased.List.Count > 0) { foreach (PurchasedEntities.ListItem purchase in purchased.List.OrderByDescending(p => p.PostedAt ?? p.CreatedAt)) { - if (purchase.FromUser != null) + // purchase.FromUser.Id is not nullable, so the default value is 0 + if (purchase.FromUser.Id != 0) { if (users.Values.Contains(purchase.FromUser.Id)) { @@ -2412,9 +2429,9 @@ public class APIService(IAuthService authService, IConfigService configService, } else { - JObject user = await GetUserInfoById($"/users/list?x[]={purchase.FromUser.Id}"); + JObject? user = await GetUserInfoById($"/users/list?x[]={purchase.FromUser.Id}"); - if (user is null) + if (user == null) { if (!configService.CurrentConfig.BypassContentForCreatorsWhoNoLongerExist) { @@ -2447,7 +2464,8 @@ public class APIService(IAuthService authService, IConfigService configService, } } } - else if (purchase.Author != null) + // purchase.Author is not nullable, so we check against the Author's Id (default value 0) + else if (purchase.Author.Id != 0) { if (users.Values.Contains(purchase.Author.Id)) { @@ -2471,7 +2489,7 @@ public class APIService(IAuthService authService, IConfigService configService, } else { - JObject user = await GetUserInfoById($"/users/list?x[]={purchase.Author.Id}"); + JObject? user = await GetUserInfoById($"/users/list?x[]={purchase.Author.Id}"); if (user is null) { @@ -2536,11 +2554,10 @@ public class APIService(IAuthService authService, IConfigService configService, { Dictionary> userPurchases = new(); List purchasedTabCollections = []; - PurchasedEntities.Purchased purchased = new(); - int post_limit = 50; + const int postLimit = 50; Dictionary getParams = new() { - { "limit", post_limit.ToString() }, + { "limit", postLimit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "skip_users", "all" } @@ -2548,9 +2565,9 @@ public class APIService(IAuthService authService, IConfigService configService, string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); PurchasedDtos.PurchasedDto? purchasedDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); - purchased = PurchasedMapper.FromDto(purchasedDto); - if (purchased != null && purchased.HasMore) + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); + PurchasedEntities.Purchased purchased = PurchasedMapper.FromDto(purchasedDto); + if (purchased.HasMore) { getParams["offset"] = purchased.List.Count.ToString(); while (true) @@ -2574,7 +2591,7 @@ public class APIService(IAuthService authService, IConfigService configService, string loopbody = await loopresponse.Content.ReadAsStringAsync(); PurchasedDtos.PurchasedDto? newPurchasedDto = JsonConvert.DeserializeObject(loopbody, - m_JsonSerializerSettings); + s_mJsonSerializerSettings); newPurchased = PurchasedMapper.FromDto(newPurchasedDto); } @@ -2584,7 +2601,7 @@ public class APIService(IAuthService authService, IConfigService configService, break; } - getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); + getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + postLimit); } } @@ -2617,7 +2634,7 @@ public class APIService(IAuthService authService, IConfigService configService, foreach (KeyValuePair> user in userPurchases) { PurchasedEntities.PurchasedTabCollection purchasedTabCollection = new(); - JObject userObject = await GetUserInfoById($"/users/list?x[]={user.Key}"); + JObject? userObject = await GetUserInfoById($"/users/list?x[]={user.Key}"); purchasedTabCollection.UserId = user.Key; purchasedTabCollection.Username = userObject is not null && @@ -3030,7 +3047,12 @@ public class APIService(IAuthService authService, IConfigService configService, { try { - string pssh = null; + if (authService.CurrentAuth == null) + { + throw new Exception("No current authentication available"); + } + + string? pssh; HttpClient client = new(); HttpRequestMessage request = new(HttpMethod.Get, mpdUrl); @@ -3115,92 +3137,7 @@ public class APIService(IAuthService authService, IConfigService configService, return DateTime.Now; } - public async Task GetDecryptionKeyCDRMProject(Dictionary drmHeaders, string licenceURL, - string pssh) - { - Log.Debug("Calling GetDecryptionKey"); - - int attempt = 0; - - try - { - string dcValue = ""; - HttpClient client = new(); - - CDRMProjectRequest cdrmProjectRequest = new() - { - Pssh = pssh, - LicenseUrl = licenceURL, - Headers = JsonConvert.SerializeObject(drmHeaders), - Cookies = "", - Data = "" - }; - - string json = JsonConvert.SerializeObject(cdrmProjectRequest); - - Log.Debug($"Posting to CDRM Project: {json}"); - - while (attempt < MaxAttempts) - { - attempt++; - - HttpRequestMessage request = new(HttpMethod.Post, "https://cdrm-project.com/api/decrypt") - { - Content = new StringContent(json, Encoding.UTF8, "application/json") - }; - - using HttpResponseMessage response = await client.SendAsync(request); - - Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}"); - - response.EnsureSuccessStatusCode(); - string body = await response.Content.ReadAsStringAsync(); - JsonDocument doc = JsonDocument.Parse(body); - - if (doc.RootElement.TryGetProperty("status", out JsonElement status)) - { - if (status.ToString().Trim().Equals("success", StringComparison.OrdinalIgnoreCase)) - { - dcValue = doc.RootElement.GetProperty("message").GetString().Trim(); - return dcValue; - } - - Log.Debug($"CDRM response status not successful. Retrying... Attempt {attempt} of {MaxAttempts}"); - if (attempt < MaxAttempts) - { - await Task.Delay(DelayBetweenAttempts); - } - } - else - { - Log.Debug($"Status not in CDRM response. Retrying... Attempt {attempt} of {MaxAttempts}"); - if (attempt < MaxAttempts) - { - await Task.Delay(DelayBetweenAttempts); - } - } - } - - throw new Exception("Maximum retry attempts reached. Unable to get a valid decryption key."); - } - catch (Exception ex) - { - Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); - Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); - if (ex.InnerException != null) - { - Console.WriteLine("\nInner Exception:"); - Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, - ex.InnerException.StackTrace); - Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, - ex.InnerException.StackTrace); - } - } - - return null; - } - - public async Task GetDecryptionKeyOFDL(Dictionary drmHeaders, string licenceURL, + public async Task GetDecryptionKeyOFDL(Dictionary drmHeaders, string licenceUrl, string pssh) { Log.Debug("Calling GetDecryptionOFDL"); @@ -3212,12 +3149,12 @@ public class APIService(IAuthService authService, IConfigService configService, OFDLRequest ofdlRequest = new() { - Pssh = pssh, LicenseUrl = licenceURL, Headers = JsonConvert.SerializeObject(drmHeaders) + Pssh = pssh, LicenseUrl = licenceUrl, Headers = JsonConvert.SerializeObject(drmHeaders) }; string json = JsonConvert.SerializeObject(ofdlRequest); - Log.Debug($"Posting to ofdl.tools: {json}"); + Log.Debug("Posting to ofdl.tools: {Json}", json); while (attempt < MaxAttempts) { @@ -3263,28 +3200,34 @@ public class APIService(IAuthService authService, IConfigService configService, return null; } - public async Task GetDecryptionKeyCDM(Dictionary drmHeaders, string licenceURL, string pssh) + public async Task? GetDecryptionKeyCDM(Dictionary drmHeaders, string licenceURL, + string pssh) { Log.Debug("Calling GetDecryptionKeyCDM"); try { - byte[] resp1 = await PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 }); + byte[] resp1 = await PostData(licenceURL, drmHeaders, [0x08, 0x04]); string certDataB64 = Convert.ToBase64String(resp1); CDMApi cdm = new(); - byte[] challenge = cdm.GetChallenge(pssh, certDataB64); + byte[]? challenge = cdm.GetChallenge(pssh, certDataB64); + if (challenge == null) + { + throw new Exception("Failed to get challenge from CDM"); + } + byte[] resp2 = await PostData(licenceURL, drmHeaders, challenge); string licenseB64 = Convert.ToBase64String(resp2); - Log.Debug($"resp1: {resp1}"); - Log.Debug($"certDataB64: {certDataB64}"); - Log.Debug($"challenge: {challenge}"); - Log.Debug($"resp2: {resp2}"); - Log.Debug($"licenseB64: {licenseB64}"); + Log.Debug("resp1: {Resp1}", resp1); + Log.Debug("certDataB64: {CertDataB64}", certDataB64); + Log.Debug("challenge: {Challenge}", challenge); + Log.Debug("resp2: {Resp2}", resp2); + Log.Debug("licenseB64: {LicenseB64}", licenseB64); cdm.ProvideLicense(licenseB64); List keys = cdm.GetKeys(); if (keys.Count > 0) { - Log.Debug($"GetDecryptionKeyCDM Key: {keys[0]}"); + Log.Debug("GetDecryptionKeyCDM Key: {ContentKey}", keys[0]); return keys[0].ToString(); } } @@ -3322,7 +3265,7 @@ public class APIService(IAuthService authService, IConfigService configService, } - private async Task BuildHttpRequestMessage(Dictionary getParams, + private Task BuildHttpRequestMessage(Dictionary getParams, string endpoint) { Log.Debug("Calling BuildHttpRequestMessage"); @@ -3340,7 +3283,7 @@ public class APIService(IAuthService authService, IConfigService configService, request.Headers.Add(keyValuePair.Key, keyValuePair.Value); } - return request; + return Task.FromResult(request); } private static double ConvertToUnixTimestampWithMicrosecondPrecision(DateTime date) @@ -3420,7 +3363,7 @@ public class APIService(IAuthService authService, IConfigService configService, string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); SubscriptionsDtos.SubscriptionsDto? subscriptionsDto = - JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); + JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings); subscriptions = SubscriptionsMapper.FromDto(subscriptionsDto); if (subscriptions.HasMore) { @@ -3435,7 +3378,7 @@ public class APIService(IAuthService authService, IConfigService configService, { SubscriptionsDtos.SubscriptionsDto? newSubscriptionsDto = JsonConvert.DeserializeObject(loopbody, - m_JsonSerializerSettings); + s_mJsonSerializerSettings); newSubscriptions = SubscriptionsMapper.FromDto(newSubscriptionsDto); } else diff --git a/OF DL/Services/AuthService.cs b/OF DL/Services/AuthService.cs index 3f9f654..2755bfb 100644 --- a/OF DL/Services/AuthService.cs +++ b/OF DL/Services/AuthService.cs @@ -1,12 +1,15 @@ +using System.Text.RegularExpressions; +using Microsoft.Extensions.DependencyInjection; using Newtonsoft.Json; using OF_DL.Models; using PuppeteerSharp; using PuppeteerSharp.BrowserData; using Serilog; +using UserEntities = OF_DL.Models.Entities.Users; namespace OF_DL.Services; -public class AuthService : IAuthService +public class AuthService(IServiceProvider serviceProvider) : IAuthService { private const int LoginTimeout = 600000; // 10 minutes private const int FeedLoadTimeout = 60000; // 1 minute @@ -126,6 +129,53 @@ public class AuthService : IAuthService private async Task GetBcToken(IPage page) => await page.EvaluateExpressionAsync("window.localStorage.getItem('bcTokenSha') || ''"); + public void ValidateCookieString() + { + if (CurrentAuth == null) + { + return; + } + + string pattern = @"(auth_id=\d+)|(sess=[^;]+)"; + MatchCollection matches = Regex.Matches(CurrentAuth.Cookie, pattern); + + string output = string.Join("; ", matches); + + if (!output.EndsWith(";")) + { + output += ";"; + } + + if (CurrentAuth.Cookie.Trim() != output.Trim()) + { + CurrentAuth.Cookie = output; + string newAuthString = JsonConvert.SerializeObject(CurrentAuth, Formatting.Indented); + File.WriteAllText("auth.json", newAuthString); + } + } + + public async Task ValidateAuthAsync() + { + // Resolve IAPIService lazily to avoid circular dependency + IAPIService apiService = serviceProvider.GetRequiredService(); + return await apiService.GetUserInfo("/users/me"); + } + + public void Logout() + { + if (Directory.Exists("chrome-data")) + { + Log.Information("Deleting chrome-data folder"); + Directory.Delete("chrome-data", true); + } + + if (File.Exists("auth.json")) + { + Log.Information("Deleting auth.json"); + File.Delete("auth.json"); + } + } + private async Task GetAuthFromBrowser(bool isDocker = false) { try @@ -184,6 +234,7 @@ public class AuthService : IAuthService } catch (Exception e) { + Log.Error(e, "Error getting bcToken"); throw new Exception("Error getting bcToken"); } diff --git a/OF DL/Services/ConfigService.cs b/OF DL/Services/ConfigService.cs index d22c617..9c43b0a 100644 --- a/OF DL/Services/ConfigService.cs +++ b/OF DL/Services/ConfigService.cs @@ -1,3 +1,4 @@ +using System.Reflection; using System.Text; using Akka.Configuration; using Akka.Configuration.Hocon; @@ -228,13 +229,12 @@ public class ConfigService(ILoggingService loggingService) : IConfigService Akka.Configuration.Config? creatorConfigsSection = hoconConfig.GetConfig("CreatorConfigs"); if (creatorConfigsSection != null) { - foreach (KeyValuePair key in creatorConfigsSection.AsEnumerable()) + foreach ((string? creatorKey, _) in creatorConfigsSection.AsEnumerable()) { - string creatorKey = key.Key; Akka.Configuration.Config? creatorHocon = creatorConfigsSection.GetConfig(creatorKey); if (!CurrentConfig.CreatorConfigs.ContainsKey(creatorKey) && creatorHocon != null) { - CurrentConfig.CreatorConfigs.Add(key.Key, + CurrentConfig.CreatorConfigs.Add(creatorKey, new CreatorConfig { PaidPostFileNameFormat = creatorHocon.GetString("PaidPostFileNameFormat"), @@ -243,14 +243,14 @@ public class ConfigService(ILoggingService loggingService) : IConfigService MessageFileNameFormat = creatorHocon.GetString("MessageFileNameFormat") }); - ValidateFileNameFormat(CurrentConfig.CreatorConfigs[key.Key].PaidPostFileNameFormat, - $"{key.Key}.PaidPostFileNameFormat"); - ValidateFileNameFormat(CurrentConfig.CreatorConfigs[key.Key].PostFileNameFormat, - $"{key.Key}.PostFileNameFormat"); - ValidateFileNameFormat(CurrentConfig.CreatorConfigs[key.Key].PaidMessageFileNameFormat, - $"{key.Key}.PaidMessageFileNameFormat"); - ValidateFileNameFormat(CurrentConfig.CreatorConfigs[key.Key].MessageFileNameFormat, - $"{key.Key}.MessageFileNameFormat"); + ValidateFileNameFormat(CurrentConfig.CreatorConfigs[creatorKey].PaidPostFileNameFormat, + $"{creatorKey}.PaidPostFileNameFormat"); + ValidateFileNameFormat(CurrentConfig.CreatorConfigs[creatorKey].PostFileNameFormat, + $"{creatorKey}.PostFileNameFormat"); + ValidateFileNameFormat(CurrentConfig.CreatorConfigs[creatorKey].PaidMessageFileNameFormat, + $"{creatorKey}.PaidMessageFileNameFormat"); + ValidateFileNameFormat(CurrentConfig.CreatorConfigs[creatorKey].MessageFileNameFormat, + $"{creatorKey}.MessageFileNameFormat"); } } } @@ -402,6 +402,50 @@ public class ConfigService(ILoggingService loggingService) : IConfigService } } + public List<(string Name, bool Value)> GetToggleableProperties() + { + List<(string Name, bool Value)> result = []; + foreach (PropertyInfo propInfo in typeof(Config).GetProperties()) + { + ToggleableConfigAttribute? attr = propInfo.GetCustomAttribute(); + if (attr != null) + { + result.Add((propInfo.Name, (bool)propInfo.GetValue(CurrentConfig)!)); + } + } + + return result; + } + + public bool ApplyToggleableSelections(List selectedNames) + { + bool configChanged = false; + Config newConfig = new(); + + foreach (PropertyInfo propInfo in typeof(Config).GetProperties()) + { + ToggleableConfigAttribute? attr = propInfo.GetCustomAttribute(); + if (attr != null) + { + bool newValue = selectedNames.Contains(propInfo.Name); + bool oldValue = (bool)propInfo.GetValue(CurrentConfig)!; + propInfo.SetValue(newConfig, newValue); + + if (newValue != oldValue) + { + configChanged = true; + } + } + else + { + propInfo.SetValue(newConfig, propInfo.GetValue(CurrentConfig)); + } + } + + UpdateConfig(newConfig); + return configChanged; + } + private VideoResolution ParseVideoResolution(string value) { if (value.Equals("source", StringComparison.OrdinalIgnoreCase)) diff --git a/OF DL/Services/DownloadOrchestrationService.cs b/OF DL/Services/DownloadOrchestrationService.cs new file mode 100644 index 0000000..cc8902c --- /dev/null +++ b/OF DL/Services/DownloadOrchestrationService.cs @@ -0,0 +1,550 @@ +using Newtonsoft.Json.Linq; +using OF_DL.Enumerations; +using OF_DL.Models; +using Serilog; +using PostEntities = OF_DL.Models.Entities.Posts; +using PurchasedEntities = OF_DL.Models.Entities.Purchased; +using UserEntities = OF_DL.Models.Entities.Users; + +namespace OF_DL.Services; + +public class DownloadOrchestrationService( + IAPIService apiService, + IConfigService configService, + IDownloadService downloadService, + IDBService dbService) : IDownloadOrchestrationService +{ + public List PaidPostIds { get; } = new(); + + public async Task GetAvailableUsersAsync() + { + UserListResult result = new(); + Config config = configService.CurrentConfig; + + Dictionary? activeSubs = + await apiService.GetActiveSubscriptions("/subscriptions/subscribes", + config.IncludeRestrictedSubscriptions); + + if (activeSubs != null) + { + Log.Debug("Subscriptions: "); + foreach (KeyValuePair activeSub in activeSubs) + { + if (!result.Users.ContainsKey(activeSub.Key)) + { + result.Users.Add(activeSub.Key, activeSub.Value); + Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}"); + } + } + } + else + { + Log.Error("Couldn't get active subscriptions. Received null response."); + } + + + if (config.IncludeExpiredSubscriptions) + { + Log.Debug("Inactive Subscriptions: "); + Dictionary? expiredSubs = + await apiService.GetExpiredSubscriptions("/subscriptions/subscribes", + config.IncludeRestrictedSubscriptions); + + if (expiredSubs != null) + { + foreach (KeyValuePair expiredSub in expiredSubs.Where(expiredSub => + !result.Users.ContainsKey(expiredSub.Key))) + { + result.Users.Add(expiredSub.Key, expiredSub.Value); + Log.Debug("Name: {ExpiredSubKey} ID: {ExpiredSubValue}", expiredSub.Key, expiredSub.Value); + } + } + else + { + Log.Error("Couldn't get expired subscriptions. Received null response."); + } + } + + result.Lists = await apiService.GetLists("/lists") ?? new Dictionary(); + + // Remove users from the list if they are in the ignored list + if (!string.IsNullOrEmpty(config.IgnoredUsersListName)) + { + if (!result.Lists.TryGetValue(config.IgnoredUsersListName, out long ignoredUsersListId)) + { + result.IgnoredListError = $"Ignored users list '{config.IgnoredUsersListName}' not found"; + Log.Error(result.IgnoredListError); + } + else + { + List ignoredUsernames = + await apiService.GetListUsers($"/lists/{ignoredUsersListId}/users") ?? []; + result.Users = result.Users.Where(x => !ignoredUsernames.Contains(x.Key)) + .ToDictionary(x => x.Key, x => x.Value); + } + } + + await dbService.CreateUsersDB(result.Users); + return result; + } + + public async Task> GetUsersForListAsync( + string listName, Dictionary allUsers, Dictionary lists) + { + long listId = lists[listName]; + List listUsernames = await apiService.GetListUsers($"/lists/{listId}/users") ?? []; + return allUsers.Where(x => listUsernames.Contains(x.Key)).Distinct() + .ToDictionary(x => x.Key, x => x.Value); + } + + public string ResolveDownloadPath(string username) => + !string.IsNullOrEmpty(configService.CurrentConfig.DownloadPath) + ? Path.Combine(configService.CurrentConfig.DownloadPath, username) + : $"__user_data__/sites/OnlyFans/{username}"; + + public async Task PrepareUserFolderAsync(string username, long userId, string path) + { + await dbService.CheckUsername(new KeyValuePair(username, userId), path); + + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + Log.Debug($"Created folder for {username}"); + } + + await dbService.CreateDB(path); + } + + public async Task DownloadCreatorContentAsync( + string username, long userId, string path, + Dictionary users, + bool clientIdBlobMissing, bool devicePrivateKeyMissing, + IDownloadEventHandler eventHandler) + { + Config config = configService.CurrentConfig; + CreatorDownloadResult counts = new(); + + eventHandler.OnUserStarting(username); + Log.Debug($"Scraping Data for {username}"); + + await PrepareUserFolderAsync(username, userId, path); + + if (config.DownloadAvatarHeaderPhoto) + { + UserEntities.User? userInfo = await apiService.GetUserInfo($"/users/{username}"); + if (userInfo != null) + { + await downloadService.DownloadAvatarHeader(userInfo.Avatar, userInfo.Header, path, username); + } + } + + if (config.DownloadPaidPosts) + { + counts.PaidPostCount = await DownloadContentTypeAsync("Paid Posts", + async statusReporter => + await apiService.GetPaidPosts("/posts/paid/post", path, username, PaidPostIds, statusReporter), + posts => posts?.PaidPosts?.Count ?? 0, + posts => posts?.PaidPostObjects?.Count ?? 0, + posts => posts?.PaidPosts?.Values?.ToList(), + async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter), + eventHandler); + } + + if (config.DownloadPosts) + { + eventHandler.OnMessage( + "Getting Posts (this may take a long time, depending on the number of Posts the creator has)"); + Log.Debug($"Calling DownloadFreePosts - {username}"); + + counts.PostCount = await DownloadContentTypeAsync("Posts", + async statusReporter => + await apiService.GetPosts($"/users/{userId}/posts", path, PaidPostIds, statusReporter), + posts => posts?.Posts?.Count ?? 0, + posts => posts?.PostObjects?.Count ?? 0, + posts => posts?.Posts?.Values?.ToList(), + async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter), + eventHandler); + } + + if (config.DownloadArchived) + { + counts.ArchivedCount = await DownloadContentTypeAsync("Archived Posts", + async statusReporter => + await apiService.GetArchived($"/users/{userId}/posts", path, statusReporter), + archived => archived?.ArchivedPosts?.Count ?? 0, + archived => archived?.ArchivedPostObjects?.Count ?? 0, + archived => archived?.ArchivedPosts?.Values?.ToList(), + async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter), + eventHandler); + } + + if (config.DownloadStreams) + { + counts.StreamsCount = await DownloadContentTypeAsync("Streams", + async statusReporter => + await apiService.GetStreams($"/users/{userId}/posts/streams", path, PaidPostIds, statusReporter), + streams => streams?.Streams?.Count ?? 0, + streams => streams?.StreamObjects?.Count ?? 0, + streams => streams?.Streams?.Values?.ToList(), + async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter), + eventHandler); + } + + if (config.DownloadStories) + { + eventHandler.OnMessage("Getting Stories"); + Dictionary? tempStories = await apiService.GetMedia(MediaType.Stories, + $"/users/{userId}/stories", null, path, PaidPostIds); + + if (tempStories != null && tempStories.Count > 0) + { + eventHandler.OnContentFound("Stories", tempStories.Count, tempStories.Count); + + long totalSize = config.ShowScrapeSize + ? await downloadService.CalculateTotalFileSize(tempStories.Values.ToList()) + : tempStories.Count; + + DownloadResult result = await eventHandler.WithProgressAsync( + $"Downloading {tempStories.Count} Stories", totalSize, config.ShowScrapeSize, + async reporter => await downloadService.DownloadStories(username, userId, path, + PaidPostIds.ToHashSet(), reporter)); + + eventHandler.OnDownloadComplete("Stories", result); + counts.StoriesCount = result.TotalCount; + } + else + { + eventHandler.OnNoContentFound("Stories"); + } + } + + if (config.DownloadHighlights) + { + eventHandler.OnMessage("Getting Highlights"); + Dictionary? tempHighlights = await apiService.GetMedia(MediaType.Highlights, + $"/users/{userId}/stories/highlights", null, path, PaidPostIds); + + if (tempHighlights != null && tempHighlights.Count > 0) + { + eventHandler.OnContentFound("Highlights", tempHighlights.Count, tempHighlights.Count); + + long totalSize = config.ShowScrapeSize + ? await downloadService.CalculateTotalFileSize(tempHighlights.Values.ToList()) + : tempHighlights.Count; + + DownloadResult result = await eventHandler.WithProgressAsync( + $"Downloading {tempHighlights.Count} Highlights", totalSize, config.ShowScrapeSize, + async reporter => await downloadService.DownloadHighlights(username, userId, path, + PaidPostIds.ToHashSet(), reporter)); + + eventHandler.OnDownloadComplete("Highlights", result); + counts.HighlightsCount = result.TotalCount; + } + else + { + eventHandler.OnNoContentFound("Highlights"); + } + } + + if (config.DownloadMessages) + { + counts.MessagesCount = await DownloadContentTypeAsync("Messages", + async statusReporter => + await apiService.GetMessages($"/chats/{userId}/messages", path, statusReporter), + messages => messages.Messages?.Count ?? 0, + messages => messages.MessageObjects?.Count ?? 0, + messages => messages?.Messages.Values.ToList(), + async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter), + eventHandler); + } + + if (config.DownloadPaidMessages) + { + counts.PaidMessagesCount = await DownloadContentTypeAsync("Paid Messages", + async statusReporter => + await apiService.GetPaidMessages("/posts/paid/chat", path, username, statusReporter), + paidMessages => paidMessages?.PaidMessages?.Count ?? 0, + paidMessages => paidMessages?.PaidMessageObjects?.Count ?? 0, + paidMessages => paidMessages?.PaidMessages?.Values?.ToList(), + async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter), + eventHandler); + } + + eventHandler.OnUserComplete(username, counts); + return counts; + } + + public async Task DownloadSinglePostAsync( + string username, long postId, string path, + Dictionary users, + bool clientIdBlobMissing, bool devicePrivateKeyMissing, + IDownloadEventHandler eventHandler) + { + Log.Debug($"Calling DownloadSinglePost - {postId}"); + eventHandler.OnMessage("Getting Post"); + + PostEntities.SinglePostCollection post = await apiService.GetPost($"/posts/{postId}", path); + if (post.SinglePosts.Count == 0) + { + eventHandler.OnMessage("Couldn't find post"); + Log.Debug("Couldn't find post"); + return; + } + + Config config = configService.CurrentConfig; + long totalSize = config.ShowScrapeSize + ? await downloadService.CalculateTotalFileSize(post.SinglePosts.Values.ToList()) + : post.SinglePosts.Count; + + DownloadResult result = await eventHandler.WithProgressAsync( + "Downloading Post", totalSize, config.ShowScrapeSize, + async reporter => await downloadService.DownloadSinglePost(username, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, post, reporter)); + + if (result.NewDownloads > 0) + { + eventHandler.OnMessage($"Post {postId} downloaded"); + Log.Debug($"Post {postId} downloaded"); + } + else + { + eventHandler.OnMessage($"Post {postId} already downloaded"); + Log.Debug($"Post {postId} already downloaded"); + } + } + + public async Task DownloadPurchasedTabAsync( + Dictionary users, + bool clientIdBlobMissing, bool devicePrivateKeyMissing, + IDownloadEventHandler eventHandler) + { + Config config = configService.CurrentConfig; + + Dictionary purchasedTabUsers = + await apiService.GetPurchasedTabUsers("/posts/paid/all", users); + + eventHandler.OnMessage("Checking folders for Users in Purchased Tab"); + + foreach (KeyValuePair user in purchasedTabUsers) + { + string path = ResolveDownloadPath(user.Key); + Log.Debug($"Download path: {path}"); + + await dbService.CheckUsername(user, path); + + if (!Directory.Exists(path)) + { + Directory.CreateDirectory(path); + Log.Debug($"Created folder for {user.Key}"); + } + + await apiService.GetUserInfo($"/users/{user.Key}"); + await dbService.CreateDB(path); + } + + string basePath = !string.IsNullOrEmpty(config.DownloadPath) + ? config.DownloadPath + : "__user_data__/sites/OnlyFans/"; + + Log.Debug($"Download path: {basePath}"); + + List purchasedTabCollections = + await apiService.GetPurchasedTab("/posts/paid/all", basePath, users); + + foreach (PurchasedEntities.PurchasedTabCollection purchasedTabCollection in purchasedTabCollections) + { + eventHandler.OnUserStarting(purchasedTabCollection.Username); + string path = ResolveDownloadPath(purchasedTabCollection.Username); + Log.Debug($"Download path: {path}"); + + int paidPostCount = 0; + int paidMessagesCount = 0; + + // Download paid posts + if (purchasedTabCollection.PaidPosts?.PaidPosts?.Count > 0) + { + eventHandler.OnContentFound("Paid Posts", + purchasedTabCollection.PaidPosts.PaidPosts.Count, + purchasedTabCollection.PaidPosts.PaidPostObjects.Count); + + long totalSize = config.ShowScrapeSize + ? await downloadService.CalculateTotalFileSize( + purchasedTabCollection.PaidPosts.PaidPosts.Values.ToList()) + : purchasedTabCollection.PaidPosts.PaidPosts.Count; + + DownloadResult postResult = await eventHandler.WithProgressAsync( + $"Downloading {purchasedTabCollection.PaidPosts.PaidPosts.Count} Paid Posts", + totalSize, config.ShowScrapeSize, + async reporter => await downloadService.DownloadPaidPostsPurchasedTab( + purchasedTabCollection.Username, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, + purchasedTabCollection.PaidPosts, reporter)); + + eventHandler.OnDownloadComplete("Paid Posts", postResult); + paidPostCount = postResult.TotalCount; + } + else + { + eventHandler.OnNoContentFound("Paid Posts"); + } + + // Download paid messages + if (purchasedTabCollection.PaidMessages?.PaidMessages?.Count > 0) + { + eventHandler.OnContentFound("Paid Messages", + purchasedTabCollection.PaidMessages.PaidMessages.Count, + purchasedTabCollection.PaidMessages.PaidMessageObjects.Count); + + long totalSize = config.ShowScrapeSize + ? await downloadService.CalculateTotalFileSize( + purchasedTabCollection.PaidMessages.PaidMessages.Values.ToList()) + : purchasedTabCollection.PaidMessages.PaidMessages.Count; + + DownloadResult msgResult = await eventHandler.WithProgressAsync( + $"Downloading {purchasedTabCollection.PaidMessages.PaidMessages.Count} Paid Messages", + totalSize, config.ShowScrapeSize, + async reporter => await downloadService.DownloadPaidMessagesPurchasedTab( + purchasedTabCollection.Username, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, + purchasedTabCollection.PaidMessages, reporter)); + + eventHandler.OnDownloadComplete("Paid Messages", msgResult); + paidMessagesCount = msgResult.TotalCount; + } + else + { + eventHandler.OnNoContentFound("Paid Messages"); + } + + eventHandler.OnPurchasedTabUserComplete(purchasedTabCollection.Username, paidPostCount, paidMessagesCount); + } + } + + public async Task DownloadSinglePaidMessageAsync( + string username, long messageId, string path, + Dictionary users, + bool clientIdBlobMissing, bool devicePrivateKeyMissing, + IDownloadEventHandler eventHandler) + { + Log.Debug($"Calling DownloadSinglePaidMessage - {username}"); + eventHandler.OnMessage("Getting Paid Message"); + + PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection = + await apiService.GetPaidMessage($"/messages/{messageId}", path); + + if (singlePaidMessageCollection.SingleMessages.Count == 0) + { + eventHandler.OnNoContentFound("Paid Messages"); + return; + } + + Config config = configService.CurrentConfig; + + // Handle preview messages + if (singlePaidMessageCollection.PreviewSingleMessages.Count > 0) + { + eventHandler.OnContentFound("Preview Paid Messages", + singlePaidMessageCollection.PreviewSingleMessages.Count, + singlePaidMessageCollection.SingleMessageObjects.Count); + + long previewSize = config.ShowScrapeSize + ? await downloadService.CalculateTotalFileSize( + singlePaidMessageCollection.PreviewSingleMessages.Values.ToList()) + : singlePaidMessageCollection.PreviewSingleMessages.Count; + + DownloadResult previewResult = await eventHandler.WithProgressAsync( + $"Downloading {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Paid Messages", + previewSize, config.ShowScrapeSize, + async reporter => await downloadService.DownloadSinglePaidMessage(username, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, singlePaidMessageCollection, reporter)); + + eventHandler.OnDownloadComplete("Paid Messages", previewResult); + } + else if (singlePaidMessageCollection.SingleMessages.Count > 0) + { + // Only actual paid messages, no preview + eventHandler.OnContentFound("Paid Messages", + singlePaidMessageCollection.SingleMessages.Count, + singlePaidMessageCollection.SingleMessageObjects.Count); + + long totalSize = config.ShowScrapeSize + ? await downloadService.CalculateTotalFileSize( + singlePaidMessageCollection.SingleMessages.Values.ToList()) + : singlePaidMessageCollection.SingleMessages.Count; + + DownloadResult result = await eventHandler.WithProgressAsync( + $"Downloading {singlePaidMessageCollection.SingleMessages.Count} Paid Messages", + totalSize, config.ShowScrapeSize, + async reporter => await downloadService.DownloadSinglePaidMessage(username, path, users, + clientIdBlobMissing, devicePrivateKeyMissing, singlePaidMessageCollection, reporter)); + + eventHandler.OnDownloadComplete("Paid Messages", result); + } + else + { + eventHandler.OnNoContentFound("Paid Messages"); + } + } + + public async Task ResolveUsernameAsync(long userId) + { + JObject? user = await apiService.GetUserInfoById($"/users/list?x[]={userId}"); + if (user == null) + { + return $"Deleted User - {userId}"; + } + + string? username = user[userId.ToString()]?["username"]?.ToString(); + return !string.IsNullOrEmpty(username) ? username : $"Deleted User - {userId}"; + } + + /// + /// Generic helper for the common pattern: fetch with status -> check count -> download with progress. + /// + private async Task DownloadContentTypeAsync( + string contentType, + Func> fetchData, + Func getMediaCount, + Func getObjectCount, + Func?> getUrls, + Func> downloadData, + IDownloadEventHandler eventHandler) + { + T data = await eventHandler.WithStatusAsync($"Getting {contentType}", + async statusReporter => await fetchData(statusReporter)); + + int mediaCount = getMediaCount(data); + if (mediaCount <= 0) + { + eventHandler.OnNoContentFound(contentType); + Log.Debug($"Found 0 {contentType}"); + return 0; + } + + int objectCount = getObjectCount(data); + eventHandler.OnContentFound(contentType, mediaCount, objectCount); + Log.Debug($"Found {mediaCount} Media from {objectCount} {contentType}"); + + Config config = configService.CurrentConfig; + List? urls = getUrls(data); + long totalSize = config.ShowScrapeSize && urls != null + ? await downloadService.CalculateTotalFileSize(urls) + : mediaCount; + + DownloadResult result = await eventHandler.WithProgressAsync( + $"Downloading {mediaCount} {contentType}", totalSize, config.ShowScrapeSize, + async reporter => await downloadData(data, reporter)); + + eventHandler.OnDownloadComplete(contentType, result); + Log.Debug( + $"{contentType} Already Downloaded: {result.ExistingDownloads} New {contentType} Downloaded: {result.NewDownloads}"); + + return result.TotalCount; + } +} diff --git a/OF DL/Services/DownloadService.cs b/OF DL/Services/DownloadService.cs index eb2c698..b9cc5ac 100644 --- a/OF DL/Services/DownloadService.cs +++ b/OF DL/Services/DownloadService.cs @@ -292,6 +292,11 @@ public class DownloadService( private async Task GetVideoStreamIndexFromMpd(string mpdUrl, string policy, string signature, string kvp, VideoResolution resolution) { + if (authService.CurrentAuth == null) + { + return null; + } + HttpClient client = new(); HttpRequestMessage request = new(HttpMethod.Get, mpdUrl); request.Headers.Add("user-agent", authService.CurrentAuth.UserAgent); @@ -307,7 +312,7 @@ public class DownloadService( XNamespace cenc = "urn:mpeg:cenc:2013"; XElement? videoAdaptationSet = doc .Descendants(ns + "AdaptationSet") - .FirstOrDefault(e => (string)e.Attribute("mimeType") == "video/mp4"); + .FirstOrDefault(e => (string?)e.Attribute("mimeType") == "video/mp4"); if (videoAdaptationSet == null) { @@ -326,7 +331,7 @@ public class DownloadService( for (int i = 0; i < representations.Count; i++) { - if ((string)representations[i].Attribute("height") == targetHeight) + if ((string?)representations[i].Attribute("height") == targetHeight) { return i; // this is the index FFmpeg will use for `-map 0:v:{i}` } @@ -512,6 +517,21 @@ public class DownloadService( try { + if (authService.CurrentAuth == null) + { + throw new Exception("No authentication information available."); + } + + if (authService.CurrentAuth.Cookie == null) + { + throw new Exception("No authentication cookie available."); + } + + if (authService.CurrentAuth.UserAgent == null) + { + throw new Exception("No user agent available."); + } + Uri uri = new(url); if (uri.Host == "cdn3.onlyfans.com" && uri.LocalPath.Contains("/dash/files")) @@ -558,8 +578,6 @@ public class DownloadService( public static async Task GetDRMVideoLastModified(string url, Auth auth) { - Uri uri = new(url); - string[] messageUrlParsed = url.Split(','); string mpdURL = messageUrlParsed[0]; string policy = messageUrlParsed[1]; @@ -946,6 +964,21 @@ public class DownloadService( { try { + if (authService.CurrentAuth == null) + { + throw new Exception("No authentication information available."); + } + + if (authService.CurrentAuth.Cookie == null) + { + throw new Exception("No authentication cookie available."); + } + + if (authService.CurrentAuth.UserAgent == null) + { + throw new Exception("No user agent available."); + } + Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; @@ -1079,7 +1112,7 @@ public class DownloadService( { Log.Debug($"Calling DownloadHighlights - {username}"); - Dictionary highlights = await apiService.GetMedia(MediaType.Highlights, + Dictionary? highlights = await apiService.GetMedia(MediaType.Highlights, $"/users/{userId}/stories/highlights", null, path, paidPostIds.ToList()); if (highlights == null || highlights.Count == 0) @@ -1133,7 +1166,7 @@ public class DownloadService( { Log.Debug($"Calling DownloadStories - {username}"); - Dictionary stories = await apiService.GetMedia(MediaType.Stories, $"/users/{userId}/stories", + Dictionary? stories = await apiService.GetMedia(MediaType.Stories, $"/users/{userId}/stories", null, path, paidPostIds.ToList()); if (stories == null || stories.Count == 0) @@ -1454,7 +1487,7 @@ public class DownloadService( bool isNew; StreamEntities.Medium? mediaInfo = streams.StreamMedia.FirstOrDefault(m => m.Id == kvpEntry.Key); StreamEntities.ListItem? streamInfo = - streams.StreamObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); + streams.StreamObjects.FirstOrDefault(p => p.Media?.Contains(mediaInfo) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? ""; string streamPath = configService.CurrentConfig.FolderPerPost && streamInfo != null && @@ -1534,7 +1567,7 @@ public class DownloadService( bool isNew; PostEntities.Medium? mediaInfo = posts.PostMedia.FirstOrDefault(m => m?.Id == postKVP.Key); PostEntities.ListItem? postInfo = - posts.PostObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); + posts.PostObjects.FirstOrDefault(p => p.Media?.Contains(mediaInfo) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? ""; string postPath = configService.CurrentConfig.FolderPerPost && postInfo != null && @@ -1665,4 +1698,363 @@ public class DownloadService( Success = true }; } + + public async Task DownloadPaidPostsPurchasedTab(string username, string path, + Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, + PurchasedEntities.PaidPostCollection purchasedPosts, IProgressReporter progressReporter) + { + Log.Debug($"Calling DownloadPaidPostsPurchasedTab - {username}"); + + if (purchasedPosts == null || purchasedPosts.PaidPosts.Count == 0) + { + Log.Debug("Found 0 Paid Posts"); + return new DownloadResult { TotalCount = 0, MediaType = "Paid Posts", Success = true }; + } + + int oldCount = 0, newCount = 0; + + foreach (KeyValuePair purchasedPostKVP in purchasedPosts.PaidPosts) + { + bool isNew; + MessageEntities.Medium? mediaInfo = + purchasedPosts?.PaidPostMedia?.FirstOrDefault(m => m.Id == purchasedPostKVP.Key); + PurchasedEntities.ListItem? postInfo = mediaInfo != null + ? purchasedPosts?.PaidPostObjects?.FirstOrDefault(p => + p?.Media?.Any(m => m.Id == purchasedPostKVP.Key) == true) + : null; + string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) + .PaidPostFileNameFormat ?? ""; + string paidPostPath = configService.CurrentConfig.FolderPerPaidPost && postInfo != null && + postInfo?.Id is not null && postInfo?.PostedAt is not null + ? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}" + : "/Posts/Paid"; + + if (purchasedPostKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) + { + string[] parsed = purchasedPostKVP.Value.Split(','); + (string decryptionKey, DateTime lastModified)? drmInfo = + await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], + parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); + if (drmInfo == null) + { + continue; + } + + isNew = await DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], + drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, purchasedPostKVP.Key, + "Posts", progressReporter, paidPostPath + "/Videos", filenameFormat, + postInfo, mediaInfo, postInfo?.FromUser, users); + } + else + { + isNew = await DownloadMedia(purchasedPostKVP.Value, path, + purchasedPostKVP.Key, "Posts", progressReporter, + paidPostPath, filenameFormat, postInfo, mediaInfo, postInfo?.FromUser, users); + } + + if (isNew) + { + newCount++; + } + else + { + oldCount++; + } + } + + Log.Debug($"Paid Posts Already Downloaded: {oldCount} New Paid Posts Downloaded: {newCount}"); + return new DownloadResult + { + TotalCount = purchasedPosts.PaidPosts.Count, + NewDownloads = newCount, + ExistingDownloads = oldCount, + MediaType = "Paid Posts", + Success = true + }; + } + + public async Task DownloadPaidMessagesPurchasedTab(string username, string path, + Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, + PurchasedEntities.PaidMessageCollection paidMessageCollection, IProgressReporter progressReporter) + { + Log.Debug($"Calling DownloadPaidMessagesPurchasedTab - {username}"); + + if (paidMessageCollection == null || paidMessageCollection.PaidMessages.Count == 0) + { + Log.Debug("Found 0 Paid Messages"); + return new DownloadResult { TotalCount = 0, MediaType = "Paid Messages", Success = true }; + } + + int oldCount = 0, newCount = 0; + + foreach (KeyValuePair paidMessageKVP in paidMessageCollection.PaidMessages) + { + bool isNew; + MessageEntities.Medium? mediaInfo = + paidMessageCollection.PaidMessageMedia.FirstOrDefault(m => m.Id == paidMessageKVP.Key); + PurchasedEntities.ListItem? messageInfo = + paidMessageCollection.PaidMessageObjects.FirstOrDefault(p => + p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true); + string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) + .PaidMessageFileNameFormat ?? ""; + string paidMsgPath = configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null && + messageInfo?.Id is not null && messageInfo?.CreatedAt is not null + ? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" + : "/Messages/Paid"; + + if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) + { + string[] parsed = paidMessageKVP.Value.Split(','); + (string decryptionKey, DateTime lastModified)? drmInfo = + await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], + parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); + if (drmInfo == null) + { + continue; + } + + isNew = await DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], + drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKVP.Key, + "Messages", progressReporter, paidMsgPath + "/Videos", filenameFormat, + messageInfo, mediaInfo, messageInfo?.FromUser, users); + } + else + { + isNew = await DownloadMedia(paidMessageKVP.Value, path, + paidMessageKVP.Key, "Messages", progressReporter, + paidMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users); + } + + if (isNew) + { + newCount++; + } + else + { + oldCount++; + } + } + + Log.Debug($"Paid Messages Already Downloaded: {oldCount} New Paid Messages Downloaded: {newCount}"); + return new DownloadResult + { + TotalCount = paidMessageCollection.PaidMessages.Count, + NewDownloads = newCount, + ExistingDownloads = oldCount, + MediaType = "Paid Messages", + Success = true + }; + } + + public async Task DownloadSinglePost(string username, string path, + Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, + PostEntities.SinglePostCollection post, IProgressReporter progressReporter) + { + Log.Debug($"Calling DownloadSinglePost - {username}"); + + if (post == null || post.SinglePosts.Count == 0) + { + Log.Debug("Couldn't find post"); + return new DownloadResult { TotalCount = 0, MediaType = "Posts", Success = true }; + } + + int oldCount = 0, newCount = 0; + + foreach (KeyValuePair postKVP in post.SinglePosts) + { + PostEntities.Medium? mediaInfo = post.SinglePostMedia.FirstOrDefault(m => m.Id == postKVP.Key); + PostEntities.SinglePost? postInfo = + post.SinglePostObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); + string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) + .PostFileNameFormat ?? ""; + string postPath = configService.CurrentConfig.FolderPerPost && postInfo != null && + postInfo?.Id is not null && postInfo?.PostedAt is not null + ? $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}" + : "/Posts/Free"; + + bool isNew; + if (postKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) + { + string[] parsed = postKVP.Value.Split(','); + (string decryptionKey, DateTime lastModified)? drmInfo = + await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], + parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); + if (drmInfo == null) + { + continue; + } + + isNew = await DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], + drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKVP.Key, "Posts", + progressReporter, postPath + "/Videos", filenameFormat, + postInfo, mediaInfo, postInfo?.Author, users); + } + else + { + try + { + isNew = await DownloadMedia(postKVP.Value, path, + postKVP.Key, "Posts", progressReporter, + postPath, filenameFormat, postInfo, mediaInfo, postInfo?.Author, users); + } + catch + { + Log.Warning("Media was null"); + continue; + } + } + + if (isNew) + { + newCount++; + } + else + { + oldCount++; + } + } + + return new DownloadResult + { + TotalCount = post.SinglePosts.Count, + NewDownloads = newCount, + ExistingDownloads = oldCount, + MediaType = "Posts", + Success = true + }; + } + + public async Task DownloadSinglePaidMessage(string username, string path, + Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, + PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection, + IProgressReporter progressReporter) + { + Log.Debug($"Calling DownloadSinglePaidMessage - {username}"); + + if (singlePaidMessageCollection == null) + { + return new DownloadResult { TotalCount = 0, MediaType = "Paid Messages", Success = true }; + } + + int totalNew = 0, totalOld = 0; + + // Download preview messages + if (singlePaidMessageCollection.PreviewSingleMessages.Count > 0) + { + foreach (KeyValuePair paidMessageKVP in singlePaidMessageCollection.PreviewSingleMessages) + { + MessageEntities.Medium? mediaInfo = + singlePaidMessageCollection.PreviewSingleMessageMedia.FirstOrDefault(m => + m.Id == paidMessageKVP.Key); + MessageEntities.SingleMessage? messageInfo = + singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p => + p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true); + string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) + .PaidMessageFileNameFormat ?? ""; + string previewMsgPath = configService.CurrentConfig.FolderPerMessage && messageInfo != null && + messageInfo?.Id is not null && messageInfo?.CreatedAt is not null + ? $"/Messages/Free/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" + : "/Messages/Free"; + + bool isNew; + if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) + { + string[] parsed = paidMessageKVP.Value.Split(','); + (string decryptionKey, DateTime lastModified)? drmInfo = + await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], + parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); + if (drmInfo == null) + { + continue; + } + + isNew = await DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], + drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKVP.Key, + "Messages", progressReporter, previewMsgPath + "/Videos", filenameFormat, + messageInfo, mediaInfo, messageInfo?.FromUser, users); + } + else + { + isNew = await DownloadMedia(paidMessageKVP.Value, path, + paidMessageKVP.Key, "Messages", progressReporter, + previewMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users); + } + + if (isNew) + { + totalNew++; + } + else + { + totalOld++; + } + } + } + + // Download actual paid messages + if (singlePaidMessageCollection.SingleMessages.Count > 0) + { + foreach (KeyValuePair paidMessageKVP in singlePaidMessageCollection.SingleMessages) + { + MessageEntities.Medium? mediaInfo = + singlePaidMessageCollection.SingleMessageMedia.FirstOrDefault(m => + m.Id == paidMessageKVP.Key); + MessageEntities.SingleMessage? messageInfo = + singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p => + p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true); + string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) + .PaidMessageFileNameFormat ?? ""; + string singlePaidMsgPath = configService.CurrentConfig.FolderPerPaidMessage && + messageInfo != null && messageInfo?.Id is not null && + messageInfo?.CreatedAt is not null + ? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" + : "/Messages/Paid"; + + bool isNew; + if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) + { + string[] parsed = paidMessageKVP.Value.Split(','); + (string decryptionKey, DateTime lastModified)? drmInfo = + await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], + parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); + if (drmInfo == null) + { + continue; + } + + isNew = await DownloadDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], + drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKVP.Key, + "Messages", progressReporter, singlePaidMsgPath + "/Videos", filenameFormat, + messageInfo, mediaInfo, messageInfo?.FromUser, users); + } + else + { + isNew = await DownloadMedia(paidMessageKVP.Value, path, + paidMessageKVP.Key, "Messages", progressReporter, + singlePaidMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users); + } + + if (isNew) + { + totalNew++; + } + else + { + totalOld++; + } + } + } + + int totalCount = singlePaidMessageCollection.PreviewSingleMessages.Count + + singlePaidMessageCollection.SingleMessages.Count; + Log.Debug($"Paid Messages Already Downloaded: {totalOld} New Paid Messages Downloaded: {totalNew}"); + return new DownloadResult + { + TotalCount = totalCount, + NewDownloads = totalNew, + ExistingDownloads = totalOld, + MediaType = "Paid Messages", + Success = true + }; + } } diff --git a/OF DL/Services/FileNameService.cs b/OF DL/Services/FileNameService.cs index 4643637..cd8732e 100644 --- a/OF DL/Services/FileNameService.cs +++ b/OF DL/Services/FileNameService.cs @@ -115,7 +115,7 @@ public class FileNameService(IAuthService authService) : IFileNameService else { object? nestedPropertyValue = GetNestedPropertyValue(author, "Id"); - if (nestedPropertyValue != null) + if (nestedPropertyValue != null && users != null) { values.Add(propertyName, users.FirstOrDefault(u => u.Value == Convert.ToInt32(nestedPropertyValue.ToString())).Key); @@ -132,11 +132,11 @@ public class FileNameService(IAuthService authService) : IFileNameService if (propertyValue != null) { HtmlDocument pageDoc = new(); - pageDoc.LoadHtml(propertyValue.ToString()); + pageDoc.LoadHtml(propertyValue.ToString() ?? ""); string str = pageDoc.DocumentNode.InnerText; - if (str.Length > 100) // todo: add length limit to config + if (str.Length > 100) // TODO: add length limit to config { - str = str.Substring(0, 100); + str = str[..100]; } values.Add(propertyName, str); diff --git a/OF DL/Services/IAPIService.cs b/OF DL/Services/IAPIService.cs index fe55d1f..9fa9349 100644 --- a/OF DL/Services/IAPIService.cs +++ b/OF DL/Services/IAPIService.cs @@ -6,13 +6,11 @@ using PostEntities = OF_DL.Models.Entities.Posts; using PurchasedEntities = OF_DL.Models.Entities.Purchased; using StreamEntities = OF_DL.Models.Entities.Streams; using UserEntities = OF_DL.Models.Entities.Users; -using Spectre.Console; namespace OF_DL.Services; public interface IAPIService { - Task GetDecryptionKeyCDRMProject(Dictionary drmHeaders, string licenceURL, string pssh); Task GetDecryptionKeyCDM(Dictionary drmHeaders, string licenceURL, string pssh); Task GetDRMMPDLastModified(string mpdUrl, string policy, string signature, string kvp); Task GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp); @@ -24,21 +22,23 @@ public interface IAPIService Task GetPaidPosts(string endpoint, string folder, string username, List paid_post_ids, - StatusContext ctx); + IStatusReporter statusReporter); Task GetPosts(string endpoint, string folder, List paid_post_ids, - StatusContext ctx); + IStatusReporter statusReporter); Task GetPost(string endpoint, string folder); Task GetStreams(string endpoint, string folder, List paid_post_ids, - StatusContext ctx); + IStatusReporter statusReporter); - Task GetArchived(string endpoint, string folder, StatusContext ctx); - Task GetMessages(string endpoint, string folder, StatusContext ctx); + Task GetArchived(string endpoint, string folder, + IStatusReporter statusReporter); + + Task GetMessages(string endpoint, string folder, IStatusReporter statusReporter); Task GetPaidMessages(string endpoint, string folder, string username, - StatusContext ctx); + IStatusReporter statusReporter); Task GetPaidMessage(string endpoint, string folder); Task> GetPurchasedTabUsers(string endpoint, Dictionary users); diff --git a/OF DL/Services/IAuthService.cs b/OF DL/Services/IAuthService.cs index 485cfdc..c990bfd 100644 --- a/OF DL/Services/IAuthService.cs +++ b/OF DL/Services/IAuthService.cs @@ -1,11 +1,30 @@ using OF_DL.Models; +using UserEntities = OF_DL.Models.Entities.Users; namespace OF_DL.Services; public interface IAuthService { Auth? CurrentAuth { get; set; } + Task LoadFromFileAsync(string filePath = "auth.json"); + Task LoadFromBrowserAsync(); + Task SaveToFileAsync(string filePath = "auth.json"); + + /// + /// Cleans up the cookie string to only contain auth_id and sess cookies. + /// + void ValidateCookieString(); + + /// + /// Validates auth by calling the API and returns the user info if valid. + /// + Task ValidateAuthAsync(); + + /// + /// Logs out by deleting chrome-data and auth.json. + /// + void Logout(); } diff --git a/OF DL/Services/IConfigService.cs b/OF DL/Services/IConfigService.cs index abb535b..6a31ba3 100644 --- a/OF DL/Services/IConfigService.cs +++ b/OF DL/Services/IConfigService.cs @@ -5,8 +5,22 @@ namespace OF_DL.Services; public interface IConfigService { Config CurrentConfig { get; } + bool IsCliNonInteractive { get; } + Task LoadConfigurationAsync(string[] args); + Task SaveConfigurationAsync(string filePath = "config.conf"); + void UpdateConfig(Config newConfig); + + /// + /// Returns property names and current values for toggleable config properties. + /// + List<(string Name, bool Value)> GetToggleableProperties(); + + /// + /// Applies selected toggleable properties. Returns true if any changed. + /// + bool ApplyToggleableSelections(List selectedNames); } diff --git a/OF DL/Services/IDownloadEventHandler.cs b/OF DL/Services/IDownloadEventHandler.cs new file mode 100644 index 0000000..d40788c --- /dev/null +++ b/OF DL/Services/IDownloadEventHandler.cs @@ -0,0 +1,63 @@ +using OF_DL.Models; + +namespace OF_DL.Services; + +/// +/// UI callback contract for download orchestration. Implementations handle +/// status display, progress bars, and notifications in a UI-framework-specific way. +/// +public interface IDownloadEventHandler +{ + /// + /// Wraps work in a status indicator (spinner) during API fetching. + /// The implementation controls how the status is displayed. + /// + Task WithStatusAsync(string statusMessage, Func> work); + + /// + /// Wraps work in a progress bar during downloading. + /// The implementation controls how progress is displayed. + /// + Task WithProgressAsync(string description, long maxValue, bool showSize, + Func> work); + + /// + /// Called when content of a specific type is found for a creator. + /// + void OnContentFound(string contentType, int mediaCount, int objectCount); + + /// + /// Called when no content of a specific type is found for a creator. + /// + void OnNoContentFound(string contentType); + + /// + /// Called when downloading of a content type completes. + /// + void OnDownloadComplete(string contentType, DownloadResult result); + + /// + /// Called when starting to process a specific user/creator. + /// + void OnUserStarting(string username); + + /// + /// Called when all downloads for a user/creator are complete. + /// + void OnUserComplete(string username, CreatorDownloadResult result); + + /// + /// Called when a purchased tab user's downloads are complete. + /// + void OnPurchasedTabUserComplete(string username, int paidPostCount, int paidMessagesCount); + + /// + /// Called when the entire scrape operation completes. + /// + void OnScrapeComplete(TimeSpan elapsed); + + /// + /// General status message display. + /// + void OnMessage(string message); +} diff --git a/OF DL/Services/IDownloadOrchestrationService.cs b/OF DL/Services/IDownloadOrchestrationService.cs new file mode 100644 index 0000000..44e48c6 --- /dev/null +++ b/OF DL/Services/IDownloadOrchestrationService.cs @@ -0,0 +1,73 @@ +using OF_DL.Models; +using UserEntities = OF_DL.Models.Entities.Users; + +namespace OF_DL.Services; + +public interface IDownloadOrchestrationService +{ + /// + /// Fetch subscriptions, lists, filter ignored users. + /// + Task GetAvailableUsersAsync(); + + /// + /// Get users for a specific list by name. + /// + Task> GetUsersForListAsync( + string listName, Dictionary allUsers, Dictionary lists); + + /// + /// Resolve download path for a username based on config. + /// + string ResolveDownloadPath(string username); + + /// + /// Prepare user folder (create dir, check username, create DB). + /// + Task PrepareUserFolderAsync(string username, long userId, string path); + + /// + /// Download all configured content types for a single creator. + /// + Task DownloadCreatorContentAsync( + string username, long userId, string path, + Dictionary users, + bool clientIdBlobMissing, bool devicePrivateKeyMissing, + IDownloadEventHandler eventHandler); + + /// + /// Download a single post by ID. + /// + Task DownloadSinglePostAsync( + string username, long postId, string path, + Dictionary users, + bool clientIdBlobMissing, bool devicePrivateKeyMissing, + IDownloadEventHandler eventHandler); + + /// + /// Download purchased tab content for all users. + /// + Task DownloadPurchasedTabAsync( + Dictionary users, + bool clientIdBlobMissing, bool devicePrivateKeyMissing, + IDownloadEventHandler eventHandler); + + /// + /// Download a single paid message by message ID. + /// + Task DownloadSinglePaidMessageAsync( + string username, long messageId, string path, + Dictionary users, + bool clientIdBlobMissing, bool devicePrivateKeyMissing, + IDownloadEventHandler eventHandler); + + /// + /// Resolve username from user ID via API. + /// + Task ResolveUsernameAsync(long userId); + + /// + /// Tracks paid post IDs across downloads. + /// + List PaidPostIds { get; } +} diff --git a/OF DL/Services/IDownloadService.cs b/OF DL/Services/IDownloadService.cs index c46996a..5f95a14 100644 --- a/OF DL/Services/IDownloadService.cs +++ b/OF DL/Services/IDownloadService.cs @@ -62,4 +62,21 @@ public interface IDownloadService Task DownloadPaidPosts(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PurchasedEntities.PaidPostCollection purchasedPosts, IProgressReporter progressReporter); + + Task DownloadPaidPostsPurchasedTab(string username, string path, + Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, + PurchasedEntities.PaidPostCollection purchasedPosts, IProgressReporter progressReporter); + + Task DownloadPaidMessagesPurchasedTab(string username, string path, + Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, + PurchasedEntities.PaidMessageCollection paidMessageCollection, IProgressReporter progressReporter); + + Task DownloadSinglePost(string username, string path, + Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, + PostEntities.SinglePostCollection post, IProgressReporter progressReporter); + + Task DownloadSinglePaidMessage(string username, string path, + Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, + PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection, + IProgressReporter progressReporter); } diff --git a/OF DL/Services/IStartupService.cs b/OF DL/Services/IStartupService.cs new file mode 100644 index 0000000..3b54466 --- /dev/null +++ b/OF DL/Services/IStartupService.cs @@ -0,0 +1,10 @@ +using OF_DL.Models; + +namespace OF_DL.Services; + +public interface IStartupService +{ + Task ValidateEnvironmentAsync(); + + Task CheckVersionAsync(); +} diff --git a/OF DL/Services/IStatusReporter.cs b/OF DL/Services/IStatusReporter.cs new file mode 100644 index 0000000..fe693c5 --- /dev/null +++ b/OF DL/Services/IStatusReporter.cs @@ -0,0 +1,14 @@ +namespace OF_DL.Services; + +/// +/// Interface for reporting status updates in a UI-agnostic way. +/// This replaces Spectre.Console's StatusContext in the service layer. +/// +public interface IStatusReporter +{ + /// + /// Reports a status message (e.g., "Getting Posts\n Found 42"). + /// The reporter implementation decides how to format and display the message. + /// + void ReportStatus(string message); +} diff --git a/OF DL/Services/StartupService.cs b/OF DL/Services/StartupService.cs new file mode 100644 index 0000000..6c0cfad --- /dev/null +++ b/OF DL/Services/StartupService.cs @@ -0,0 +1,254 @@ +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using Newtonsoft.Json; +using OF_DL.Helpers; +using OF_DL.Models; +using Serilog; +using WidevineConstants = OF_DL.Widevine.Constants; + +namespace OF_DL.Services; + +public class StartupService(IConfigService configService, IAuthService authService) : IStartupService +{ + public async Task ValidateEnvironmentAsync() + { + StartupResult result = new(); + + // OS validation + OperatingSystem os = Environment.OSVersion; + result.OsVersionString = os.VersionString; + Log.Debug($"Operating system information: {os.VersionString}"); + + if (os.Platform == PlatformID.Win32NT && os.Version.Major < 10) + { + result.IsWindowsVersionValid = false; + Log.Error("Windows version prior to 10.x: {0}", os.VersionString); + } + + // FFmpeg detection + DetectFfmpeg(result); + + if (result.FfmpegFound) + { + // Escape backslashes for Windows + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && + result.FfmpegPath!.Contains(@":\") && + !result.FfmpegPath.Contains(@":\\")) + { + result.FfmpegPath = result.FfmpegPath.Replace(@"\", @"\\"); + configService.CurrentConfig!.FFmpegPath = result.FfmpegPath; + } + + // Get FFmpeg version + result.FfmpegVersion = await GetFfmpegVersionAsync(result.FfmpegPath!); + } + + // Widevine device checks + result.ClientIdBlobMissing = !File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER, + WidevineConstants.DEVICE_NAME, "device_client_id_blob")); + result.DevicePrivateKeyMissing = !File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER, + WidevineConstants.DEVICE_NAME, "device_private_key")); + + if (!result.ClientIdBlobMissing) + { + Log.Debug("device_client_id_blob found"); + } + else + { + Log.Debug("device_client_id_blob missing"); + } + + if (!result.DevicePrivateKeyMissing) + { + Log.Debug("device_private_key found"); + } + else + { + Log.Debug("device_private_key missing"); + } + + // rules.json validation + if (File.Exists("rules.json")) + { + result.RulesJsonExists = true; + try + { + JsonConvert.DeserializeObject(File.ReadAllText("rules.json")); + Log.Debug("Rules.json: "); + Log.Debug(JsonConvert.SerializeObject(File.ReadAllText("rules.json"), Formatting.Indented)); + result.RulesJsonValid = true; + } + catch (Exception e) + { + result.RulesJsonError = e.Message; + Log.Error("rules.json processing failed.", e.Message); + } + } + + return result; + } + + public async Task CheckVersionAsync() + { + VersionCheckResult result = new(); + +#if !DEBUG + try + { + result.LocalVersion = Assembly.GetEntryAssembly()?.GetName().Version; + + using CancellationTokenSource cts = new(TimeSpan.FromSeconds(30)); + string? latestReleaseTag = null; + + try + { + latestReleaseTag = await VersionHelper.GetLatestReleaseTag(cts.Token); + } + catch (OperationCanceledException) + { + result.TimedOut = true; + Log.Warning("Version check timed out after 30 seconds"); + return result; + } + + if (latestReleaseTag == null) + { + result.CheckFailed = true; + Log.Error("Failed to get the latest release tag."); + return result; + } + + result.LatestVersion = new Version(latestReleaseTag.Replace("OFDLV", "")); + int versionComparison = result.LocalVersion!.CompareTo(result.LatestVersion); + result.IsUpToDate = versionComparison >= 0; + + Log.Debug("Detected client running version " + + $"{result.LocalVersion.Major}.{result.LocalVersion.Minor}.{result.LocalVersion.Build}"); + Log.Debug("Latest release version " + + $"{result.LatestVersion.Major}.{result.LatestVersion.Minor}.{result.LatestVersion.Build}"); + } + catch (Exception e) + { + result.CheckFailed = true; + Log.Error("Error checking latest release on GitHub.", e.Message); + } +#else + Log.Debug("Running in Debug/Local mode. Version check skipped."); + result.IsUpToDate = true; +#endif + + return result; + } + + private void DetectFfmpeg(StartupResult result) + { + if (!string.IsNullOrEmpty(configService.CurrentConfig!.FFmpegPath) && + ValidateFilePath(configService.CurrentConfig.FFmpegPath)) + { + result.FfmpegFound = true; + result.FfmpegPath = configService.CurrentConfig.FFmpegPath; + Log.Debug($"FFMPEG found: {result.FfmpegPath}"); + Log.Debug("FFMPEG path set in config.conf"); + } + else if (!string.IsNullOrEmpty(authService.CurrentAuth?.FfmpegPath) && + ValidateFilePath(authService.CurrentAuth.FfmpegPath)) + { + result.FfmpegFound = true; + result.FfmpegPath = authService.CurrentAuth.FfmpegPath; + configService.CurrentConfig.FFmpegPath = result.FfmpegPath; + Log.Debug($"FFMPEG found: {result.FfmpegPath}"); + Log.Debug("FFMPEG path set in auth.json"); + } + else if (string.IsNullOrEmpty(configService.CurrentConfig.FFmpegPath)) + { + string? ffmpegPath = GetFullPath("ffmpeg") ?? GetFullPath("ffmpeg.exe"); + if (ffmpegPath != null) + { + result.FfmpegFound = true; + result.FfmpegPathAutoDetected = true; + result.FfmpegPath = ffmpegPath; + configService.CurrentConfig.FFmpegPath = ffmpegPath; + Log.Debug($"FFMPEG found: {ffmpegPath}"); + Log.Debug("FFMPEG path found via PATH or current directory"); + } + } + + if (!result.FfmpegFound) + { + Log.Error($"Cannot locate FFmpeg with path: {configService.CurrentConfig.FFmpegPath}"); + } + } + + private static async Task GetFfmpegVersionAsync(string ffmpegPath) + { + try + { + ProcessStartInfo processStartInfo = new() + { + FileName = ffmpegPath, + Arguments = "-version", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using Process? process = Process.Start(processStartInfo); + if (process != null) + { + string output = await process.StandardOutput.ReadToEndAsync(); + await process.WaitForExitAsync(); + + Log.Information("FFmpeg version output:\n{Output}", output); + + string firstLine = output.Split('\n')[0].Trim(); + if (firstLine.StartsWith("ffmpeg version")) + { + int versionStart = "ffmpeg version ".Length; + int copyrightIndex = firstLine.IndexOf(" Copyright"); + return copyrightIndex > versionStart + ? firstLine.Substring(versionStart, copyrightIndex - versionStart) + : firstLine.Substring(versionStart); + } + } + } + catch (Exception ex) + { + Log.Warning(ex, "Failed to get FFmpeg version"); + } + + return null; + } + + private static bool ValidateFilePath(string path) + { + char[] invalidChars = Path.GetInvalidPathChars(); + if (path.Any(c => invalidChars.Contains(c))) + { + return false; + } + + return File.Exists(path); + } + + public static string? GetFullPath(string filename) + { + if (File.Exists(filename)) + { + return Path.GetFullPath(filename); + } + + string pathEnv = Environment.GetEnvironmentVariable("PATH") ?? ""; + foreach (string path in pathEnv.Split(Path.PathSeparator)) + { + string fullPath = Path.Combine(path, filename); + if (File.Exists(fullPath)) + { + return fullPath; + } + } + + return null; + } +} diff --git a/OF DL/Widevine/CDM.cs b/OF DL/Widevine/CDM.cs index 64e352c..9cb20c3 100644 --- a/OF DL/Widevine/CDM.cs +++ b/OF DL/Widevine/CDM.cs @@ -78,7 +78,7 @@ public class CDM } Session session; - dynamic parsedInitData = ParseInitData(initData); + dynamic? parsedInitData = ParseInitData(initData); if (parsedInitData != null) { @@ -409,18 +409,22 @@ public class CDM byte[] decryptedKey; using MemoryStream mstream = new(); - using AesCryptoServiceProvider aesProvider = new() { Mode = CipherMode.CBC, Padding = PaddingMode.PKCS7 }; + + using AesCryptoServiceProvider aesProvider = new(); + aesProvider.Mode = CipherMode.CBC; + aesProvider.Padding = PaddingMode.PKCS7; + using CryptoStream cryptoStream = new(mstream, aesProvider.CreateDecryptor(session.DerivedKeys.Enc, iv), CryptoStreamMode.Write); cryptoStream.Write(encryptedKey, 0, encryptedKey.Length); decryptedKey = mstream.ToArray(); - List permissions = new(); + List permissions = []; if (type == "OperatorSession") { foreach (PropertyInfo perm in key._OperatorSessionKeyPermissions.GetType().GetProperties()) { - if ((uint)perm.GetValue(key._OperatorSessionKeyPermissions) == 1) + if ((uint?)perm.GetValue(key._OperatorSessionKeyPermissions) == 1) { permissions.Add(perm.Name); } diff --git a/OF DL/Widevine/CDMApi.cs b/OF DL/Widevine/CDMApi.cs index c1d3193..f00c83b 100644 --- a/OF DL/Widevine/CDMApi.cs +++ b/OF DL/Widevine/CDMApi.cs @@ -1,20 +1,32 @@ +using Serilog; + namespace OF_DL.Widevine; public class CDMApi { - private string SessionId { get; set; } + private string? SessionId { get; set; } - public byte[] GetChallenge(string initDataB64, string certDataB64, bool offline = false, bool raw = false) + public byte[]? GetChallenge(string initDataB64, string certDataB64, bool offline = false, bool raw = false) { SessionId = CDM.OpenSession(initDataB64, Constants.DEVICE_NAME, offline, raw); + if (SessionId == null) + { + Log.Debug("CDM.OpenSession returned null, unable to proceed with challenge generation"); + return null; + } + CDM.SetServiceCertificate(SessionId, Convert.FromBase64String(certDataB64)); return CDM.GetLicenseRequest(SessionId); } - public bool ProvideLicense(string licenseB64) + public void ProvideLicense(string licenseB64) { + if (SessionId == null) + { + throw new Exception("No session ID set. Could not provide license"); + } + CDM.ProvideLicense(SessionId, Convert.FromBase64String(licenseB64)); - return true; } public List GetKeys() => CDM.GetKeys(SessionId);