forked from sim0n00ps/OF-DL
2658 lines
122 KiB
C#
2658 lines
122 KiB
C#
using System.Diagnostics;
|
|
using System.Reflection;
|
|
using System.Runtime.InteropServices;
|
|
using System.Text.RegularExpressions;
|
|
using Microsoft.Extensions.DependencyInjection;
|
|
using Newtonsoft.Json;
|
|
using Newtonsoft.Json.Linq;
|
|
using OF_DL.CLI;
|
|
using OF_DL.Models;
|
|
using OF_DL.Enumerations;
|
|
using OF_DL.Helpers;
|
|
using ArchivedEntities = OF_DL.Models.Entities.Archived;
|
|
using MessageEntities = OF_DL.Models.Entities.Messages;
|
|
using PostEntities = OF_DL.Models.Entities.Posts;
|
|
using PurchasedEntities = OF_DL.Models.Entities.Purchased;
|
|
using StreamEntities = OF_DL.Models.Entities.Streams;
|
|
using UserEntities = OF_DL.Models.Entities.Users;
|
|
using OF_DL.Services;
|
|
using Serilog;
|
|
using Spectre.Console;
|
|
using WidevineConstants = OF_DL.Widevine.Constants;
|
|
|
|
namespace OF_DL;
|
|
|
|
public class Program(IServiceProvider serviceProvider)
|
|
{
|
|
public static List<long> paid_post_ids = new();
|
|
|
|
private static bool clientIdBlobMissing;
|
|
private static bool devicePrivateKeyMissing;
|
|
|
|
private async Task LoadAuthFromBrowser()
|
|
{
|
|
IAuthService authService = serviceProvider.GetRequiredService<IAuthService>();
|
|
bool runningInDocker = Environment.GetEnvironmentVariable("OFDL_DOCKER") != null;
|
|
|
|
// Show the initial message
|
|
AnsiConsole.MarkupLine("[yellow]Downloading dependencies. Please wait ...[/]");
|
|
|
|
// Show instructions based on the environment
|
|
await Task.Delay(5000);
|
|
if (runningInDocker)
|
|
{
|
|
AnsiConsole.MarkupLine(
|
|
"[yellow]In your web browser, navigate to the port forwarded from your docker container.[/]");
|
|
AnsiConsole.MarkupLine(
|
|
"[yellow]For instance, if your docker run command included \"-p 8080:8080\", open your web browser to \"http://localhost:8080\".[/]");
|
|
AnsiConsole.MarkupLine(
|
|
"[yellow]Once on that webpage, please use it to log in to your OF account. Do not navigate away from the page.[/]");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.MarkupLine(
|
|
"[yellow]In the new window that has opened, please log in to your OF account. Do not close the window or tab. Do not navigate away from the page.[/]\n");
|
|
AnsiConsole.MarkupLine(
|
|
"[yellow]Note: Some users have reported that \"Sign in with Google\" has not been working with the new authentication method.[/]");
|
|
AnsiConsole.MarkupLine(
|
|
"[yellow]If you use this method or encounter other issues while logging in, use one of the legacy authentication methods documented here:[/]");
|
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
|
}
|
|
|
|
// Load auth from browser using the service
|
|
bool success = await authService.LoadFromBrowserAsync();
|
|
|
|
if (!success || authService.CurrentAuth == null)
|
|
{
|
|
AnsiConsole.MarkupLine(
|
|
"\n[red]Authentication failed. Be sure to log into to OF using the new window that opened automatically.[/]");
|
|
AnsiConsole.MarkupLine(
|
|
"[red]The window will close automatically when the authentication process is finished.[/]");
|
|
AnsiConsole.MarkupLine(
|
|
"[red]If the problem persists, you may want to try using a legacy authentication method documented here:[/]\n");
|
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
|
Log.Error("auth invalid after attempt to get auth from browser");
|
|
|
|
Environment.Exit(2);
|
|
}
|
|
|
|
await authService.SaveToFileAsync();
|
|
}
|
|
|
|
public static async Task Main(string[] args)
|
|
{
|
|
AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
|
|
AnsiConsole.Markup("Documentation: [link]https://docs.ofdl.tools/[/]\n");
|
|
AnsiConsole.Markup("Discord server: [link]https://discord.com/invite/6bUW8EJ53j[/]\n\n");
|
|
|
|
|
|
ServiceCollection services = await ConfigureServices(args);
|
|
ServiceProvider serviceProvider = services.BuildServiceProvider();
|
|
|
|
// Get the Program instance and run
|
|
Program program = serviceProvider.GetRequiredService<Program>();
|
|
await program.RunAsync();
|
|
}
|
|
|
|
private static async Task<ServiceCollection> ConfigureServices(string[] args)
|
|
{
|
|
// Set up dependency injection with LoggingService and ConfigService
|
|
ServiceCollection services = new();
|
|
services.AddSingleton<ILoggingService, LoggingService>();
|
|
services.AddSingleton<IConfigService, ConfigService>();
|
|
ServiceProvider tempServiceProvider = services.BuildServiceProvider();
|
|
|
|
ILoggingService loggingService = tempServiceProvider.GetRequiredService<ILoggingService>();
|
|
IConfigService configService = tempServiceProvider.GetRequiredService<IConfigService>();
|
|
|
|
|
|
if (!await configService.LoadConfigurationAsync(args))
|
|
{
|
|
AnsiConsole.MarkupLine("\n[red]config.conf is not valid, check your syntax![/]\n");
|
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
|
if (!configService.IsCliNonInteractive)
|
|
{
|
|
Console.ReadKey();
|
|
}
|
|
|
|
Environment.Exit(3);
|
|
}
|
|
|
|
AnsiConsole.Markup("[green]config.conf located successfully!\n[/]");
|
|
|
|
// Set up full dependency injection with loaded config
|
|
services = [];
|
|
services.AddSingleton(loggingService);
|
|
services.AddSingleton(configService);
|
|
services.AddSingleton<IAuthService, AuthService>();
|
|
services.AddSingleton<IAPIService, APIService>();
|
|
services.AddSingleton<IDBService, DBService>();
|
|
services.AddSingleton<IDownloadService, DownloadService>();
|
|
services.AddSingleton<IFileNameService, FileNameService>();
|
|
services.AddSingleton<Program>();
|
|
|
|
return services;
|
|
}
|
|
|
|
private async Task RunAsync()
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAuthService authService = serviceProvider.GetRequiredService<IAuthService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
|
|
try
|
|
{
|
|
OperatingSystem os = Environment.OSVersion;
|
|
|
|
Log.Debug($"Operating system information: {os.VersionString}");
|
|
|
|
if (os.Platform == PlatformID.Win32NT)
|
|
{
|
|
// check if this is windows 10+
|
|
if (os.Version.Major < 10)
|
|
{
|
|
Console.Write(
|
|
"This appears to be running on an older version of Windows which is not supported.\n\n");
|
|
Console.Write(
|
|
"OF-DL requires Windows 10 or higher when being run on Windows. Your reported version is: {0}\n\n",
|
|
os.VersionString);
|
|
Console.Write("Press any key to continue.\n");
|
|
Log.Error("Windows version prior to 10.x: {0}", os.VersionString);
|
|
|
|
if (!configService.CurrentConfig.NonInteractiveMode)
|
|
{
|
|
Console.ReadKey();
|
|
}
|
|
|
|
Environment.Exit(1);
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup("[green]Valid version of Windows found.\n[/]");
|
|
}
|
|
}
|
|
|
|
try
|
|
{
|
|
// Only run the version check if not in DEBUG mode
|
|
#if !DEBUG
|
|
Version localVersion =
|
|
Assembly.GetEntryAssembly()?.GetName().Version; //Only tested with numeric values.
|
|
|
|
// Create a cancellation token with 30 second timeout
|
|
using CancellationTokenSource cts = new(TimeSpan.FromSeconds(30));
|
|
string? latestReleaseTag = null;
|
|
|
|
try
|
|
{
|
|
latestReleaseTag = await VersionHelper.GetLatestReleaseTag(cts.Token);
|
|
}
|
|
catch (OperationCanceledException)
|
|
{
|
|
AnsiConsole.Markup("[yellow]Version check timed out after 30 seconds.\n[/]");
|
|
Log.Warning("Version check timed out after 30 seconds");
|
|
latestReleaseTag = null;
|
|
}
|
|
|
|
if (latestReleaseTag == null)
|
|
{
|
|
AnsiConsole.Markup("[yellow]Failed to verify that OF-DL is up-to-date.\n[/]");
|
|
Log.Error("Failed to get the latest release tag.");
|
|
}
|
|
else
|
|
{
|
|
Version latestGiteaRelease = new(latestReleaseTag.Replace("OFDLV", ""));
|
|
|
|
// Compare the Versions
|
|
int versionComparison = localVersion.CompareTo(latestGiteaRelease);
|
|
if (versionComparison < 0)
|
|
{
|
|
// The version on GitHub is more up to date than this local release.
|
|
AnsiConsole.Markup("[red]You are running OF-DL version " +
|
|
$"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}\n[/]");
|
|
AnsiConsole.Markup("[red]Please update to the current release, " +
|
|
$"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}: [link=https://git.ofdl.tools/sim0n00ps/OF-DL/releases]https://git.ofdl.tools/sim0n00ps/OF-DL/releases[/]\n[/]");
|
|
Log.Debug("Detected outdated client running version " +
|
|
$"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}");
|
|
Log.Debug("Latest release version " +
|
|
$"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}");
|
|
}
|
|
else
|
|
{
|
|
// This local version is greater than the release version on GitHub.
|
|
AnsiConsole.Markup("[green]You are running OF-DL version " +
|
|
$"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}\n[/]");
|
|
AnsiConsole.Markup("[green]Latest Release version: " +
|
|
$"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}\n[/]");
|
|
Log.Debug("Detected client running version " +
|
|
$"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}");
|
|
Log.Debug("Latest release version " +
|
|
$"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}");
|
|
}
|
|
}
|
|
|
|
#else
|
|
AnsiConsole.Markup("[yellow]Running in Debug/Local mode. Version check skipped.\n[/]");
|
|
Log.Debug("Running in Debug/Local mode. Version check skipped.");
|
|
#endif
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
AnsiConsole.Markup("[red]Error checking latest release on GitHub:\n[/]");
|
|
Console.WriteLine(e);
|
|
Log.Error("Error checking latest release on GitHub.", e.Message);
|
|
}
|
|
|
|
|
|
if (await authService.LoadFromFileAsync())
|
|
{
|
|
AnsiConsole.Markup("[green]auth.json located successfully!\n[/]");
|
|
}
|
|
else if (File.Exists("auth.json"))
|
|
{
|
|
// File exists but failed to load
|
|
Log.Information("Auth file found but could not be deserialized");
|
|
if (!configService.CurrentConfig!.DisableBrowserAuth)
|
|
{
|
|
Log.Debug("Deleting auth.json");
|
|
File.Delete("auth.json");
|
|
}
|
|
|
|
if (configService.CurrentConfig.NonInteractiveMode)
|
|
{
|
|
AnsiConsole.MarkupLine(
|
|
"\n[red]auth.json has invalid JSON syntax. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n");
|
|
AnsiConsole.MarkupLine(
|
|
"[red]You may also want to try using the browser extension which is documented here:[/]\n");
|
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
|
|
|
Console.ReadKey();
|
|
Environment.Exit(2);
|
|
}
|
|
|
|
|
|
if (!configService.CurrentConfig!.DisableBrowserAuth)
|
|
{
|
|
await LoadAuthFromBrowser();
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.MarkupLine(
|
|
"\n[red]auth.json is missing. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n");
|
|
AnsiConsole.MarkupLine(
|
|
"[red]You may also want to try using the browser extension which is documented here:[/]\n");
|
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
|
|
|
Console.ReadKey();
|
|
Environment.Exit(2);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (configService.CurrentConfig.NonInteractiveMode)
|
|
{
|
|
AnsiConsole.MarkupLine(
|
|
"\n[red]auth.json is missing. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n");
|
|
AnsiConsole.MarkupLine(
|
|
"[red]You may also want to try using the browser extension which is documented here:[/]\n");
|
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
|
|
|
Console.ReadKey();
|
|
Environment.Exit(2);
|
|
}
|
|
|
|
if (!configService.CurrentConfig!.DisableBrowserAuth)
|
|
{
|
|
await LoadAuthFromBrowser();
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.MarkupLine(
|
|
"\n[red]auth.json is missing. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n");
|
|
AnsiConsole.MarkupLine(
|
|
"[red]You may also want to try using the browser extension which is documented here:[/]\n");
|
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
|
|
|
Console.ReadKey();
|
|
Environment.Exit(2);
|
|
}
|
|
}
|
|
|
|
//Added to stop cookie being filled with un-needed headers
|
|
ValidateCookieString(authService.CurrentAuth!);
|
|
|
|
if (File.Exists("rules.json"))
|
|
{
|
|
AnsiConsole.Markup("[green]rules.json located successfully!\n[/]");
|
|
try
|
|
{
|
|
JsonConvert.DeserializeObject<DynamicRules>(File.ReadAllText("rules.json"));
|
|
Log.Debug("Rules.json: ");
|
|
Log.Debug(JsonConvert.SerializeObject(File.ReadAllText("rules.json"), Formatting.Indented));
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.WriteLine(e);
|
|
AnsiConsole.MarkupLine("\n[red]rules.json is not valid, check your JSON syntax![/]\n");
|
|
AnsiConsole.MarkupLine("[red]Please ensure you are using the latest version of the software.[/]\n");
|
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
|
Log.Error("rules.json processing failed.", e.Message);
|
|
|
|
if (!configService.CurrentConfig.NonInteractiveMode)
|
|
{
|
|
Console.ReadKey();
|
|
}
|
|
|
|
Environment.Exit(2);
|
|
}
|
|
}
|
|
|
|
if (configService.CurrentConfig.NonInteractiveMode)
|
|
{
|
|
// CLI argument overrides configuration
|
|
configService.CurrentConfig!.NonInteractiveMode = true;
|
|
Log.Debug("NonInteractiveMode = true");
|
|
}
|
|
|
|
if (configService.CurrentConfig!.NonInteractiveMode)
|
|
{
|
|
configService.CurrentConfig.NonInteractiveMode =
|
|
true; // If it was set in the config, reset the cli value so exception handling works
|
|
Log.Debug("NonInteractiveMode = true (set via config)");
|
|
}
|
|
|
|
bool ffmpegFound = false;
|
|
bool pathAutoDetected = false;
|
|
if (!string.IsNullOrEmpty(configService.CurrentConfig!.FFmpegPath) &&
|
|
ValidateFilePath(configService.CurrentConfig.FFmpegPath))
|
|
{
|
|
// FFmpeg path is set in config.json and is valid
|
|
ffmpegFound = true;
|
|
Log.Debug($"FFMPEG found: {configService.CurrentConfig.FFmpegPath}");
|
|
Log.Debug("FFMPEG path set in config.conf");
|
|
}
|
|
else if (!string.IsNullOrEmpty(authService.CurrentAuth!.FfmpegPath) &&
|
|
ValidateFilePath(authService.CurrentAuth.FfmpegPath))
|
|
{
|
|
// FFmpeg path is set in auth.json and is valid (config.conf takes precedence and auth.json is only available for backward compatibility)
|
|
ffmpegFound = true;
|
|
configService.CurrentConfig.FFmpegPath = authService.CurrentAuth.FfmpegPath;
|
|
Log.Debug($"FFMPEG found: {configService.CurrentConfig.FFmpegPath}");
|
|
Log.Debug("FFMPEG path set in auth.json");
|
|
}
|
|
else if (string.IsNullOrEmpty(configService.CurrentConfig.FFmpegPath))
|
|
{
|
|
// FFmpeg path is not set in config.conf, so we will try to locate it in the PATH or current directory
|
|
string? ffmpegPath = GetFullPath("ffmpeg");
|
|
if (ffmpegPath != null)
|
|
{
|
|
// FFmpeg is found in the PATH or current directory
|
|
ffmpegFound = true;
|
|
pathAutoDetected = true;
|
|
configService.CurrentConfig.FFmpegPath = ffmpegPath;
|
|
Log.Debug($"FFMPEG found: {ffmpegPath}");
|
|
Log.Debug("FFMPEG path found via PATH or current directory");
|
|
}
|
|
else
|
|
{
|
|
// FFmpeg is not found in the PATH or current directory, so we will try to locate the windows executable
|
|
ffmpegPath = GetFullPath("ffmpeg.exe");
|
|
if (ffmpegPath != null)
|
|
{
|
|
// FFmpeg windows executable is found in the PATH or current directory
|
|
ffmpegFound = true;
|
|
pathAutoDetected = true;
|
|
configService.CurrentConfig.FFmpegPath = ffmpegPath;
|
|
Log.Debug($"FFMPEG found: {ffmpegPath}");
|
|
Log.Debug("FFMPEG path found in windows excutable directory");
|
|
}
|
|
}
|
|
}
|
|
|
|
if (ffmpegFound)
|
|
{
|
|
if (pathAutoDetected)
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[green]FFmpeg located successfully. Path auto-detected: {configService.CurrentConfig.FFmpegPath}\n[/]");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup("[green]FFmpeg located successfully\n[/]");
|
|
}
|
|
|
|
// Escape backslashes in the path for Windows
|
|
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
|
|
configService.CurrentConfig.FFmpegPath!.Contains(@":\") &&
|
|
!configService.CurrentConfig.FFmpegPath.Contains(@":\\"))
|
|
{
|
|
configService.CurrentConfig.FFmpegPath =
|
|
configService.CurrentConfig.FFmpegPath.Replace(@"\", @"\\");
|
|
}
|
|
|
|
// Get FFmpeg version
|
|
try
|
|
{
|
|
ProcessStartInfo processStartInfo = new()
|
|
{
|
|
FileName = configService.CurrentConfig.FFmpegPath,
|
|
Arguments = "-version",
|
|
RedirectStandardOutput = true,
|
|
RedirectStandardError = true,
|
|
UseShellExecute = false,
|
|
CreateNoWindow = true
|
|
};
|
|
|
|
using (Process? process = Process.Start(processStartInfo))
|
|
{
|
|
if (process != null)
|
|
{
|
|
string output = await process.StandardOutput.ReadToEndAsync();
|
|
await process.WaitForExitAsync();
|
|
|
|
// Log full output
|
|
Log.Information("FFmpeg version output:\n{Output}", output);
|
|
|
|
// Parse first line for console output
|
|
string firstLine = output.Split('\n')[0].Trim();
|
|
if (firstLine.StartsWith("ffmpeg version"))
|
|
{
|
|
// Extract version string (text between "ffmpeg version " and " Copyright")
|
|
int versionStart = "ffmpeg version ".Length;
|
|
int copyrightIndex = firstLine.IndexOf(" Copyright");
|
|
if (copyrightIndex > versionStart)
|
|
{
|
|
string version = firstLine.Substring(versionStart, copyrightIndex - versionStart);
|
|
AnsiConsole.Markup($"[green]ffmpeg version detected as {version}[/]\n");
|
|
}
|
|
else
|
|
{
|
|
// Fallback if Copyright not found
|
|
string version = firstLine.Substring(versionStart);
|
|
AnsiConsole.Markup($"[green]ffmpeg version detected as {version}[/]\n");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup("[yellow]ffmpeg version could not be parsed[/]\n");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warning(ex, "Failed to get FFmpeg version");
|
|
AnsiConsole.Markup("[yellow]Could not retrieve ffmpeg version[/]\n");
|
|
}
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup(
|
|
"[red]Cannot locate FFmpeg; please modify config.conf with the correct path. Press any key to exit.[/]");
|
|
Log.Error($"Cannot locate FFmpeg with path: {configService.CurrentConfig.FFmpegPath}");
|
|
if (!configService.CurrentConfig.NonInteractiveMode)
|
|
{
|
|
Console.ReadKey();
|
|
}
|
|
|
|
Environment.Exit(4);
|
|
}
|
|
|
|
if (!File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER, WidevineConstants.DEVICE_NAME,
|
|
"device_client_id_blob")))
|
|
{
|
|
clientIdBlobMissing = true;
|
|
Log.Debug("clientIdBlobMissing missing");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup("[green]device_client_id_blob located successfully![/]\n");
|
|
Log.Debug("clientIdBlobMissing found: " + File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER,
|
|
WidevineConstants.DEVICE_NAME, "device_client_id_blob")));
|
|
}
|
|
|
|
if (!File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER, WidevineConstants.DEVICE_NAME,
|
|
"device_private_key")))
|
|
{
|
|
devicePrivateKeyMissing = true;
|
|
Log.Debug("devicePrivateKeyMissing missing");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup("[green]device_private_key located successfully![/]\n");
|
|
Log.Debug("devicePrivateKeyMissing found: " + File.Exists(Path.Join(WidevineConstants.DEVICES_FOLDER,
|
|
WidevineConstants.DEVICE_NAME, "device_private_key")));
|
|
}
|
|
|
|
if (clientIdBlobMissing || devicePrivateKeyMissing)
|
|
{
|
|
AnsiConsole.Markup(
|
|
"[yellow]device_client_id_blob and/or device_private_key missing, https://ofdl.tools/ or https://cdrm-project.com/ will be used instead for DRM protected videos\n[/]");
|
|
}
|
|
|
|
UserEntities.User? validate = await apiService.GetUserInfo("/users/me");
|
|
if (validate == null || (validate?.Name == null && validate?.Username == null))
|
|
{
|
|
Log.Error("Auth failed");
|
|
|
|
authService.CurrentAuth = null;
|
|
if (!configService.CurrentConfig!.DisableBrowserAuth)
|
|
{
|
|
if (File.Exists("auth.json"))
|
|
{
|
|
File.Delete("auth.json");
|
|
}
|
|
}
|
|
|
|
if (!configService.CurrentConfig.NonInteractiveMode && !configService.CurrentConfig!.DisableBrowserAuth)
|
|
{
|
|
await LoadAuthFromBrowser();
|
|
}
|
|
|
|
if (authService.CurrentAuth == null)
|
|
{
|
|
AnsiConsole.MarkupLine(
|
|
"\n[red]Auth failed. Please try again or use other authentication methods detailed here:[/]\n");
|
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth[/]\n");
|
|
Console.ReadKey();
|
|
Environment.Exit(2);
|
|
}
|
|
}
|
|
|
|
AnsiConsole.Markup($"[green]Logged In successfully as {validate.Name} {validate.Username}\n[/]");
|
|
await DownloadAllData();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
|
|
Console.WriteLine("\nPress any key to exit.");
|
|
if (!configService.CurrentConfig.NonInteractiveMode)
|
|
{
|
|
Console.ReadKey();
|
|
}
|
|
|
|
Environment.Exit(5);
|
|
}
|
|
}
|
|
|
|
private async Task DownloadAllData()
|
|
{
|
|
IDBService dbService = serviceProvider.GetRequiredService<IDBService>();
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAuthService authService = serviceProvider.GetRequiredService<IAuthService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
Config Config = configService.CurrentConfig!;
|
|
|
|
Log.Debug("Calling DownloadAllData");
|
|
|
|
do
|
|
{
|
|
DateTime startTime = DateTime.Now;
|
|
Dictionary<string, long> users = new();
|
|
Dictionary<string, long> activeSubs =
|
|
await apiService.GetActiveSubscriptions("/subscriptions/subscribes",
|
|
Config.IncludeRestrictedSubscriptions);
|
|
|
|
Log.Debug("Subscriptions: ");
|
|
|
|
foreach (KeyValuePair<string, long> activeSub in activeSubs)
|
|
{
|
|
if (!users.ContainsKey(activeSub.Key))
|
|
{
|
|
users.Add(activeSub.Key, activeSub.Value);
|
|
Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}");
|
|
}
|
|
}
|
|
|
|
if (Config!.IncludeExpiredSubscriptions)
|
|
{
|
|
Log.Debug("Inactive Subscriptions: ");
|
|
|
|
Dictionary<string, long> expiredSubs =
|
|
await apiService.GetExpiredSubscriptions("/subscriptions/subscribes",
|
|
Config.IncludeRestrictedSubscriptions);
|
|
foreach (KeyValuePair<string, long> expiredSub in expiredSubs)
|
|
{
|
|
if (!users.ContainsKey(expiredSub.Key))
|
|
{
|
|
users.Add(expiredSub.Key, expiredSub.Value);
|
|
Log.Debug($"Name: {expiredSub.Key} ID: {expiredSub.Value}");
|
|
}
|
|
}
|
|
}
|
|
|
|
Dictionary<string, long> lists = await apiService.GetLists("/lists");
|
|
|
|
// Remove users from the list if they are in the ignored list
|
|
if (!string.IsNullOrEmpty(Config.IgnoredUsersListName))
|
|
{
|
|
if (!lists.TryGetValue(Config.IgnoredUsersListName, out long ignoredUsersListId))
|
|
{
|
|
AnsiConsole.Markup($"[red]Ignored users list '{Config.IgnoredUsersListName}' not found\n[/]");
|
|
Log.Error($"Ignored users list '{Config.IgnoredUsersListName}' not found");
|
|
}
|
|
else
|
|
{
|
|
List<string> ignoredUsernames =
|
|
await apiService.GetListUsers($"/lists/{ignoredUsersListId}/users") ?? [];
|
|
users = users.Where(x => !ignoredUsernames.Contains(x.Key)).ToDictionary(x => x.Key, x => x.Value);
|
|
}
|
|
}
|
|
|
|
await dbService.CreateUsersDB(users);
|
|
KeyValuePair<bool, Dictionary<string, long>> hasSelectedUsersKVP;
|
|
if (Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab)
|
|
{
|
|
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, long>>(true,
|
|
new Dictionary<string, long> { { "PurchasedTab", 0 } });
|
|
}
|
|
else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName))
|
|
{
|
|
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, long>>(true, users);
|
|
}
|
|
else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName))
|
|
{
|
|
long listId = lists[Config.NonInteractiveModeListName];
|
|
List<string> listUsernames = await apiService.GetListUsers($"/lists/{listId}/users") ?? [];
|
|
Dictionary<string, long> selectedUsers = users.Where(x => listUsernames.Contains(x.Key)).Distinct()
|
|
.ToDictionary(x => x.Key, x => x.Value);
|
|
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, long>>(true, selectedUsers);
|
|
}
|
|
else
|
|
{
|
|
ILoggingService loggingService = serviceProvider.GetRequiredService<ILoggingService>();
|
|
(bool IsExit, Dictionary<string, long>? selectedUsers) userSelectionResult =
|
|
await HandleUserSelection(users, lists);
|
|
|
|
Config = configService.CurrentConfig!;
|
|
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, long>>(userSelectionResult.IsExit,
|
|
userSelectionResult.selectedUsers);
|
|
}
|
|
|
|
if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null &&
|
|
hasSelectedUsersKVP.Value.ContainsKey("SinglePost"))
|
|
{
|
|
AnsiConsole.Markup(
|
|
"[red]To find an individual post URL, click on the ... at the top right corner of the post and select 'Copy link to post'.\n\nTo return to the main menu, enter 'back' or 'exit' when prompted for the URL.\n\n[/]");
|
|
string postUrl = AnsiConsole.Prompt(
|
|
new TextPrompt<string>("[red]Please enter a post URL: [/]")
|
|
.ValidationErrorMessage("[red]Please enter a valid post URL[/]")
|
|
.Validate(url =>
|
|
{
|
|
Log.Debug($"Single Post URL: {url}");
|
|
Regex regex = new("https://onlyfans\\.com/[0-9]+/[A-Za-z0-9]+", RegexOptions.IgnoreCase);
|
|
if (regex.IsMatch(url))
|
|
{
|
|
return ValidationResult.Success();
|
|
}
|
|
|
|
if (url == "" || url == "exit" || url == "back")
|
|
{
|
|
return ValidationResult.Success();
|
|
}
|
|
|
|
Log.Error("Post URL invalid");
|
|
return ValidationResult.Error("[red]Please enter a valid post URL[/]");
|
|
}));
|
|
|
|
if (postUrl != "" && postUrl != "exit" && postUrl != "back")
|
|
{
|
|
long post_id = Convert.ToInt64(postUrl.Split("/")[3]);
|
|
string username = postUrl.Split("/")[4];
|
|
|
|
Log.Debug($"Single Post ID: {post_id.ToString()}");
|
|
Log.Debug($"Single Post Creator: {username}");
|
|
|
|
if (users.ContainsKey(username))
|
|
{
|
|
string path = "";
|
|
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
|
{
|
|
path = Path.Combine(Config.DownloadPath, username);
|
|
}
|
|
else
|
|
{
|
|
path = $"__user_data__/sites/OnlyFans/{username}";
|
|
}
|
|
|
|
Log.Debug($"Download path: {path}");
|
|
|
|
if (!Directory.Exists(path))
|
|
{
|
|
Directory.CreateDirectory(path);
|
|
AnsiConsole.Markup($"[red]Created folder for {username}\n[/]");
|
|
Log.Debug($"Created folder for {username}");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup($"[red]Folder for {username} already created\n[/]");
|
|
}
|
|
|
|
await dbService.CreateDB(path);
|
|
|
|
await DownloadSinglePost(username, post_id, path, users);
|
|
}
|
|
}
|
|
}
|
|
else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null &&
|
|
hasSelectedUsersKVP.Value.ContainsKey("PurchasedTab"))
|
|
{
|
|
Dictionary<string, long> purchasedTabUsers =
|
|
await apiService.GetPurchasedTabUsers("/posts/paid/all", users);
|
|
AnsiConsole.Markup("[red]Checking folders for Users in Purchased Tab\n[/]");
|
|
foreach (KeyValuePair<string, long> user in purchasedTabUsers)
|
|
{
|
|
string path = "";
|
|
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
|
{
|
|
path = Path.Combine(Config.DownloadPath, user.Key);
|
|
}
|
|
else
|
|
{
|
|
path = $"__user_data__/sites/OnlyFans/{user.Key}";
|
|
}
|
|
|
|
Log.Debug($"Download path: {path}");
|
|
|
|
await dbService.CheckUsername(user, path);
|
|
|
|
if (!Directory.Exists(path))
|
|
{
|
|
Directory.CreateDirectory(path);
|
|
AnsiConsole.Markup($"[red]Created folder for {user.Key}\n[/]");
|
|
Log.Debug($"Created folder for {user.Key}");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup($"[red]Folder for {user.Key} already created\n[/]");
|
|
Log.Debug($"Folder for {user.Key} already created");
|
|
}
|
|
|
|
UserEntities.User user_info = await apiService.GetUserInfo($"/users/{user.Key}");
|
|
|
|
await dbService.CreateDB(path);
|
|
}
|
|
|
|
string p = "";
|
|
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
|
{
|
|
p = Config.DownloadPath;
|
|
}
|
|
else
|
|
{
|
|
p = "__user_data__/sites/OnlyFans/";
|
|
}
|
|
|
|
Log.Debug($"Download path: {p}");
|
|
|
|
List<PurchasedEntities.PurchasedTabCollection> purchasedTabCollections =
|
|
await apiService.GetPurchasedTab("/posts/paid/all", p, users);
|
|
foreach (PurchasedEntities.PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
|
|
{
|
|
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]");
|
|
string path = "";
|
|
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
|
{
|
|
path = Path.Combine(Config.DownloadPath, purchasedTabCollection.Username);
|
|
}
|
|
else
|
|
{
|
|
path = $"__user_data__/sites/OnlyFans/{purchasedTabCollection.Username}";
|
|
}
|
|
|
|
|
|
Log.Debug($"Download path: {path}");
|
|
|
|
int paidPostCount = 0;
|
|
int paidMessagesCount = 0;
|
|
paidPostCount = await DownloadPaidPostsPurchasedTab(purchasedTabCollection.Username,
|
|
purchasedTabCollection.PaidPosts,
|
|
users.FirstOrDefault(u => u.Value == purchasedTabCollection.UserId), paidPostCount, path,
|
|
users);
|
|
paidMessagesCount = await DownloadPaidMessagesPurchasedTab(purchasedTabCollection.Username,
|
|
purchasedTabCollection.PaidMessages,
|
|
users.FirstOrDefault(u => u.Value == purchasedTabCollection.UserId), paidMessagesCount, path,
|
|
users);
|
|
|
|
AnsiConsole.Markup("\n");
|
|
AnsiConsole.Write(new BreakdownChart()
|
|
.FullSize()
|
|
.AddItem("Paid Posts", paidPostCount, Color.Red)
|
|
.AddItem("Paid Messages", paidMessagesCount, Color.Aqua));
|
|
AnsiConsole.Markup("\n");
|
|
}
|
|
|
|
DateTime endTime = DateTime.Now;
|
|
TimeSpan totalTime = endTime - startTime;
|
|
AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]");
|
|
Log.Debug($"Scrape Completed in {totalTime.TotalMinutes:0.00} minutes");
|
|
}
|
|
else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null &&
|
|
hasSelectedUsersKVP.Value.ContainsKey("SingleMessage"))
|
|
{
|
|
AnsiConsole.Markup(
|
|
"[red]To find an individual message URL, note that you can only do so for PPV messages that you have unlocked. Go the main OnlyFans timeline, click on the Purchased tab, find the relevant message, click on the ... at the top right corner of the message, and select 'Copy link to message'. For all other messages, you cannot scrape them individually, you must scrape all messages from that creator.\n\nTo return to the main menu, enter 'back' or 'exit' when prompted for the URL.\n\n[/]");
|
|
string messageUrl = AnsiConsole.Prompt(
|
|
new TextPrompt<string>("[red]Please enter a message URL: [/]")
|
|
.ValidationErrorMessage("[red]Please enter a valid message URL[/]")
|
|
.Validate(url =>
|
|
{
|
|
Log.Debug($"Single Paid Message URL: {url}");
|
|
Regex regex = new("https://onlyfans\\.com/my/chats/chat/[0-9]+/\\?firstId=[0-9]+$",
|
|
RegexOptions.IgnoreCase);
|
|
if (regex.IsMatch(url))
|
|
{
|
|
return ValidationResult.Success();
|
|
}
|
|
|
|
if (url == "" || url == "back" || url == "exit")
|
|
{
|
|
return ValidationResult.Success();
|
|
}
|
|
|
|
Log.Error("Message URL invalid");
|
|
return ValidationResult.Error("[red]Please enter a valid message URL[/]");
|
|
}));
|
|
|
|
if (messageUrl != "" && messageUrl != "exit" && messageUrl != "back")
|
|
{
|
|
long message_id = Convert.ToInt64(messageUrl.Split("?firstId=")[1]);
|
|
long user_id = Convert.ToInt64(messageUrl.Split("/")[6]);
|
|
JObject user = await apiService.GetUserInfoById($"/users/list?x[]={user_id.ToString()}");
|
|
string username = "";
|
|
|
|
Log.Debug($"Message ID: {message_id}");
|
|
Log.Debug($"User ID: {user_id}");
|
|
|
|
if (user is null)
|
|
{
|
|
username = $"Deleted User - {user_id.ToString()}";
|
|
Log.Debug("Content creator not longer exists - ", user_id.ToString());
|
|
}
|
|
else if (!string.IsNullOrEmpty(user[user_id.ToString()]["username"].ToString()))
|
|
{
|
|
username = user[user_id.ToString()]["username"].ToString();
|
|
Log.Debug("Content creator: ", username);
|
|
}
|
|
|
|
string path = "";
|
|
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
|
{
|
|
path = Path.Combine(Config.DownloadPath, username);
|
|
}
|
|
else
|
|
{
|
|
path = $"__user_data__/sites/OnlyFans/{username}";
|
|
}
|
|
|
|
Log.Debug("Download path: ", path);
|
|
|
|
if (!Directory.Exists(path))
|
|
{
|
|
Directory.CreateDirectory(path);
|
|
AnsiConsole.Markup($"[red]Created folder for {username}\n[/]");
|
|
Log.Debug($"Created folder for {username}");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup($"[red]Folder for {username} already created\n[/]");
|
|
Log.Debug($"Folder for {username} already created");
|
|
}
|
|
|
|
await dbService.CreateDB(path);
|
|
|
|
await DownloadPaidMessage(username, hasSelectedUsersKVP, 1, path, message_id);
|
|
}
|
|
}
|
|
else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
|
{
|
|
//Iterate over each user in the list of users
|
|
foreach (KeyValuePair<string, long> user in hasSelectedUsersKVP.Value)
|
|
{
|
|
int paidPostCount = 0;
|
|
int postCount = 0;
|
|
int archivedCount = 0;
|
|
int streamsCount = 0;
|
|
int storiesCount = 0;
|
|
int highlightsCount = 0;
|
|
int messagesCount = 0;
|
|
int paidMessagesCount = 0;
|
|
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]");
|
|
|
|
Log.Debug($"Scraping Data for {user.Key}");
|
|
|
|
string path = "";
|
|
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
|
{
|
|
path = Path.Combine(Config.DownloadPath, user.Key);
|
|
}
|
|
else
|
|
{
|
|
path = $"__user_data__/sites/OnlyFans/{user.Key}";
|
|
}
|
|
|
|
Log.Debug("Download path: ", path);
|
|
|
|
await dbService.CheckUsername(user, path);
|
|
|
|
if (!Directory.Exists(path))
|
|
{
|
|
Directory.CreateDirectory(path);
|
|
AnsiConsole.Markup($"[red]Created folder for {user.Key}\n[/]");
|
|
Log.Debug($"Created folder for {user.Key}");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup($"[red]Folder for {user.Key} already created\n[/]");
|
|
Log.Debug($"Folder for {user.Key} already created");
|
|
}
|
|
|
|
await dbService.CreateDB(path);
|
|
|
|
if (Config.DownloadAvatarHeaderPhoto)
|
|
{
|
|
UserEntities.User? user_info = await apiService.GetUserInfo($"/users/{user.Key}");
|
|
if (user_info != null)
|
|
{
|
|
await downloadService.DownloadAvatarHeader(user_info.Avatar, user_info.Header, path,
|
|
user.Key);
|
|
}
|
|
}
|
|
|
|
if (Config.DownloadPaidPosts)
|
|
{
|
|
paidPostCount =
|
|
await DownloadPaidPosts(user.Key, hasSelectedUsersKVP, user, paidPostCount, path);
|
|
}
|
|
|
|
if (Config.DownloadPosts)
|
|
{
|
|
postCount = await DownloadFreePosts(user.Key, hasSelectedUsersKVP, user, postCount, path);
|
|
}
|
|
|
|
if (Config.DownloadArchived)
|
|
{
|
|
archivedCount =
|
|
await DownloadArchived(user.Key, hasSelectedUsersKVP, user, archivedCount, path);
|
|
}
|
|
|
|
if (Config.DownloadStreams)
|
|
{
|
|
streamsCount = await DownloadStreams(user.Key, hasSelectedUsersKVP, user, streamsCount, path);
|
|
}
|
|
|
|
if (Config.DownloadStories)
|
|
{
|
|
storiesCount = await DownloadStories(user.Key, user, storiesCount, path);
|
|
}
|
|
|
|
if (Config.DownloadHighlights)
|
|
{
|
|
highlightsCount = await DownloadHighlights(user.Key, user, highlightsCount, path);
|
|
}
|
|
|
|
if (Config.DownloadMessages)
|
|
{
|
|
messagesCount =
|
|
await DownloadMessages(user.Key, hasSelectedUsersKVP, user, messagesCount, path);
|
|
}
|
|
|
|
if (Config.DownloadPaidMessages)
|
|
{
|
|
paidMessagesCount = await DownloadPaidMessages(user.Key, hasSelectedUsersKVP, user,
|
|
paidMessagesCount, path);
|
|
}
|
|
|
|
AnsiConsole.Markup("\n");
|
|
AnsiConsole.Write(new BreakdownChart()
|
|
.FullSize()
|
|
.AddItem("Paid Posts", paidPostCount, Color.Red)
|
|
.AddItem("Posts", postCount, Color.Blue)
|
|
.AddItem("Archived", archivedCount, Color.Green)
|
|
.AddItem("Streams", streamsCount, Color.Purple)
|
|
.AddItem("Stories", storiesCount, Color.Yellow)
|
|
.AddItem("Highlights", highlightsCount, Color.Orange1)
|
|
.AddItem("Messages", messagesCount, Color.LightGreen)
|
|
.AddItem("Paid Messages", paidMessagesCount, Color.Aqua));
|
|
AnsiConsole.Markup("\n");
|
|
}
|
|
|
|
DateTime endTime = DateTime.Now;
|
|
TimeSpan totalTime = endTime - startTime;
|
|
AnsiConsole.Markup($"[green]Scrape Completed in {totalTime.TotalMinutes:0.00} minutes\n[/]");
|
|
}
|
|
else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null &&
|
|
hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
|
{
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
} while (!Config.NonInteractiveMode);
|
|
}
|
|
|
|
private async Task<int> DownloadPaidMessages(string username,
|
|
KeyValuePair<bool, Dictionary<string, long>> hasSelectedUsersKVP, KeyValuePair<string, long> user,
|
|
int paidMessagesCount, string path)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
PurchasedEntities.PaidMessageCollection paidMessageCollection = new();
|
|
|
|
await AnsiConsole.Status()
|
|
.StartAsync("[red]Getting Paid Messages[/]",
|
|
async ctx =>
|
|
{
|
|
paidMessageCollection = await apiService.GetPaidMessages("/posts/paid/chat", path, user.Key, ctx);
|
|
});
|
|
|
|
if (paidMessageCollection != null && paidMessageCollection.PaidMessages.Count > 0)
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[red]Found {paidMessageCollection.PaidMessages.Count} Media from {paidMessageCollection.PaidMessageObjects.Count} Paid Messages\n[/]");
|
|
|
|
long totalSize = 0;
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(paidMessageCollection.PaidMessages.Values
|
|
.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = paidMessageCollection.PaidMessages.Count;
|
|
}
|
|
|
|
DownloadResult result = null;
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
ProgressTask task =
|
|
ctx.AddTask($"[red]Downloading {paidMessageCollection.PaidMessages.Count} Paid Messages[/]",
|
|
false);
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
|
|
SpectreProgressReporter progressReporter = new(task);
|
|
result = await downloadService.DownloadPaidMessages(username, path, hasSelectedUsersKVP.Value,
|
|
clientIdBlobMissing, devicePrivateKeyMissing, paidMessageCollection, progressReporter);
|
|
|
|
task.StopTask();
|
|
});
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Paid Messages Already Downloaded: {result.ExistingDownloads} New Paid Messages Downloaded: {result.NewDownloads}[/]\n");
|
|
return result.TotalCount;
|
|
}
|
|
|
|
AnsiConsole.Markup("[red]Found 0 Paid Messages\n[/]");
|
|
return 0;
|
|
}
|
|
|
|
private async Task<int> DownloadMessages(string username,
|
|
KeyValuePair<bool, Dictionary<string, long>> hasSelectedUsersKVP, KeyValuePair<string, long> user,
|
|
int messagesCount, string path)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
MessageEntities.MessageCollection messages = new();
|
|
|
|
await AnsiConsole.Status()
|
|
.StartAsync("[red]Getting Messages[/]",
|
|
async ctx => { messages = await apiService.GetMessages($"/chats/{user.Value}/messages", path, ctx); });
|
|
|
|
if (messages != null && messages.Messages.Count > 0)
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[red]Found {messages.Messages.Count} Media from {messages.MessageObjects.Count} Messages\n[/]");
|
|
|
|
long totalSize = 0;
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(messages.Messages.Values.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = messages.Messages.Count;
|
|
}
|
|
|
|
DownloadResult result = null;
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
ProgressTask task = ctx.AddTask($"[red]Downloading {messages.Messages.Count} Messages[/]", false);
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
|
|
SpectreProgressReporter progressReporter = new(task);
|
|
result = await downloadService.DownloadMessages(username, user.Value, path,
|
|
hasSelectedUsersKVP.Value, clientIdBlobMissing, devicePrivateKeyMissing, messages,
|
|
progressReporter);
|
|
|
|
task.StopTask();
|
|
});
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Messages Already Downloaded: {result.ExistingDownloads} New Messages Downloaded: {result.NewDownloads}[/]\n");
|
|
return result.TotalCount;
|
|
}
|
|
|
|
AnsiConsole.Markup("[red]Found 0 Messages\n[/]");
|
|
return 0;
|
|
}
|
|
|
|
private async Task<int> DownloadHighlights(string username, KeyValuePair<string, long> user, int highlightsCount,
|
|
string path)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
|
|
AnsiConsole.Markup("[red]Getting Highlights\n[/]");
|
|
|
|
// Calculate total size for progress bar
|
|
long totalSize = 0;
|
|
Dictionary<long, string>? tempHighlights = await apiService.GetMedia(MediaType.Highlights,
|
|
$"/users/{user.Value}/stories/highlights", null, path, paid_post_ids);
|
|
if (tempHighlights != null && tempHighlights.Count > 0)
|
|
{
|
|
AnsiConsole.Markup($"[red]Found {tempHighlights.Count} Highlights\n[/]");
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(tempHighlights.Values.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = tempHighlights.Count;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup("[red]Found 0 Highlights\n[/]");
|
|
return 0;
|
|
}
|
|
|
|
DownloadResult result = null;
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
ProgressTask task = ctx.AddTask($"[red]Downloading {tempHighlights.Count} Highlights[/]", false);
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
|
|
SpectreProgressReporter progressReporter = new(task);
|
|
result = await downloadService.DownloadHighlights(username, user.Value, path, paid_post_ids.ToHashSet(),
|
|
progressReporter);
|
|
|
|
task.StopTask();
|
|
});
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Highlights Already Downloaded: {result.ExistingDownloads} New Highlights Downloaded: {result.NewDownloads}[/]\n");
|
|
return result.TotalCount;
|
|
}
|
|
|
|
private async Task<int> DownloadStories(string username, KeyValuePair<string, long> user, int storiesCount,
|
|
string path)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
AnsiConsole.Markup("[red]Getting Stories\n[/]");
|
|
|
|
// Calculate total size for progress bar
|
|
long totalSize = 0;
|
|
Dictionary<long, string>? tempStories = await serviceProvider.GetRequiredService<IAPIService>()
|
|
.GetMedia(MediaType.Stories, $"/users/{user.Value}/stories", null, path, paid_post_ids);
|
|
if (tempStories != null && tempStories.Count > 0)
|
|
{
|
|
AnsiConsole.Markup($"[red]Found {tempStories.Count} Stories\n[/]");
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(tempStories.Values.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = tempStories.Count;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup("[red]Found 0 Stories\n[/]");
|
|
return 0;
|
|
}
|
|
|
|
DownloadResult result = null;
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
ProgressTask task = ctx.AddTask($"[red]Downloading {tempStories.Count} Stories[/]", false);
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
|
|
SpectreProgressReporter progressReporter = new(task);
|
|
result = await downloadService.DownloadStories(username, user.Value, path, paid_post_ids.ToHashSet(),
|
|
progressReporter);
|
|
|
|
task.StopTask();
|
|
});
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Stories Already Downloaded: {result.ExistingDownloads} New Stories Downloaded: {result.NewDownloads}[/]\n");
|
|
return result.TotalCount;
|
|
}
|
|
|
|
private async Task<int> DownloadArchived(string username,
|
|
KeyValuePair<bool, Dictionary<string, long>> hasSelectedUsersKVP, KeyValuePair<string, long> user,
|
|
int archivedCount, string path)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
ArchivedEntities.ArchivedCollection archived = new();
|
|
|
|
await AnsiConsole.Status()
|
|
.StartAsync("[red]Getting Archived Posts[/]",
|
|
async ctx => { archived = await apiService.GetArchived($"/users/{user.Value}/posts", path, ctx); });
|
|
|
|
if (archived != null && archived.ArchivedPosts.Count > 0)
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[red]Found {archived.ArchivedPosts.Count} Media from {archived.ArchivedPostObjects.Count} Archived Posts\n[/]");
|
|
|
|
long totalSize = 0;
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(archived.ArchivedPosts.Values.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = archived.ArchivedPosts.Count;
|
|
}
|
|
|
|
DownloadResult result = null;
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
ProgressTask task =
|
|
ctx.AddTask($"[red]Downloading {archived.ArchivedPosts.Count} Archived Posts[/]", false);
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
|
|
SpectreProgressReporter progressReporter = new(task);
|
|
result = await downloadService.DownloadArchived(username, user.Value, path,
|
|
hasSelectedUsersKVP.Value, clientIdBlobMissing, devicePrivateKeyMissing, archived,
|
|
progressReporter);
|
|
|
|
task.StopTask();
|
|
});
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Archived Posts Already Downloaded: {result.ExistingDownloads} New Archived Posts Downloaded: {result.NewDownloads}[/]\n");
|
|
return result.TotalCount;
|
|
}
|
|
|
|
AnsiConsole.Markup("[red]Found 0 Archived Posts\n[/]");
|
|
return 0;
|
|
}
|
|
|
|
private async Task<int> DownloadFreePosts(string username,
|
|
KeyValuePair<bool, Dictionary<string, long>> hasSelectedUsersKVP, KeyValuePair<string, long> user,
|
|
int postCount, string path)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
AnsiConsole.Markup(
|
|
"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/]");
|
|
Log.Debug($"Calling DownloadFreePosts - {user.Key}");
|
|
|
|
PostEntities.PostCollection posts = new();
|
|
|
|
await AnsiConsole.Status()
|
|
.StartAsync("[red]Getting Posts[/]",
|
|
async ctx =>
|
|
{
|
|
posts = await apiService.GetPosts($"/users/{user.Value}/posts", path, paid_post_ids, ctx);
|
|
});
|
|
|
|
if (posts == null || posts.Posts.Count <= 0)
|
|
{
|
|
AnsiConsole.Markup("[red]Found 0 Posts\n[/]");
|
|
Log.Debug("Found 0 Posts");
|
|
return 0;
|
|
}
|
|
|
|
AnsiConsole.Markup($"[red]Found {posts.Posts.Count} Media from {posts.PostObjects.Count} Posts\n[/]");
|
|
Log.Debug($"Found {posts.Posts.Count} Media from {posts.PostObjects.Count} Posts");
|
|
|
|
long totalSize = configService.CurrentConfig.ShowScrapeSize
|
|
? await downloadService.CalculateTotalFileSize(posts.Posts.Values.ToList())
|
|
: posts.Posts.Count;
|
|
|
|
DownloadResult result = null;
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
ProgressTask task = ctx.AddTask($"[red]Downloading {posts.Posts.Count} Posts[/]", false);
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
|
|
SpectreProgressReporter progressReporter = new(task);
|
|
result = await downloadService.DownloadFreePosts(username, user.Value, path, hasSelectedUsersKVP.Value,
|
|
clientIdBlobMissing, devicePrivateKeyMissing, posts, progressReporter);
|
|
|
|
task.StopTask();
|
|
});
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Posts Already Downloaded: {result.ExistingDownloads} New Posts Downloaded: {result.NewDownloads}[/]\n");
|
|
Log.Debug($"Posts Already Downloaded: {result.ExistingDownloads} New Posts Downloaded: {result.NewDownloads}");
|
|
|
|
return result.TotalCount;
|
|
}
|
|
|
|
private async Task<int> DownloadPaidPosts(string username,
|
|
KeyValuePair<bool, Dictionary<string, long>> hasSelectedUsersKVP, KeyValuePair<string, long> user,
|
|
int paidPostCount, string path)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
AnsiConsole.Markup("[red]Getting Paid Posts\n[/]");
|
|
Log.Debug($"Calling DownloadPaidPosts - {user.Key}");
|
|
|
|
PurchasedEntities.PaidPostCollection purchasedPosts = new();
|
|
|
|
await AnsiConsole.Status()
|
|
.StartAsync("[red]Getting Paid Posts[/]",
|
|
async ctx =>
|
|
{
|
|
purchasedPosts =
|
|
await apiService.GetPaidPosts("/posts/paid/post", path, user.Key, paid_post_ids, ctx);
|
|
});
|
|
|
|
if (purchasedPosts == null || purchasedPosts.PaidPosts.Count <= 0)
|
|
{
|
|
AnsiConsole.Markup("[red]Found 0 Paid Posts\n[/]");
|
|
Log.Debug("Found 0 Paid Posts");
|
|
return 0;
|
|
}
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts\n[/]");
|
|
Log.Debug(
|
|
$"Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts");
|
|
|
|
long totalSize = configService.CurrentConfig.ShowScrapeSize
|
|
? await downloadService.CalculateTotalFileSize(purchasedPosts.PaidPosts.Values.ToList())
|
|
: purchasedPosts.PaidPosts.Count;
|
|
|
|
DownloadResult result = null;
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
ProgressTask task = ctx.AddTask($"[red]Downloading {purchasedPosts.PaidPosts.Count} Paid Posts[/]",
|
|
false);
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
|
|
SpectreProgressReporter progressReporter = new(task);
|
|
result = await downloadService.DownloadPaidPosts(username, user.Value, path, hasSelectedUsersKVP.Value,
|
|
clientIdBlobMissing, devicePrivateKeyMissing, purchasedPosts, progressReporter);
|
|
|
|
task.StopTask();
|
|
});
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Paid Posts Already Downloaded: {result.ExistingDownloads} New Paid Posts Downloaded: {result.NewDownloads}[/]\n");
|
|
Log.Debug(
|
|
$"Paid Posts Already Downloaded: {result.ExistingDownloads} New Paid Posts Downloaded: {result.NewDownloads}");
|
|
|
|
return result.TotalCount;
|
|
}
|
|
|
|
private async Task<int> DownloadPaidPostsPurchasedTab(string username,
|
|
PurchasedEntities.PaidPostCollection purchasedPosts,
|
|
KeyValuePair<string, long> user, int paidPostCount, string path, Dictionary<string, long> users)
|
|
{
|
|
IDBService dbService = serviceProvider.GetRequiredService<IDBService>();
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAuthService authService = serviceProvider.GetRequiredService<IAuthService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
int oldPaidPostCount = 0;
|
|
int newPaidPostCount = 0;
|
|
if (purchasedPosts == null || purchasedPosts.PaidPosts.Count <= 0)
|
|
{
|
|
AnsiConsole.Markup("[red]Found 0 Paid Posts\n[/]");
|
|
Log.Debug("Found 0 Paid Posts");
|
|
return 0;
|
|
}
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts\n[/]");
|
|
Log.Debug(
|
|
$"Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts");
|
|
|
|
paidPostCount = purchasedPosts.PaidPosts.Count;
|
|
long totalSize = 0;
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(purchasedPosts.PaidPosts.Values.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = paidPostCount;
|
|
}
|
|
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
// Define tasks
|
|
ProgressTask task = ctx.AddTask($"[red]Downloading {purchasedPosts.PaidPosts.Count} Paid Posts[/]",
|
|
false);
|
|
Log.Debug($"Downloading {purchasedPosts.PaidPosts.Count} Paid Posts");
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
foreach (KeyValuePair<long, string> purchasedPostKVP in purchasedPosts.PaidPosts)
|
|
{
|
|
bool isNew;
|
|
if (purchasedPostKVP.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
|
{
|
|
string[] messageUrlParsed = purchasedPostKVP.Value.Split(',');
|
|
string mpdURL = messageUrlParsed[0];
|
|
string policy = messageUrlParsed[1];
|
|
string signature = messageUrlParsed[2];
|
|
string kvp = messageUrlParsed[3];
|
|
string mediaId = messageUrlParsed[4];
|
|
string postId = messageUrlParsed[5];
|
|
string? licenseURL = null;
|
|
string? pssh = await apiService.GetDRMMPDPSSH(mpdURL, policy, signature, kvp);
|
|
if (pssh == null)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
DateTime lastModified = await apiService.GetDRMMPDLastModified(mpdURL, policy, signature, kvp);
|
|
Dictionary<string, string> drmHeaders =
|
|
apiService.GetDynamicHeaders($"/api2/v2/users/media/{mediaId}/drm/post/{postId}",
|
|
"?type=widevine");
|
|
string decryptionKey;
|
|
if (clientIdBlobMissing || devicePrivateKeyMissing)
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyOFDL(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/post/{postId}?type=widevine",
|
|
pssh);
|
|
}
|
|
else
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyCDM(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/post/{postId}?type=widevine",
|
|
pssh);
|
|
}
|
|
|
|
MessageEntities.Medium? mediaInfo =
|
|
purchasedPosts?.PaidPostMedia?.FirstOrDefault(m => m.Id == purchasedPostKVP.Key);
|
|
PurchasedEntities.ListItem? postInfo = mediaInfo != null
|
|
? purchasedPosts?.PaidPostObjects?.FirstOrDefault(p =>
|
|
p?.Media?.Any(m => m.Id == purchasedPostKVP.Key) == true)
|
|
: null;
|
|
|
|
isNew = await downloadService.DownloadPurchasedPostDRMVideo(
|
|
policy,
|
|
signature,
|
|
kvp,
|
|
mpdURL,
|
|
decryptionKey,
|
|
path,
|
|
lastModified,
|
|
purchasedPostKVP.Key,
|
|
"Posts",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
.PaidPostFileNameFormat ?? "",
|
|
postInfo,
|
|
mediaInfo,
|
|
postInfo?.FromUser,
|
|
users);
|
|
if (isNew)
|
|
{
|
|
newPaidPostCount++;
|
|
}
|
|
else
|
|
{
|
|
oldPaidPostCount++;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
MessageEntities.Medium? mediaInfo =
|
|
purchasedPosts?.PaidPostMedia?.FirstOrDefault(m => m.Id == purchasedPostKVP.Key);
|
|
PurchasedEntities.ListItem? postInfo = mediaInfo != null
|
|
? purchasedPosts?.PaidPostObjects?.FirstOrDefault(p =>
|
|
p?.Media?.Any(m => m.Id == purchasedPostKVP.Key) == true)
|
|
: null;
|
|
|
|
isNew = await downloadService.DownloadPurchasedPostMedia(
|
|
purchasedPostKVP.Value,
|
|
path,
|
|
purchasedPostKVP.Key,
|
|
"Posts",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
.PaidPostFileNameFormat ?? "",
|
|
postInfo,
|
|
mediaInfo,
|
|
postInfo?.FromUser,
|
|
users);
|
|
if (isNew)
|
|
{
|
|
newPaidPostCount++;
|
|
}
|
|
else
|
|
{
|
|
oldPaidPostCount++;
|
|
}
|
|
}
|
|
}
|
|
|
|
task.StopTask();
|
|
});
|
|
AnsiConsole.Markup(
|
|
$"[red]Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}[/]\n");
|
|
Log.Debug($"Paid Posts Already Downloaded: {oldPaidPostCount} New Paid Posts Downloaded: {newPaidPostCount}");
|
|
return paidPostCount;
|
|
}
|
|
|
|
private async Task<int> DownloadPaidMessagesPurchasedTab(string username,
|
|
PurchasedEntities.PaidMessageCollection paidMessageCollection, KeyValuePair<string, long> user,
|
|
int paidMessagesCount,
|
|
string path, Dictionary<string, long> users)
|
|
{
|
|
IDBService dbService = serviceProvider.GetRequiredService<IDBService>();
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAuthService authService = serviceProvider.GetRequiredService<IAuthService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
int oldPaidMessagesCount = 0;
|
|
int newPaidMessagesCount = 0;
|
|
if (paidMessageCollection != null && paidMessageCollection.PaidMessages.Count > 0)
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[red]Found {paidMessageCollection.PaidMessages.Count} Media from {paidMessageCollection.PaidMessageObjects.Count} Paid Messages\n[/]");
|
|
Log.Debug(
|
|
$"Found {paidMessageCollection.PaidMessages.Count} Media from {paidMessageCollection.PaidMessageObjects.Count} Paid Messages");
|
|
paidMessagesCount = paidMessageCollection.PaidMessages.Count;
|
|
long totalSize = 0;
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(paidMessageCollection.PaidMessages.Values
|
|
.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = paidMessagesCount;
|
|
}
|
|
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
// Define tasks
|
|
ProgressTask task =
|
|
ctx.AddTask($"[red]Downloading {paidMessageCollection.PaidMessages.Count} Paid Messages[/]",
|
|
false);
|
|
Log.Debug($"Downloading {paidMessageCollection.PaidMessages.Count} Paid Messages");
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
foreach (KeyValuePair<long, string> paidMessageKVP in paidMessageCollection.PaidMessages)
|
|
{
|
|
bool isNew;
|
|
if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
|
{
|
|
string[] messageUrlParsed = paidMessageKVP.Value.Split(',');
|
|
string mpdURL = messageUrlParsed[0];
|
|
string policy = messageUrlParsed[1];
|
|
string signature = messageUrlParsed[2];
|
|
string kvp = messageUrlParsed[3];
|
|
string mediaId = messageUrlParsed[4];
|
|
string messageId = messageUrlParsed[5];
|
|
string? licenseURL = null;
|
|
string? pssh = await apiService.GetDRMMPDPSSH(mpdURL, policy, signature, kvp);
|
|
if (pssh != null)
|
|
{
|
|
DateTime lastModified =
|
|
await apiService.GetDRMMPDLastModified(mpdURL, policy, signature, kvp);
|
|
Dictionary<string, string> drmHeaders =
|
|
apiService.GetDynamicHeaders(
|
|
$"/api2/v2/users/media/{mediaId}/drm/message/{messageId}", "?type=widevine");
|
|
string decryptionKey;
|
|
if (clientIdBlobMissing || devicePrivateKeyMissing)
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyOFDL(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine",
|
|
pssh);
|
|
}
|
|
else
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyCDM(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine",
|
|
pssh);
|
|
}
|
|
|
|
MessageEntities.Medium? mediaInfo =
|
|
paidMessageCollection.PaidMessageMedia.FirstOrDefault(m =>
|
|
m.Id == paidMessageKVP.Key);
|
|
PurchasedEntities.ListItem? messageInfo =
|
|
paidMessageCollection.PaidMessageObjects.FirstOrDefault(p =>
|
|
p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true);
|
|
|
|
isNew = await downloadService.DownloadPurchasedMessageDRMVideo(
|
|
policy,
|
|
signature,
|
|
kvp,
|
|
mpdURL,
|
|
decryptionKey,
|
|
path,
|
|
lastModified,
|
|
paidMessageKVP.Key,
|
|
"Messages",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
.PaidMessageFileNameFormat ?? "",
|
|
messageInfo,
|
|
mediaInfo,
|
|
messageInfo?.FromUser,
|
|
users);
|
|
|
|
if (isNew)
|
|
{
|
|
newPaidMessagesCount++;
|
|
}
|
|
else
|
|
{
|
|
oldPaidMessagesCount++;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
MessageEntities.Medium? mediaInfo =
|
|
paidMessageCollection.PaidMessageMedia.FirstOrDefault(m => m.Id == paidMessageKVP.Key);
|
|
PurchasedEntities.ListItem messageInfo =
|
|
paidMessageCollection.PaidMessageObjects.FirstOrDefault(p =>
|
|
p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true);
|
|
|
|
isNew = await downloadService.DownloadPurchasedMedia(
|
|
paidMessageKVP.Value,
|
|
path,
|
|
paidMessageKVP.Key,
|
|
"Messages",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
.PaidMessageFileNameFormat ?? "",
|
|
messageInfo,
|
|
mediaInfo,
|
|
messageInfo?.FromUser,
|
|
users);
|
|
if (isNew)
|
|
{
|
|
newPaidMessagesCount++;
|
|
}
|
|
else
|
|
{
|
|
oldPaidMessagesCount++;
|
|
}
|
|
}
|
|
}
|
|
|
|
task.StopTask();
|
|
});
|
|
AnsiConsole.Markup(
|
|
$"[red]Paid Messages Already Downloaded: {oldPaidMessagesCount} New Paid Messages Downloaded: {newPaidMessagesCount}[/]\n");
|
|
Log.Debug(
|
|
$"[red]Paid Messages Already Downloaded: {oldPaidMessagesCount} New Paid Messages Downloaded: {newPaidMessagesCount}");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup("[red]Found 0 Paid Messages\n[/]");
|
|
Log.Debug("Found 0 Paid Messages");
|
|
}
|
|
|
|
return paidMessagesCount;
|
|
}
|
|
|
|
private async Task<int> DownloadStreams(string username,
|
|
KeyValuePair<bool, Dictionary<string, long>> hasSelectedUsersKVP, KeyValuePair<string, long> user,
|
|
int streamsCount, string path)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
StreamEntities.StreamsCollection streams = new();
|
|
|
|
await AnsiConsole.Status()
|
|
.StartAsync("[red]Getting Streams[/]",
|
|
async ctx =>
|
|
{
|
|
streams = await apiService.GetStreams($"/users/{user.Value}/posts/streams", path, paid_post_ids,
|
|
ctx);
|
|
});
|
|
|
|
if (streams != null && streams.Streams.Count > 0)
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[red]Found {streams.Streams.Count} Media from {streams.StreamObjects.Count} Streams\n[/]");
|
|
|
|
long totalSize = 0;
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(streams.Streams.Values.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = streams.Streams.Count;
|
|
}
|
|
|
|
DownloadResult result = null;
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
ProgressTask task = ctx.AddTask($"[red]Downloading {streams.Streams.Count} Streams[/]", false);
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
|
|
SpectreProgressReporter progressReporter = new(task);
|
|
result = await downloadService.DownloadStreams(username, user.Value, path,
|
|
hasSelectedUsersKVP.Value, clientIdBlobMissing, devicePrivateKeyMissing, streams,
|
|
progressReporter);
|
|
|
|
task.StopTask();
|
|
});
|
|
|
|
AnsiConsole.Markup(
|
|
$"[red]Streams Already Downloaded: {result.ExistingDownloads} New Streams Downloaded: {result.NewDownloads}[/]\n");
|
|
return result.TotalCount;
|
|
}
|
|
|
|
AnsiConsole.Markup("[red]Found 0 Streams\n[/]");
|
|
return 0;
|
|
}
|
|
|
|
private async Task<int> DownloadPaidMessage(string username,
|
|
KeyValuePair<bool, Dictionary<string, long>> hasSelectedUsersKVP, int paidMessagesCount, string path,
|
|
long message_id)
|
|
{
|
|
IDBService dbService = serviceProvider.GetRequiredService<IDBService>();
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAuthService authService = serviceProvider.GetRequiredService<IAuthService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
Log.Debug($"Calling DownloadPaidMessage - {username}");
|
|
|
|
AnsiConsole.Markup("[red]Getting Paid Message\n[/]");
|
|
|
|
PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection =
|
|
await apiService.GetPaidMessage($"/messages/{message_id.ToString()}", path);
|
|
int oldPreviewPaidMessagesCount = 0;
|
|
int newPreviewPaidMessagesCount = 0;
|
|
int oldPaidMessagesCount = 0;
|
|
int newPaidMessagesCount = 0;
|
|
if (singlePaidMessageCollection != null && singlePaidMessageCollection.PreviewSingleMessages.Count > 0)
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[red]Found {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages\n[/]");
|
|
Log.Debug(
|
|
$"Found {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages");
|
|
paidMessagesCount = singlePaidMessageCollection.SingleMessages.Count;
|
|
long totalSize = 0;
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(singlePaidMessageCollection
|
|
.PreviewSingleMessages.Values.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = paidMessagesCount;
|
|
}
|
|
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
// Define tasks
|
|
ProgressTask task =
|
|
ctx.AddTask(
|
|
$"[red]Downloading {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Paid Messages[/]",
|
|
false);
|
|
Log.Debug($"Downloading {singlePaidMessageCollection.PreviewSingleMessages.Count} Paid Messages");
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
foreach (KeyValuePair<long, string> paidMessageKVP in singlePaidMessageCollection
|
|
.PreviewSingleMessages)
|
|
{
|
|
bool isNew;
|
|
if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
|
{
|
|
string[] messageUrlParsed = paidMessageKVP.Value.Split(',');
|
|
string mpdURL = messageUrlParsed[0];
|
|
string policy = messageUrlParsed[1];
|
|
string signature = messageUrlParsed[2];
|
|
string kvp = messageUrlParsed[3];
|
|
string mediaId = messageUrlParsed[4];
|
|
string messageId = messageUrlParsed[5];
|
|
string? licenseURL = null;
|
|
string? pssh = await apiService.GetDRMMPDPSSH(mpdURL, policy, signature, kvp);
|
|
if (pssh != null)
|
|
{
|
|
DateTime lastModified =
|
|
await apiService.GetDRMMPDLastModified(mpdURL, policy, signature, kvp);
|
|
Dictionary<string, string> drmHeaders =
|
|
apiService.GetDynamicHeaders(
|
|
$"/api2/v2/users/media/{mediaId}/drm/message/{messageId}", "?type=widevine");
|
|
string decryptionKey;
|
|
if (clientIdBlobMissing || devicePrivateKeyMissing)
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyOFDL(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine",
|
|
pssh);
|
|
}
|
|
else
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyCDM(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine",
|
|
pssh);
|
|
}
|
|
|
|
MessageEntities.Medium? mediaInfo =
|
|
singlePaidMessageCollection.PreviewSingleMessageMedia.FirstOrDefault(m =>
|
|
m.Id == paidMessageKVP.Key);
|
|
MessageEntities.SingleMessage? messageInfo =
|
|
singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p =>
|
|
p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true);
|
|
|
|
isNew = await downloadService.DownloadSingleMessagePreviewDRMVideo(
|
|
policy,
|
|
signature,
|
|
kvp,
|
|
mpdURL,
|
|
decryptionKey,
|
|
path,
|
|
lastModified,
|
|
paidMessageKVP.Key,
|
|
"Messages",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
.PaidMessageFileNameFormat ?? "",
|
|
messageInfo,
|
|
mediaInfo,
|
|
messageInfo?.FromUser,
|
|
hasSelectedUsersKVP.Value);
|
|
|
|
if (isNew)
|
|
{
|
|
newPreviewPaidMessagesCount++;
|
|
}
|
|
else
|
|
{
|
|
oldPreviewPaidMessagesCount++;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
MessageEntities.Medium? mediaInfo =
|
|
singlePaidMessageCollection.PreviewSingleMessageMedia.FirstOrDefault(m =>
|
|
m.Id == paidMessageKVP.Key);
|
|
MessageEntities.SingleMessage? messageInfo =
|
|
singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p =>
|
|
p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true);
|
|
|
|
isNew = await downloadService.DownloadMessagePreviewMedia(
|
|
paidMessageKVP.Value,
|
|
path,
|
|
paidMessageKVP.Key,
|
|
"Messages",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
.PaidMessageFileNameFormat ?? "",
|
|
messageInfo,
|
|
mediaInfo,
|
|
messageInfo?.FromUser,
|
|
hasSelectedUsersKVP.Value);
|
|
if (isNew)
|
|
{
|
|
newPreviewPaidMessagesCount++;
|
|
}
|
|
else
|
|
{
|
|
oldPreviewPaidMessagesCount++;
|
|
}
|
|
}
|
|
}
|
|
|
|
task.StopTask();
|
|
});
|
|
AnsiConsole.Markup(
|
|
$"[red]Preview Paid Messages Already Downloaded: {oldPreviewPaidMessagesCount} New Preview Paid Messages Downloaded: {newPreviewPaidMessagesCount}[/]\n");
|
|
Log.Debug(
|
|
$"Preview Paid Messages Already Downloaded: {oldPreviewPaidMessagesCount} New Preview Paid Messages Downloaded: {newPreviewPaidMessagesCount}");
|
|
}
|
|
|
|
if (singlePaidMessageCollection != null && singlePaidMessageCollection.SingleMessages.Count > 0)
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[red]Found {singlePaidMessageCollection.SingleMessages.Count} Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages\n[/]");
|
|
Log.Debug(
|
|
$"Found {singlePaidMessageCollection.SingleMessages.Count} Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages");
|
|
paidMessagesCount = singlePaidMessageCollection.SingleMessages.Count;
|
|
long totalSize = 0;
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(singlePaidMessageCollection.SingleMessages
|
|
.Values.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = paidMessagesCount;
|
|
}
|
|
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
// Define tasks
|
|
ProgressTask task =
|
|
ctx.AddTask(
|
|
$"[red]Downloading {singlePaidMessageCollection.SingleMessages.Count} Paid Messages[/]",
|
|
false);
|
|
Log.Debug($"Downloading {singlePaidMessageCollection.SingleMessages.Count} Paid Messages");
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
foreach (KeyValuePair<long, string> paidMessageKVP in singlePaidMessageCollection.SingleMessages)
|
|
{
|
|
bool isNew;
|
|
if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
|
{
|
|
string[] messageUrlParsed = paidMessageKVP.Value.Split(',');
|
|
string mpdURL = messageUrlParsed[0];
|
|
string policy = messageUrlParsed[1];
|
|
string signature = messageUrlParsed[2];
|
|
string kvp = messageUrlParsed[3];
|
|
string mediaId = messageUrlParsed[4];
|
|
string messageId = messageUrlParsed[5];
|
|
string? licenseURL = null;
|
|
string? pssh = await apiService.GetDRMMPDPSSH(mpdURL, policy, signature, kvp);
|
|
if (pssh != null)
|
|
{
|
|
DateTime lastModified =
|
|
await apiService.GetDRMMPDLastModified(mpdURL, policy, signature, kvp);
|
|
Dictionary<string, string> drmHeaders =
|
|
apiService.GetDynamicHeaders(
|
|
$"/api2/v2/users/media/{mediaId}/drm/message/{messageId}", "?type=widevine");
|
|
string decryptionKey;
|
|
if (clientIdBlobMissing || devicePrivateKeyMissing)
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyOFDL(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine",
|
|
pssh);
|
|
}
|
|
else
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyCDM(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine",
|
|
pssh);
|
|
}
|
|
|
|
MessageEntities.Medium? mediaInfo =
|
|
singlePaidMessageCollection.SingleMessageMedia.FirstOrDefault(m =>
|
|
m.Id == paidMessageKVP.Key);
|
|
MessageEntities.SingleMessage? messageInfo =
|
|
singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p =>
|
|
p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true);
|
|
|
|
isNew = await downloadService.DownloadSinglePurchasedMessageDRMVideo(
|
|
policy,
|
|
signature,
|
|
kvp,
|
|
mpdURL,
|
|
decryptionKey,
|
|
path,
|
|
lastModified,
|
|
paidMessageKVP.Key,
|
|
"Messages",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
.PaidMessageFileNameFormat ?? "",
|
|
messageInfo,
|
|
mediaInfo,
|
|
messageInfo?.FromUser,
|
|
hasSelectedUsersKVP.Value);
|
|
|
|
if (isNew)
|
|
{
|
|
newPaidMessagesCount++;
|
|
}
|
|
else
|
|
{
|
|
oldPaidMessagesCount++;
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
MessageEntities.Medium? mediaInfo =
|
|
singlePaidMessageCollection.SingleMessageMedia.FirstOrDefault(m =>
|
|
m.Id == paidMessageKVP.Key);
|
|
MessageEntities.SingleMessage? messageInfo =
|
|
singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p =>
|
|
p?.Media?.Any(m => m.Id == paidMessageKVP.Key) == true);
|
|
|
|
isNew = await downloadService.DownloadSinglePurchasedMedia(
|
|
paidMessageKVP.Value,
|
|
path,
|
|
paidMessageKVP.Key,
|
|
"Messages",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
.PaidMessageFileNameFormat ?? "",
|
|
messageInfo,
|
|
mediaInfo,
|
|
messageInfo?.FromUser,
|
|
hasSelectedUsersKVP.Value);
|
|
if (isNew)
|
|
{
|
|
newPaidMessagesCount++;
|
|
}
|
|
else
|
|
{
|
|
oldPaidMessagesCount++;
|
|
}
|
|
}
|
|
}
|
|
|
|
task.StopTask();
|
|
});
|
|
AnsiConsole.Markup(
|
|
$"[red]Paid Messages Already Downloaded: {oldPaidMessagesCount} New Paid Messages Downloaded: {newPaidMessagesCount}[/]\n");
|
|
Log.Debug(
|
|
$"Paid Messages Already Downloaded: {oldPaidMessagesCount} New Paid Messages Downloaded: {newPaidMessagesCount}");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup("[red]Found 0 Paid Messages\n[/]");
|
|
Log.Debug("Found 0 Paid Messages");
|
|
}
|
|
|
|
return paidMessagesCount;
|
|
}
|
|
|
|
private async Task DownloadSinglePost(string username, long post_id, string path, Dictionary<string, long> users)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
IDownloadService downloadService = serviceProvider.GetRequiredService<IDownloadService>();
|
|
|
|
Log.Debug($"Calling DownloadSinglePost - {post_id.ToString()}");
|
|
|
|
AnsiConsole.Markup("[red]Getting Post\n[/]");
|
|
PostEntities.SinglePostCollection post = await apiService.GetPost($"/posts/{post_id.ToString()}", path);
|
|
if (post == null)
|
|
{
|
|
AnsiConsole.Markup("[red]Couldn't find post\n[/]");
|
|
Log.Debug("Couldn't find post");
|
|
return;
|
|
}
|
|
|
|
long totalSize = 0;
|
|
if (configService.CurrentConfig.ShowScrapeSize)
|
|
{
|
|
totalSize = await downloadService.CalculateTotalFileSize(post.SinglePosts.Values.ToList());
|
|
}
|
|
else
|
|
{
|
|
totalSize = post.SinglePosts.Count;
|
|
}
|
|
|
|
bool isNew = false;
|
|
await AnsiConsole.Progress()
|
|
.Columns(GetProgressColumns(configService.CurrentConfig.ShowScrapeSize))
|
|
.StartAsync(async ctx =>
|
|
{
|
|
ProgressTask task = ctx.AddTask("[red]Downloading Post[/]", false);
|
|
task.MaxValue = totalSize;
|
|
task.StartTask();
|
|
foreach (KeyValuePair<long, string> postKVP in post.SinglePosts)
|
|
{
|
|
if (postKVP.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
|
{
|
|
string[] messageUrlParsed = postKVP.Value.Split(',');
|
|
string mpdURL = messageUrlParsed[0];
|
|
string policy = messageUrlParsed[1];
|
|
string signature = messageUrlParsed[2];
|
|
string kvp = messageUrlParsed[3];
|
|
string mediaId = messageUrlParsed[4];
|
|
string postId = messageUrlParsed[5];
|
|
string? licenseURL = null;
|
|
string? pssh = await apiService.GetDRMMPDPSSH(mpdURL, policy, signature, kvp);
|
|
if (pssh == null)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
DateTime lastModified = await apiService.GetDRMMPDLastModified(mpdURL, policy, signature, kvp);
|
|
Dictionary<string, string> drmHeaders =
|
|
apiService.GetDynamicHeaders($"/api2/v2/users/media/{mediaId}/drm/post/{postId}",
|
|
"?type=widevine");
|
|
string decryptionKey;
|
|
if (clientIdBlobMissing || devicePrivateKeyMissing)
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyOFDL(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/post/{postId}?type=widevine",
|
|
pssh);
|
|
}
|
|
else
|
|
{
|
|
decryptionKey = await apiService.GetDecryptionKeyCDM(drmHeaders,
|
|
$"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/post/{postId}?type=widevine",
|
|
pssh);
|
|
}
|
|
|
|
PostEntities.Medium mediaInfo = post.SinglePostMedia.FirstOrDefault(m => m.Id == postKVP.Key);
|
|
PostEntities.SinglePost postInfo =
|
|
post.SinglePostObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true);
|
|
|
|
isNew = await downloadService.DownloadPostDRMVideo(
|
|
policy,
|
|
signature,
|
|
kvp,
|
|
mpdURL,
|
|
decryptionKey,
|
|
path,
|
|
lastModified,
|
|
postKVP.Key,
|
|
"Posts",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ??
|
|
"",
|
|
postInfo,
|
|
mediaInfo,
|
|
postInfo?.Author,
|
|
users);
|
|
}
|
|
else
|
|
{
|
|
try
|
|
{
|
|
PostEntities.Medium? mediaInfo =
|
|
post.SinglePostMedia.FirstOrDefault(m => m.Id == postKVP.Key);
|
|
PostEntities.SinglePost? postInfo =
|
|
post.SinglePostObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true);
|
|
|
|
isNew = await downloadService.DownloadPostMedia(
|
|
postKVP.Value,
|
|
path,
|
|
postKVP.Key,
|
|
"Posts",
|
|
new SpectreProgressReporter(task),
|
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
.PostFileNameFormat ?? "",
|
|
postInfo,
|
|
mediaInfo,
|
|
postInfo?.Author,
|
|
users);
|
|
}
|
|
catch
|
|
{
|
|
Console.WriteLine("Media was null");
|
|
}
|
|
}
|
|
}
|
|
|
|
task.StopTask();
|
|
});
|
|
if (isNew)
|
|
{
|
|
AnsiConsole.Markup($"[red]Post {post_id} downloaded\n[/]");
|
|
Log.Debug($"Post {post_id} downloaded");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup($"[red]Post {post_id} already downloaded\n[/]");
|
|
Log.Debug($"Post {post_id} already downloaded");
|
|
}
|
|
}
|
|
|
|
public async Task<(bool IsExit, Dictionary<string, long>? selectedUsers)> HandleUserSelection(
|
|
Dictionary<string, long> users, Dictionary<string, long> lists)
|
|
{
|
|
IConfigService configService = serviceProvider.GetRequiredService<IConfigService>();
|
|
IAuthService authService = serviceProvider.GetRequiredService<IAuthService>();
|
|
IAPIService apiService = serviceProvider.GetRequiredService<IAPIService>();
|
|
ILoggingService loggingService = serviceProvider.GetRequiredService<ILoggingService>();
|
|
|
|
bool hasSelectedUsers = false;
|
|
Dictionary<string, long> selectedUsers = new();
|
|
Config currentConfig = configService.CurrentConfig!;
|
|
|
|
while (!hasSelectedUsers)
|
|
{
|
|
List<string> mainMenuOptions = GetMainMenuOptions(users, lists);
|
|
|
|
string mainMenuSelection = AnsiConsole.Prompt(
|
|
new SelectionPrompt<string>()
|
|
.Title(
|
|
"[red]Select Accounts to Scrape | Select All = All Accounts | List = Download content from users on List | Custom = Specific Account(s)[/]")
|
|
.AddChoices(mainMenuOptions)
|
|
);
|
|
|
|
switch (mainMenuSelection)
|
|
{
|
|
case "[red]Select All[/]":
|
|
selectedUsers = users;
|
|
hasSelectedUsers = true;
|
|
break;
|
|
case "[red]List[/]":
|
|
while (true)
|
|
{
|
|
MultiSelectionPrompt<string> listSelectionPrompt = new();
|
|
listSelectionPrompt.Title = "[red]Select List[/]";
|
|
listSelectionPrompt.PageSize = 10;
|
|
listSelectionPrompt.AddChoice("[red]Go Back[/]");
|
|
foreach (string key in lists.Keys.Select(k => $"[red]{k}[/]").ToList())
|
|
{
|
|
listSelectionPrompt.AddChoice(key);
|
|
}
|
|
|
|
List<string> listSelection = AnsiConsole.Prompt(listSelectionPrompt);
|
|
|
|
if (listSelection.Contains("[red]Go Back[/]"))
|
|
{
|
|
break; // Go back to the main menu
|
|
}
|
|
|
|
hasSelectedUsers = true;
|
|
List<string> listUsernames = new();
|
|
foreach (string item in listSelection)
|
|
{
|
|
long listId = lists[item.Replace("[red]", "").Replace("[/]", "")];
|
|
List<string> usernames = await apiService.GetListUsers($"/lists/{listId}/users");
|
|
foreach (string user in usernames)
|
|
{
|
|
listUsernames.Add(user);
|
|
}
|
|
}
|
|
|
|
selectedUsers = users.Where(x => listUsernames.Contains($"{x.Key}")).Distinct()
|
|
.ToDictionary(x => x.Key, x => x.Value);
|
|
AnsiConsole.Markup(string.Format("[red]Downloading from List(s): {0}[/]",
|
|
string.Join(", ", listSelection)));
|
|
break;
|
|
}
|
|
|
|
break;
|
|
case "[red]Custom[/]":
|
|
while (true)
|
|
{
|
|
MultiSelectionPrompt<string> selectedNamesPrompt = new();
|
|
selectedNamesPrompt.MoreChoicesText("[grey](Move up and down to reveal more choices)[/]");
|
|
selectedNamesPrompt.InstructionsText(
|
|
"[grey](Press <space> to select, <enter> to accept)[/]\n[grey](Press A-Z to easily navigate the list)[/]");
|
|
selectedNamesPrompt.Title("[red]Select users[/]");
|
|
selectedNamesPrompt.PageSize(10);
|
|
selectedNamesPrompt.AddChoice("[red]Go Back[/]");
|
|
foreach (string key in users.Keys.OrderBy(k => k).Select(k => $"[red]{k}[/]").ToList())
|
|
{
|
|
selectedNamesPrompt.AddChoice(key);
|
|
}
|
|
|
|
List<string> userSelection = AnsiConsole.Prompt(selectedNamesPrompt);
|
|
if (userSelection.Contains("[red]Go Back[/]"))
|
|
{
|
|
break; // Go back to the main menu
|
|
}
|
|
|
|
hasSelectedUsers = true;
|
|
selectedUsers = users.Where(x => userSelection.Contains($"[red]{x.Key}[/]"))
|
|
.ToDictionary(x => x.Key, x => x.Value);
|
|
break;
|
|
}
|
|
|
|
break;
|
|
case "[red]Download Single Post[/]":
|
|
return (true, new Dictionary<string, long> { { "SinglePost", 0 } });
|
|
case "[red]Download Single Paid Message[/]":
|
|
return (true, new Dictionary<string, long> { { "SingleMessage", 0 } });
|
|
case "[red]Download Purchased Tab[/]":
|
|
return (true, new Dictionary<string, long> { { "PurchasedTab", 0 } });
|
|
case "[red]Edit config.conf[/]":
|
|
while (true)
|
|
{
|
|
if (currentConfig == null)
|
|
{
|
|
currentConfig = new Config();
|
|
}
|
|
|
|
List<(string choice, bool isSelected)> choices = new() { ("[red]Go Back[/]", false) };
|
|
|
|
foreach (PropertyInfo propInfo in typeof(Config).GetProperties())
|
|
{
|
|
ToggleableConfigAttribute? attr = propInfo.GetCustomAttribute<ToggleableConfigAttribute>();
|
|
if (attr != null)
|
|
{
|
|
string itemLabel = $"[red]{propInfo.Name}[/]";
|
|
choices.Add(new ValueTuple<string, bool>(itemLabel,
|
|
(bool)propInfo.GetValue(currentConfig)!));
|
|
}
|
|
}
|
|
|
|
MultiSelectionPrompt<string> multiSelectionPrompt = new MultiSelectionPrompt<string>()
|
|
.Title("[red]Edit config.conf[/]")
|
|
.PageSize(25);
|
|
|
|
foreach ((string choice, bool isSelected) choice in choices)
|
|
{
|
|
multiSelectionPrompt.AddChoices(choice.choice, selectionItem =>
|
|
{
|
|
if (choice.isSelected)
|
|
{
|
|
selectionItem.Select();
|
|
}
|
|
});
|
|
}
|
|
|
|
List<string> configOptions = AnsiConsole.Prompt(multiSelectionPrompt);
|
|
|
|
if (configOptions.Contains("[red]Go Back[/]"))
|
|
{
|
|
break;
|
|
}
|
|
|
|
bool configChanged = false;
|
|
|
|
Config newConfig = new();
|
|
foreach (PropertyInfo propInfo in typeof(Config).GetProperties())
|
|
{
|
|
ToggleableConfigAttribute? attr = propInfo.GetCustomAttribute<ToggleableConfigAttribute>();
|
|
if (attr != null)
|
|
{
|
|
//
|
|
// Get the new choice from the selection
|
|
//
|
|
string itemLabel = $"[red]{propInfo.Name}[/]";
|
|
bool newValue = configOptions.Contains(itemLabel);
|
|
bool oldValue = choices.Where(c => c.choice == itemLabel).Select(c => c.isSelected)
|
|
.First();
|
|
propInfo.SetValue(newConfig, newValue);
|
|
|
|
if (newValue != oldValue)
|
|
{
|
|
configChanged = true;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
//
|
|
// Reassign any non toggleable values
|
|
//
|
|
propInfo.SetValue(newConfig, propInfo.GetValue(currentConfig));
|
|
}
|
|
}
|
|
|
|
configService.UpdateConfig(newConfig);
|
|
await configService.SaveConfigurationAsync();
|
|
|
|
currentConfig = newConfig;
|
|
if (configChanged)
|
|
{
|
|
return (true, new Dictionary<string, long> { { "ConfigChanged", 0 } });
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
break;
|
|
case "[red]Change logging level[/]":
|
|
while (true)
|
|
{
|
|
List<(string choice, bool isSelected)> choices = new() { ("[red]Go Back[/]", false) };
|
|
|
|
foreach (string name in typeof(LoggingLevel).GetEnumNames())
|
|
{
|
|
string itemLabel = $"[red]{name}[/]";
|
|
choices.Add(new ValueTuple<string, bool>(itemLabel,
|
|
name == loggingService.GetCurrentLoggingLevel().ToString()));
|
|
}
|
|
|
|
SelectionPrompt<string> selectionPrompt = new SelectionPrompt<string>()
|
|
.Title("[red]Select logging level[/]")
|
|
.PageSize(25);
|
|
|
|
foreach ((string choice, bool isSelected) choice in choices)
|
|
{
|
|
selectionPrompt.AddChoice(choice.choice);
|
|
}
|
|
|
|
string levelOption = AnsiConsole.Prompt(selectionPrompt);
|
|
|
|
if (levelOption.Contains("[red]Go Back[/]"))
|
|
{
|
|
break;
|
|
}
|
|
|
|
levelOption = levelOption.Replace("[red]", "").Replace("[/]", "");
|
|
LoggingLevel newLogLevel = (LoggingLevel)Enum.Parse(typeof(LoggingLevel), levelOption, true);
|
|
|
|
Log.Debug($"Logging level changed to: {levelOption}");
|
|
|
|
bool configChanged = false;
|
|
|
|
Config newConfig = new();
|
|
|
|
newConfig = currentConfig;
|
|
|
|
newConfig.LoggingLevel = newLogLevel;
|
|
|
|
currentConfig = newConfig;
|
|
|
|
configService.UpdateConfig(newConfig);
|
|
await configService.SaveConfigurationAsync();
|
|
|
|
if (configChanged)
|
|
{
|
|
return (true, new Dictionary<string, long> { { "ConfigChanged", 0 } });
|
|
}
|
|
|
|
break;
|
|
}
|
|
|
|
break;
|
|
case "[red]Logout and Exit[/]":
|
|
if (Directory.Exists("chrome-data"))
|
|
{
|
|
Log.Information("Deleting chrome-data folder");
|
|
Directory.Delete("chrome-data", true);
|
|
}
|
|
|
|
if (File.Exists("auth.json"))
|
|
{
|
|
Log.Information("Deleting auth.json");
|
|
File.Delete("auth.json");
|
|
}
|
|
|
|
return (false, null); // Return false to indicate exit
|
|
case "[red]Exit[/]":
|
|
return (false, null); // Return false to indicate exit
|
|
}
|
|
}
|
|
|
|
return (true, selectedUsers); // Return true to indicate selected users
|
|
}
|
|
|
|
public static List<string> GetMainMenuOptions(Dictionary<string, long> users, Dictionary<string, long> lists)
|
|
{
|
|
if (lists.Count > 0)
|
|
{
|
|
return new List<string>
|
|
{
|
|
"[red]Select All[/]",
|
|
"[red]List[/]",
|
|
"[red]Custom[/]",
|
|
"[red]Download Single Post[/]",
|
|
"[red]Download Single Paid Message[/]",
|
|
"[red]Download Purchased Tab[/]",
|
|
"[red]Edit config.conf[/]",
|
|
"[red]Change logging level[/]",
|
|
"[red]Logout and Exit[/]",
|
|
"[red]Exit[/]"
|
|
};
|
|
}
|
|
|
|
return new List<string>
|
|
{
|
|
"[red]Select All[/]",
|
|
"[red]Custom[/]",
|
|
"[red]Download Single Post[/]",
|
|
"[red]Download Single Paid Message[/]",
|
|
"[red]Download Purchased Tab[/]",
|
|
"[red]Edit config.conf[/]",
|
|
"[red]Change logging level[/]",
|
|
"[red]Logout and Exit[/]",
|
|
"[red]Exit[/]"
|
|
};
|
|
}
|
|
|
|
private static bool ValidateFilePath(string path)
|
|
{
|
|
char[] invalidChars = Path.GetInvalidPathChars();
|
|
char[] foundInvalidChars = path.Where(c => invalidChars.Contains(c)).ToArray();
|
|
|
|
if (foundInvalidChars.Any())
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[red]Invalid characters found in path {path}:[/] {string.Join(", ", foundInvalidChars)}\n");
|
|
return false;
|
|
}
|
|
|
|
if (!File.Exists(path))
|
|
{
|
|
if (Directory.Exists(path))
|
|
{
|
|
AnsiConsole.Markup(
|
|
$"[red]The provided path {path} improperly points to a directory and not a file.[/]\n");
|
|
}
|
|
else
|
|
{
|
|
AnsiConsole.Markup($"[red]The provided path {path} does not exist or is not accessible.[/]\n");
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
private static ProgressColumn[] GetProgressColumns(bool showScrapeSize)
|
|
{
|
|
List<ProgressColumn> progressColumns;
|
|
if (showScrapeSize)
|
|
{
|
|
progressColumns = new List<ProgressColumn>
|
|
{
|
|
new TaskDescriptionColumn(),
|
|
new ProgressBarColumn(),
|
|
new PercentageColumn(),
|
|
new DownloadedColumn(),
|
|
new RemainingTimeColumn()
|
|
};
|
|
}
|
|
else
|
|
{
|
|
progressColumns = new List<ProgressColumn>
|
|
{
|
|
new TaskDescriptionColumn(), new ProgressBarColumn(), new PercentageColumn()
|
|
};
|
|
}
|
|
|
|
return progressColumns.ToArray();
|
|
}
|
|
|
|
public static string? GetFullPath(string filename)
|
|
{
|
|
if (File.Exists(filename))
|
|
{
|
|
return Path.GetFullPath(filename);
|
|
}
|
|
|
|
string pathEnv = Environment.GetEnvironmentVariable("PATH") ?? "";
|
|
foreach (string path in pathEnv.Split(Path.PathSeparator))
|
|
{
|
|
string fullPath = Path.Combine(path, filename);
|
|
if (File.Exists(fullPath))
|
|
{
|
|
return fullPath;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public static void ValidateCookieString(Auth auth)
|
|
{
|
|
string pattern = @"(auth_id=\d+)|(sess=[^;]+)";
|
|
MatchCollection matches = Regex.Matches(auth.Cookie, pattern);
|
|
|
|
string output = string.Join("; ", matches);
|
|
|
|
if (!output.EndsWith(";"))
|
|
{
|
|
output += ";";
|
|
}
|
|
|
|
if (auth.Cookie.Trim() != output.Trim())
|
|
{
|
|
auth.Cookie = output;
|
|
string newAuthString = JsonConvert.SerializeObject(auth, Formatting.Indented);
|
|
File.WriteAllText("auth.json", newAuthString);
|
|
}
|
|
}
|
|
}
|