using System.Globalization; using System.Diagnostics; using System.Security.Cryptography; using System.Text.RegularExpressions; using FFmpeg.NET; using FFmpeg.NET.Events; using OF_DL.Models; using OF_DL.Enumerations; using OF_DL.Helpers; using OF_DL.Models.Downloads; using ArchivedEntities = OF_DL.Models.Entities.Archived; using MessageEntities = OF_DL.Models.Entities.Messages; using PostEntities = OF_DL.Models.Entities.Posts; using PurchasedEntities = OF_DL.Models.Entities.Purchased; using StreamEntities = OF_DL.Models.Entities.Streams; using OF_DL.Utils; using Serilog; using Serilog.Events; namespace OF_DL.Services; public class DownloadService( IAuthService authService, IConfigService configService, IDbService dbService, IFileNameService fileNameService, IApiService apiService) : IDownloadService { private TaskCompletionSource _completionSource = new(); /// /// Downloads profile avatar and header images for a creator. /// /// The avatar URL. /// The header URL. /// The creator folder path. /// The creator username. public async Task DownloadAvatarHeader(string? avatarUrl, string? headerUrl, string folder, string username) { try { const string path = "/Profile"; if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(avatarUrl)) { await DownloadProfileImage(avatarUrl, folder, $"{path}/Avatars", username); } if (!string.IsNullOrEmpty(headerUrl)) { await DownloadProfileImage(headerUrl, folder, $"{path}/Headers", username); } } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } } private static async Task DownloadProfileImage(string url, string folder, string subFolder, string username) { if (!Directory.Exists(folder + subFolder)) { Directory.CreateDirectory(folder + subFolder); } List md5Hashes = CalculateFolderMd5(folder + subFolder); Uri uri = new(url); string destinationPath = $"{folder}{subFolder}/"; HttpClient client = new(); HttpRequestMessage request = new() { Method = HttpMethod.Get, RequestUri = uri }; using HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); using MemoryStream memoryStream = new(); await response.Content.CopyToAsync(memoryStream); memoryStream.Seek(0, SeekOrigin.Begin); MD5 md5 = MD5.Create(); byte[] hash = await md5.ComputeHashAsync(memoryStream); memoryStream.Seek(0, SeekOrigin.Begin); if (!md5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant())) { destinationPath = destinationPath + string.Format("{0} {1}.jpg", username, response.Content.Headers.LastModified.HasValue ? response.Content.Headers.LastModified.Value.LocalDateTime.ToString("dd-MM-yyyy") : DateTime.Now.ToString("dd-MM-yyyy")); await using (FileStream fileStream = File.Create(destinationPath)) { await memoryStream.CopyToAsync(fileStream); } File.SetLastWriteTime(destinationPath, response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now); } } private async Task DownloadDrmMedia( string userAgent, string policy, string signature, string kvp, string sess, string url, string decryptionKey, string folder, DateTime lastModified, long mediaId, string apiType, IProgressReporter progressReporter, string customFileName, string filename, string path, double? expectedDurationSeconds) { try { int pos1 = decryptionKey.IndexOf(':'); string decKey = ""; if (pos1 >= 0) { decKey = decryptionKey[(pos1 + 1)..]; } const int streamIndex = 0; string tempFilename = $"{folder}{path}/{filename}_source.mp4"; string finalFilePath = !string.IsNullOrEmpty(customFileName) ? $"{folder}{path}/{customFileName}.mp4" : tempFilename; // Configure ffmpeg log level and optional report file location. string ffToolLogLevel = GetFfToolLogLevel(); bool enableFfReport = string.Equals(ffToolLogLevel, "debug", StringComparison.OrdinalIgnoreCase); string logLevelArgs = enableFfReport ? "-loglevel debug -report" : $"-loglevel {ffToolLogLevel}"; if (enableFfReport) { // Use a relative path so FFREPORT parsing works on Windows (drive-letter ':' breaks option parsing). string logDir = Path.Combine(Environment.CurrentDirectory, "logs"); Directory.CreateDirectory(logDir); string ffReportPath = Path.Combine("logs", "ffmpeg-%p-%t.log").Replace("\\", "/"); Environment.SetEnvironmentVariable("FFREPORT", $"file={ffReportPath}:level=32"); Log.Debug("FFREPORT enabled at: {FFREPORT} (cwd: {Cwd})", Environment.GetEnvironmentVariable("FFREPORT"), Environment.CurrentDirectory); } else { Environment.SetEnvironmentVariable("FFREPORT", null); Log.Debug("FFREPORT disabled (cwd: {Cwd})", Environment.CurrentDirectory); } string cookieHeader = "Cookie: " + $"CloudFront-Policy={policy}; " + $"CloudFront-Signature={signature}; " + $"CloudFront-Key-Pair-Id={kvp}; " + $"{sess}"; if (expectedDurationSeconds.HasValue) { Log.Debug("Expected DRM video duration for media {MediaId}: {ExpectedDurationSeconds:F2}s", mediaId, expectedDurationSeconds.Value); } else { Log.Warning("MPD video duration missing for media {MediaId}; skipping DRM duration validation.", mediaId); } double threshold = configService.CurrentConfig.DrmVideoDurationMatchThreshold; for (int attempt = 1; attempt <= Constants.DrmDownloadMaxRetries; attempt++) { TryDeleteFile(tempFilename); if (!string.Equals(finalFilePath, tempFilename, StringComparison.OrdinalIgnoreCase)) { TryDeleteFile(finalFilePath); } _completionSource = new TaskCompletionSource(); string parameters = $"{logLevelArgs} " + $"-cenc_decryption_key {decKey} " + $"-headers \"{cookieHeader}\" " + $"-user_agent \"{userAgent}\" " + "-referer \"https://onlyfans.com\" " + "-rw_timeout 20000000 " + "-reconnect 1 -reconnect_streamed 1 -reconnect_on_network_error 1 -reconnect_delay_max 10 " + "-y " + $"-i \"{url}\" " + $"-map 0:v:{streamIndex} -map 0:a? " + "-c copy " + $"\"{tempFilename}\""; Log.Debug("Calling FFmpeg with Parameters: {Parameters}", parameters); Engine ffmpeg = new(configService.CurrentConfig.FFmpegPath); ffmpeg.Error += OnError; ffmpeg.Complete += (_, _) => { _completionSource.TrySetResult(true); }; await ffmpeg.ExecuteAsync(parameters, progressReporter.CancellationToken); bool ffmpegSuccess = await _completionSource.Task; if (!ffmpegSuccess || !File.Exists(tempFilename)) { Log.Warning("DRM download attempt {Attempt}/{MaxAttempts} failed for media {MediaId}.", attempt, Constants.DrmDownloadMaxRetries, mediaId); continue; } if (!expectedDurationSeconds.HasValue) { return await FinalizeDrmDownload(tempFilename, lastModified, folder, path, customFileName, filename, mediaId, apiType, progressReporter); } double? actualDurationSeconds = await TryGetVideoDurationSeconds(tempFilename); if (!actualDurationSeconds.HasValue) { Log.Warning( "DRM download attempt {Attempt}/{MaxAttempts} could not determine output duration for media {MediaId}.", attempt, Constants.DrmDownloadMaxRetries, mediaId); continue; } double durationRatio = actualDurationSeconds.Value / expectedDurationSeconds.Value; Log.Debug("Expected duration: {ExpectedSeconds:F2}s Actual duration: {ActualSeconds:F2}s", expectedDurationSeconds.Value, actualDurationSeconds.Value); Log.Debug("Ratio: {Ratio:P2} Threshold: {Threshold:P2}", durationRatio, threshold); if (durationRatio >= threshold) { return await FinalizeDrmDownload(tempFilename, lastModified, folder, path, customFileName, filename, mediaId, apiType, progressReporter); } Log.Warning( "DRM download attempt {Attempt}/{MaxAttempts} produced a short file for media {MediaId}. Expected={ExpectedSeconds:F2}s Actual={ActualSeconds:F2}s Ratio={Ratio:P2} Threshold={Threshold:P2}.", attempt, Constants.DrmDownloadMaxRetries, mediaId, expectedDurationSeconds.Value, actualDurationSeconds.Value, durationRatio, threshold); } TryDeleteFile(tempFilename); Log.Warning("DRM download failed validation after {MaxAttempts} attempts for media {MediaId}.", Constants.DrmDownloadMaxRetries, mediaId); return false; } catch (OperationCanceledException) { throw; } catch (Exception ex) { ExceptionLoggerHelper.LogException(ex); return false; } } private async Task TryGetVideoDurationSeconds(string filePath) { try { string ffprobePath = configService.CurrentConfig.FFprobePath ?? throw new InvalidOperationException("FFprobePath is not configured."); string ffprobeLogLevel = GetFfToolLogLevel(); ProcessStartInfo startInfo = new() { FileName = ffprobePath, Arguments = $"-v {ffprobeLogLevel} -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 \"{filePath}\"", RedirectStandardOutput = true, RedirectStandardError = true, UseShellExecute = false, CreateNoWindow = true }; using Process process = new(); process.StartInfo = startInfo; process.Start(); Task outputTask = process.StandardOutput.ReadToEndAsync(); Task errorTask = process.StandardError.ReadToEndAsync(); Task waitForExitTask = process.WaitForExitAsync(); await Task.WhenAll(outputTask, errorTask, waitForExitTask); string output = outputTask.Result; string error = errorTask.Result; if (process.ExitCode != 0) { Log.Warning("FFprobe failed for file {FilePath}. ExitCode={ExitCode} Error={Error}", filePath, process.ExitCode, error); return null; } string[] outputLines = output.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries); foreach (string outputLine in outputLines) { if (double.TryParse(outputLine.Trim(), NumberStyles.Float, CultureInfo.InvariantCulture, out double durationSeconds)) { return durationSeconds > 0 ? durationSeconds : null; } } Log.Warning("Unable to parse FFprobe duration for file {FilePath}. RawOutput={RawOutput}", filePath, output.Trim()); return null; } catch (Exception ex) { Log.Warning(ex, "Failed to inspect downloaded video duration for {FilePath}", filePath); return null; } } private string GetFfToolLogLevel() { bool ffToolDebugLogging = Log.IsEnabled(LogEventLevel.Debug); if (ffToolDebugLogging || configService.CurrentConfig.LoggingLevel is LoggingLevel.Verbose or LoggingLevel.Debug) { return "debug"; } return configService.CurrentConfig.LoggingLevel switch { LoggingLevel.Information => "info", LoggingLevel.Warning => "warning", LoggingLevel.Error => "error", LoggingLevel.Fatal => "fatal", _ => "error" }; } private async Task FinalizeDrmDownload(string tempFilename, DateTime lastModified, string folder, string path, string customFileName, string filename, long mediaId, string apiType, IProgressReporter progressReporter) { try { if (!File.Exists(tempFilename)) { return false; } File.SetLastWriteTime(tempFilename, lastModified); string finalPath = tempFilename; string finalName = filename + "_source.mp4"; if (!string.IsNullOrEmpty(customFileName)) { finalPath = $"{folder}{path}/{customFileName}.mp4"; finalName = customFileName + ".mp4"; if (!AreSamePath(tempFilename, finalPath)) { TryDeleteFile(finalPath); File.Move(tempFilename, finalPath); } } long fileSizeInBytes = new FileInfo(finalPath).Length; ReportProgress(progressReporter, fileSizeInBytes); await dbService.UpdateMedia(folder, mediaId, apiType, folder + path, finalName, fileSizeInBytes, true, lastModified); return true; } catch (Exception ex) { ExceptionLoggerHelper.LogException(ex); return false; } } private static void TryDeleteFile(string filePath) { try { if (File.Exists(filePath)) { File.Delete(filePath); } } catch { // Best-effort cleanup only. } } private static bool AreSamePath(string firstPath, string secondPath) { try { string firstFullPath = Path.GetFullPath(firstPath); string secondFullPath = Path.GetFullPath(secondPath); StringComparison comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; return string.Equals(firstFullPath, secondFullPath, comparison); } catch { StringComparison comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; return string.Equals(firstPath, secondPath, comparison); } } private void OnError(object? sender, ConversionErrorEventArgs e) { // Guard all fields to avoid NullReference exceptions from FFmpeg.NET string input = e.Input?.Name ?? ""; string output = e.Output?.Name ?? ""; string exitCode = e.Exception?.ExitCode.ToString() ?? ""; string message = e.Exception?.Message ?? ""; string inner = e.Exception?.InnerException?.Message ?? ""; Log.Error("FFmpeg failed. Input={Input} Output={Output} ExitCode={ExitCode} Message={Message} Inner={Inner}", input, output, exitCode, message, inner); _completionSource.TrySetResult(false); } private static List CalculateFolderMd5(string folder) { List md5Hashes = []; if (!Directory.Exists(folder)) { return md5Hashes; } string[] files = Directory.GetFiles(folder); md5Hashes.AddRange(files.Select(CalculateMd5)); return md5Hashes; } private static string CalculateMd5(string filePath) { using MD5 md5 = MD5.Create(); using FileStream stream = File.OpenRead(filePath); byte[] hash = md5.ComputeHash(stream); return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); } /// /// /// /// /// /// /// /// /// /// /// private async Task CreateDirectoriesAndDownloadMedia(string path, string url, string folder, long mediaId, string apiType, IProgressReporter progressReporter, string serverFileName, string resolvedFileName) { try { if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } string extension = Path.GetExtension(url.Split("?")[0]); path = UpdatePathBasedOnExtension(folder, path, extension); return await ProcessMediaDownload(folder, mediaId, apiType, url, path, serverFileName, resolvedFileName, extension, progressReporter); } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } /// /// Updates the given path based on the file extension. /// /// The parent folder. /// The initial relative path. /// The file extension. /// A string that represents the updated path based on the file extension. private static string UpdatePathBasedOnExtension(string folder, string path, string extension) { string subdirectory = ""; switch (extension.ToLower()) { case ".jpg": case ".jpeg": case ".png": subdirectory = "/Images"; break; case ".mp4": case ".avi": case ".wmv": case ".gif": case ".mov": subdirectory = "/Videos"; break; case ".mp3": case ".wav": case ".ogg": subdirectory = "/Audios"; break; } if (!string.IsNullOrEmpty(subdirectory)) { path += subdirectory; string fullPath = folder + path; if (!Directory.Exists(fullPath)) { Directory.CreateDirectory(fullPath); } } return path; } /// /// Generates a custom filename based on the given format and properties. /// /// /// The format string for the filename. /// General information about the post. /// Media associated with the post. /// Author of the post. /// /// Dictionary containing user-related data. /// /// /// A Task resulting in a string that represents the custom filename. private static async Task GenerateCustomFileName(string filename, string? filenameFormat, object? postInfo, object? postMedia, object? author, string username, Dictionary users, IFileNameService fileNameService, CustomFileNameOption option) { if (string.IsNullOrEmpty(filenameFormat) || postInfo == null || postMedia == null || author == null) { return option switch { CustomFileNameOption.ReturnOriginal => filename, CustomFileNameOption.ReturnEmpty => "", _ => filename }; } List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); properties.AddRange(matches.Select(match => match.Groups[1].Value)); Dictionary values = await fileNameService.GetFilename(postInfo, postMedia, author, properties, username, users); return await fileNameService.BuildFilename(filenameFormat, values); } private async Task GetFileSizeAsync(string url) { long fileSize = 0; try { if (authService.CurrentAuth == null) { throw new Exception("No authentication information available."); } if (authService.CurrentAuth.Cookie == null) { throw new Exception("No authentication cookie available."); } if (authService.CurrentAuth.UserAgent == null) { throw new Exception("No user agent available."); } Uri uri = new(url); if (uri.Host == "cdn3.onlyfans.com" && uri.LocalPath.Contains("/dash/files")) { string[] messageUrlParsed = url.Split(','); string mpdUrl = messageUrlParsed[0]; string policy = messageUrlParsed[1]; string signature = messageUrlParsed[2]; string kvp = messageUrlParsed[3]; mpdUrl = mpdUrl.Replace(".mpd", "_source.mp4"); using HttpClient client = new(); client.DefaultRequestHeaders.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {authService.CurrentAuth.Cookie}"); client.DefaultRequestHeaders.Add("User-Agent", authService.CurrentAuth.UserAgent); using HttpResponseMessage response = await client.GetAsync(mpdUrl, HttpCompletionOption.ResponseHeadersRead); if (response.IsSuccessStatusCode) { fileSize = response.Content.Headers.ContentLength ?? 0; } } else { using HttpClient client = new(); client.DefaultRequestHeaders.Add("User-Agent", authService.CurrentAuth.UserAgent); using HttpResponseMessage response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead); if (response.IsSuccessStatusCode) { fileSize = response.Content.Headers.ContentLength ?? 0; } } } catch (Exception ex) { Console.WriteLine($"Error getting file size for URL '{url}': {ex.Message}"); } return fileSize; } /// /// Retrieves the last modified timestamp for a DRM media URL. /// /// The DRM media URL (including CloudFront tokens). /// The current auth context. /// The last modified timestamp if available. public static async Task GetDrmVideoLastModified(string url, Auth auth) { string[] messageUrlParsed = url.Split(','); string mpdUrl = messageUrlParsed[0]; string policy = messageUrlParsed[1]; string signature = messageUrlParsed[2]; string kvp = messageUrlParsed[3]; mpdUrl = mpdUrl.Replace(".mpd", "_source.mp4"); using HttpClient client = new(); client.DefaultRequestHeaders.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.Cookie}"); client.DefaultRequestHeaders.Add("User-Agent", auth.UserAgent); using HttpResponseMessage response = await client.GetAsync(mpdUrl, HttpCompletionOption.ResponseHeadersRead); return response is { IsSuccessStatusCode: true, Content.Headers.LastModified: not null } ? response.Content.Headers.LastModified.Value.DateTime : DateTime.Now; } /// /// Retrieves the last modified timestamp for a media URL. /// /// The media URL. /// The last modified timestamp if available. public static async Task GetMediaLastModified(string url) { using HttpClient client = new(); using HttpResponseMessage response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); if (!response.IsSuccessStatusCode) { return DateTime.Now; } return response.Content.Headers.LastModified?.DateTime ?? DateTime.Now; } /// /// Processes the download and database update of media. /// /// The folder where the media is stored. /// The ID of the media. /// /// The URL from where to download the media. /// The relative path to the media. /// /// The filename after any required manipulations. /// The file extension. /// /// A Task resulting in a boolean indicating whether the media is newly downloaded or not. public async Task ProcessMediaDownload(string folder, long mediaId, string apiType, string url, string path, string serverFilename, string resolvedFilename, string extension, IProgressReporter progressReporter) { try { if (!await dbService.CheckDownloaded(folder, mediaId, apiType)) { return await HandleNewMedia(folder, mediaId, apiType, url, path, serverFilename, resolvedFilename, extension, progressReporter); } bool status = await HandlePreviouslyDownloadedMediaAsync(folder, mediaId, apiType, progressReporter); if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && serverFilename != resolvedFilename) { await HandleRenamingOfExistingFilesAsync(folder, mediaId, apiType, path, serverFilename, resolvedFilename, extension); } return status; } catch (Exception ex) { // Handle exception (e.g., log it) Console.WriteLine($"An error occurred: {ex.Message}"); return false; } } private async Task HandleRenamingOfExistingFilesAsync(string folder, long mediaId, string apiType, string path, string serverFilename, string resolvedFilename, string extension) { string fullPathWithTheServerFileName = $"{folder}{path}/{serverFilename}{extension}"; string fullPathWithTheNewFileName = $"{folder}{path}/{resolvedFilename}{extension}"; if (!File.Exists(fullPathWithTheServerFileName)) { return; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return; } long size = await dbService.GetStoredFileSize(folder, mediaId, apiType); DateTime lastModified = File.GetLastWriteTime(fullPathWithTheNewFileName); await dbService.UpdateMedia(folder, mediaId, apiType, folder + path, resolvedFilename + extension, size, true, lastModified); } /// /// Handles new media by downloading and updating the database. /// /// /// /// /// /// /// /// /// /// /// A Task resulting in a boolean indicating whether the media is newly downloaded or not. private async Task HandleNewMedia(string folder, long mediaId, string apiType, string url, string path, string serverFilename, string resolvedFilename, string extension, IProgressReporter progressReporter) { long fileSizeInBytes; DateTime lastModified; bool status; string fullPathWithTheServerFileName = $"{folder}{path}/{serverFilename}{extension}"; string fullPathWithTheNewFileName = $"{folder}{path}/{resolvedFilename}{extension}"; //there are a few possibilities here. //1.file has been downloaded in the past but it has the server filename // in that case it should be set as existing and it should be renamed //2.file has been downloaded in the past but it has custom filename. // it should be set as existing and nothing else. // of coures 1 and 2 depends in the fact that there may be a difference in the resolved file name // (ie user has selected a custom format. If he doesn't then the resolved name will be the same as the server filename //3.file doesn't exist and it should be downloaded. // Handle the case where the file has been downloaded in the past with the server filename //but it has downloaded outsite of this application so it doesn't exist in the database if (File.Exists(fullPathWithTheServerFileName)) { string finalPath; if (fullPathWithTheServerFileName != fullPathWithTheNewFileName) { finalPath = fullPathWithTheNewFileName; //rename. try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); } } else { finalPath = fullPathWithTheServerFileName; } fileSizeInBytes = GetLocalFileSize(finalPath); lastModified = File.GetLastWriteTime(finalPath); ReportProgress(progressReporter, fileSizeInBytes); status = false; } // Handle the case where the file has been downloaded in the past with a custom filename. // but it has downloaded outside of this application so it doesn't exist in the database // this is a bit improbable but we should check for that. else if (File.Exists(fullPathWithTheNewFileName)) { fileSizeInBytes = GetLocalFileSize(fullPathWithTheNewFileName); lastModified = File.GetLastWriteTime(fullPathWithTheNewFileName); ReportProgress(progressReporter, fileSizeInBytes); status = false; } else //file doesn't exist and we should download it. { lastModified = await DownloadFile(url, fullPathWithTheNewFileName, progressReporter); fileSizeInBytes = GetLocalFileSize(fullPathWithTheNewFileName); status = true; } //finaly check which filename we should use. Custom or the server one. //if a custom is used, then the serverFilename will be different from the resolved filename. string finalName = serverFilename == resolvedFilename ? serverFilename : resolvedFilename; await dbService.UpdateMedia(folder, mediaId, apiType, folder + path, finalName + extension, fileSizeInBytes, true, lastModified); return status; } /// /// Handles media that has been previously downloaded and updates the task accordingly. /// /// /// /// /// /// A boolean indicating whether the media is newly downloaded or not. private async Task HandlePreviouslyDownloadedMediaAsync(string folder, long mediaId, string apiType, IProgressReporter progressReporter) { long size = configService.CurrentConfig.ShowScrapeSize ? await dbService.GetStoredFileSize(folder, mediaId, apiType) : 1; ReportProgress(progressReporter, size); return false; } /// /// Gets the file size of the media. /// /// The path to the file. /// The file size in bytes. private static long GetLocalFileSize(string filePath) => new FileInfo(filePath).Length; /// /// Downloads a file from the given URL and saves it to the specified destination path. /// /// The URL to download the file from. /// The path where the downloaded file will be saved. /// /// A Task resulting in a DateTime indicating the last modified date of the downloaded file. private async Task DownloadFile(string url, string destinationPath, IProgressReporter progressReporter) { using HttpClient client = new(); HttpRequestMessage request = new() { Method = HttpMethod.Get, RequestUri = new Uri(url) }; using HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, progressReporter.CancellationToken); response.EnsureSuccessStatusCode(); Stream body = await response.Content.ReadAsStreamAsync(progressReporter.CancellationToken); // Wrap the body stream with the ThrottledStream to limit read rate. await using (ThrottledStream throttledStream = new(body, configService.CurrentConfig.DownloadLimitInMbPerSec * 1_000_000, configService.CurrentConfig.LimitDownloadRate)) { await using FileStream fileStream = new(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, 16384, true); byte[] buffer = new byte[16384]; int read; while ((read = await throttledStream.ReadAsync(buffer, progressReporter.CancellationToken)) > 0) { if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(read); } await fileStream.WriteAsync(buffer.AsMemory(0, read), progressReporter.CancellationToken); } } File.SetLastWriteTime(destinationPath, response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now); if (!configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(1); } return response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now; } /// /// Calculates the total size of a set of URLs by fetching their metadata. /// /// The media URLs. /// The total size in bytes. public async Task CalculateTotalFileSize(List urls) { long totalFileSize = 0; if (urls.Count > 250) { const int batchSize = 250; List> tasks = []; for (int i = 0; i < urls.Count; i += batchSize) { List batchUrls = urls.Skip(i).Take(batchSize).ToList(); Task[] batchTasks = batchUrls.Select(GetFileSizeAsync).ToArray(); tasks.AddRange(batchTasks); await Task.WhenAll(batchTasks); await Task.Delay(5000); } long[] fileSizes = await Task.WhenAll(tasks); totalFileSize += fileSizes.Sum(); } else { List> tasks = []; tasks.AddRange(urls.Select(GetFileSizeAsync)); long[] fileSizes = await Task.WhenAll(tasks); totalFileSize += fileSizes.Sum(); } return totalFileSize; } /// /// Downloads a single media item, applying filename formatting and folder rules. /// /// The media URL. /// The creator folder path. /// The media ID. /// The API type label. /// Progress reporter. /// The relative folder path. /// Optional filename format. /// Post or message info. /// Media info. /// Author info. /// Known users map. /// True when the media is newly downloaded. private async Task DownloadMedia(string url, string folder, long mediaId, string apiType, IProgressReporter progressReporter, string path, string? filenameFormat, object? postInfo, object? postMedia, object? author, Dictionary users) { Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, postInfo, postMedia, author, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, mediaId, apiType, progressReporter, filename, resolvedFilename); } /// /// Downloads a DRM-protected video using the provided decryption key. /// /// CloudFront policy token. /// CloudFront signature token. /// CloudFront key pair ID. /// The MPD URL. /// The decryption key. /// The creator folder path. /// The source last modified timestamp. /// The media ID. /// The API type label. /// Progress reporter. /// The relative folder path. /// Optional filename format. /// Post or message info. /// Media info. /// Author info. /// Known users map. /// The expected duration of the video in seconds. /// True when the media is newly downloaded. private async Task DownloadDrmVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long mediaId, string apiType, IProgressReporter progressReporter, string path, string? filenameFormat, object? postInfo, object? postMedia, object? author, Dictionary users, double? expectedDurationSeconds) { try { if (authService.CurrentAuth == null) { throw new Exception("No authentication information available."); } if (authService.CurrentAuth.Cookie == null) { throw new Exception("No authentication cookie available."); } if (authService.CurrentAuth.UserAgent == null) { throw new Exception("No user agent available."); } Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } string customFileName = await GenerateCustomFileName(filename, filenameFormat, postInfo, postMedia, author, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnEmpty); if (!await dbService.CheckDownloaded(folder, mediaId, apiType)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.UserAgent, policy, signature, kvp, authService.CurrentAuth.Cookie, url, decryptionKey, folder, lastModified, mediaId, apiType, progressReporter, customFileName, filename, path, expectedDurationSeconds); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; ReportProgress(progressReporter, fileSizeInBytes); await dbService.UpdateMedia(folder, mediaId, apiType, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + ".mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long storedFileSize = await dbService.GetStoredFileSize(folder, mediaId, apiType); await dbService.UpdateMedia(folder, mediaId, apiType, folder + path, customFileName + ".mp4", storedFileSize, true, lastModified); } } long progressSize = configService.CurrentConfig.ShowScrapeSize ? await dbService.GetStoredFileSize(folder, mediaId, apiType) : 1; ReportProgress(progressReporter, progressSize); } return false; } catch (Exception ex) { ExceptionLoggerHelper.LogException(ex); } return false; } private void ReportProgress(IProgressReporter reporter, long sizeOrCount) => reporter.ReportProgress(configService.CurrentConfig.ShowScrapeSize ? sizeOrCount : 1); /// /// Retrieves decryption information for a DRM media item. /// /// The MPD URL. /// CloudFront policy token. /// CloudFront signature token. /// CloudFront key pair ID. /// The media ID. /// The content ID. /// The DRM type. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The decryption key, last modified timestamp, and MPD duration seconds. public async Task<(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)?> GetDecryptionInfo( string mpdUrl, string policy, string signature, string kvp, string mediaId, string contentId, string drmType, bool clientIdBlobMissing, bool devicePrivateKeyMissing) { (string pssh, DateTime lastModified, double? durationSeconds) = await apiService.GetDrmMpdInfo(mpdUrl, policy, signature, kvp); Dictionary drmHeaders = apiService.GetDynamicHeaders($"/api2/v2/users/media/{mediaId}/drm/{drmType}/{contentId}", "?type=widevine"); string licenseUrl = $"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/{drmType}/{contentId}?type=widevine"; string decryptionKey = clientIdBlobMissing || devicePrivateKeyMissing ? await apiService.GetDecryptionKeyOfdl(drmHeaders, licenseUrl, pssh) : await apiService.GetDecryptionKeyCdm(drmHeaders, licenseUrl, pssh); return (decryptionKey, lastModified, durationSeconds); } /// /// Downloads highlight media for a creator. /// /// The creator username. /// The creator user ID. /// The creator folder path. /// Paid post media IDs. /// Progress reporter. /// The download result. public async Task DownloadHighlights(string username, long userId, string path, HashSet paidPostIds, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadHighlights - {username}"); Dictionary? highlights = await apiService.GetMedia(MediaType.Highlights, $"/users/{userId}/stories/highlights", null, path); if (highlights == null || highlights.Count == 0) { Log.Debug("Found 0 Highlights"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Highlights", Success = true }; } Log.Debug($"Found {highlights.Count} Highlights"); int oldHighlightsCount = 0; int newHighlightsCount = 0; foreach (KeyValuePair highlightKvp in highlights) { bool isNew = await DownloadMedia(highlightKvp.Value, path, highlightKvp.Key, "Stories", progressReporter, "/Stories/Free", null, null, null, null, new Dictionary()); if (isNew) { newHighlightsCount++; } else { oldHighlightsCount++; } } Log.Debug( $"Highlights Media Already Downloaded: {oldHighlightsCount} New Highlights Media Downloaded: {newHighlightsCount}"); return new DownloadResult { TotalCount = highlights.Count, NewDownloads = newHighlightsCount, ExistingDownloads = oldHighlightsCount, MediaType = "Highlights", Success = true }; } /// /// Downloads story media for a creator. /// /// The creator username. /// The creator user ID. /// The creator folder path. /// Paid post media IDs. /// Progress reporter. /// The download result. public async Task DownloadStories(string username, long userId, string path, HashSet paidPostIds, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadStories - {username}"); Dictionary? stories = await apiService.GetMedia(MediaType.Stories, $"/users/{userId}/stories", null, path); if (stories == null || stories.Count == 0) { Log.Debug("Found 0 Stories"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Stories", Success = true }; } Log.Debug($"Found {stories.Count} Stories"); int oldStoriesCount = 0; int newStoriesCount = 0; foreach (KeyValuePair storyKvp in stories) { bool isNew = await DownloadMedia(storyKvp.Value, path, storyKvp.Key, "Stories", progressReporter, "/Stories/Free", null, null, null, null, new Dictionary()); if (isNew) { newStoriesCount++; } else { oldStoriesCount++; } } Log.Debug( $"Stories Media Already Downloaded: {oldStoriesCount} New Stories Media Downloaded: {newStoriesCount}"); return new DownloadResult { TotalCount = stories.Count, NewDownloads = newStoriesCount, ExistingDownloads = oldStoriesCount, MediaType = "Stories", Success = true }; } /// /// Downloads archived posts for a creator. /// /// The creator username. /// The creator user ID. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The archived posts collection. /// Progress reporter. /// The download result. public async Task DownloadArchived(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, ArchivedEntities.ArchivedCollection archived, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadArchived - {username}"); if (archived.ArchivedPosts.Count == 0) { Log.Debug("Found 0 Archived Posts"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Archived Posts", Success = true }; } Log.Debug( $"Found {archived.ArchivedPosts.Count} Media from {archived.ArchivedPostObjects.Count} Archived Posts"); int oldArchivedCount = 0; int newArchivedCount = 0; foreach (KeyValuePair archivedKvp in archived.ArchivedPosts) { bool isNew; ArchivedEntities.Medium? mediaInfo = archived.ArchivedPostMedia.FirstOrDefault(m => m.Id == archivedKvp.Key); ArchivedEntities.ListItem? postInfo = mediaInfo == null ? null : archived.ArchivedPostObjects.FirstOrDefault(p => p.Media?.Contains(mediaInfo) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? ""; if (archivedKvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = archivedKvp.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, archivedKvp.Key, "Posts", progressReporter, "/Archived/Posts/Free/Videos", filenameFormat, postInfo, mediaInfo, postInfo?.Author, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(archivedKvp.Value, path, archivedKvp.Key, "Posts", progressReporter, "/Archived/Posts/Free", filenameFormat, postInfo, mediaInfo, postInfo?.Author, users); } if (isNew) { newArchivedCount++; } else { oldArchivedCount++; } } Log.Debug( $"Archived Posts Media Already Downloaded: {oldArchivedCount} New Archived Posts Media Downloaded: {newArchivedCount}"); return new DownloadResult { TotalCount = archived.ArchivedPosts.Count, NewDownloads = newArchivedCount, ExistingDownloads = oldArchivedCount, MediaType = "Archived Posts", Success = true }; } /// /// Downloads free messages for a creator. /// /// The creator username. /// The creator user ID. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The messages collection. /// Progress reporter. /// The download result. public async Task DownloadMessages(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, MessageEntities.MessageCollection messages, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadMessages - {username}"); if (messages.Messages.Count == 0) { Log.Debug("Found 0 Messages"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Messages", Success = true }; } Log.Debug($"Found {messages.Messages.Count} Media from {messages.MessageObjects.Count} Messages"); int oldMessagesCount = 0; int newMessagesCount = 0; foreach (KeyValuePair messageKvp in messages.Messages) { bool isNew; MessageEntities.Medium? mediaInfo = messages.MessageMedia.FirstOrDefault(m => m.Id == messageKvp.Key); MessageEntities.ListItem? messageInfo = messages.MessageObjects.FirstOrDefault(p => p.Media?.Any(m => m.Id == messageKvp.Key) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).MessageFileNameFormat ?? ""; string messagePath = configService.CurrentConfig.FolderPerMessage && messageInfo != null && messageInfo.Id != 0 && messageInfo.CreatedAt is not null ? $"/Messages/Free/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" : "/Messages/Free"; if (messageKvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = messageKvp.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, messageKvp.Key, "Messages", progressReporter, messagePath + "/Videos", filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(messageKvp.Value, path, messageKvp.Key, "Messages", progressReporter, messagePath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users); } if (isNew) { newMessagesCount++; } else { oldMessagesCount++; } } Log.Debug( $"Messages Media Already Downloaded: {oldMessagesCount} New Messages Media Downloaded: {newMessagesCount}"); return new DownloadResult { TotalCount = messages.Messages.Count, NewDownloads = newMessagesCount, ExistingDownloads = oldMessagesCount, MediaType = "Messages", Success = true }; } /// /// Downloads paid messages for a creator. /// /// The creator username. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The paid message collection. /// Progress reporter. /// The download result. public async Task DownloadPaidMessages(string username, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PurchasedEntities.PaidMessageCollection paidMessageCollection, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadPaidMessages - {username}"); if (paidMessageCollection.PaidMessages.Count == 0) { Log.Debug("Found 0 Paid Messages"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Paid Messages", Success = true }; } Log.Debug( $"Found {paidMessageCollection.PaidMessages.Count} Media from {paidMessageCollection.PaidMessageObjects.Count} Paid Messages"); int oldCount = 0; int newCount = 0; foreach (KeyValuePair kvpEntry in paidMessageCollection.PaidMessages) { bool isNew; MessageEntities.Medium? mediaInfo = paidMessageCollection.PaidMessageMedia.FirstOrDefault(m => m.Id == kvpEntry.Key); PurchasedEntities.ListItem? messageInfo = paidMessageCollection.PaidMessageObjects.FirstOrDefault(p => p.Media?.Any(m => m.Id == kvpEntry.Key) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PaidMessageFileNameFormat ?? ""; string paidMsgPath = configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null && messageInfo.Id != 0 && messageInfo.CreatedAt is not null ? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" : "/Messages/Paid"; object? messageAuthor = messageInfo?.FromUser ?? (object?)messageInfo?.Author; if (kvpEntry.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = kvpEntry.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, kvpEntry.Key, "Messages", progressReporter, paidMsgPath + "/Videos", filenameFormat, messageInfo, mediaInfo, messageAuthor, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(kvpEntry.Value, path, kvpEntry.Key, "Messages", progressReporter, paidMsgPath, filenameFormat, messageInfo, mediaInfo, messageAuthor, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Paid Messages Media Already Downloaded: {oldCount} New Paid Messages Media Downloaded: {newCount}"); return new DownloadResult { TotalCount = paidMessageCollection.PaidMessages.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Paid Messages", Success = true }; } /// /// Downloads stream posts for a creator. /// /// The creator username. /// The creator user ID. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The streams collection. /// Progress reporter. /// The download result. public async Task DownloadStreams(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, StreamEntities.StreamsCollection streams, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadStreams - {username}"); if (streams.Streams.Count == 0) { Log.Debug("Found 0 Streams"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Streams", Success = true }; } Log.Debug($"Found {streams.Streams.Count} Media from {streams.StreamObjects.Count} Streams"); int oldCount = 0; int newCount = 0; foreach (KeyValuePair kvpEntry in streams.Streams) { bool isNew; StreamEntities.Medium? mediaInfo = streams.StreamMedia.FirstOrDefault(m => m.Id == kvpEntry.Key); StreamEntities.ListItem? streamInfo = mediaInfo == null ? null : streams.StreamObjects.FirstOrDefault(p => p.Media?.Contains(mediaInfo) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? ""; string streamPath = configService.CurrentConfig.FolderPerPost && streamInfo != null && streamInfo.Id != 0 ? $"/Posts/Free/{streamInfo.Id} {streamInfo.PostedAt:yyyy-MM-dd HH-mm-ss}" : "/Posts/Free"; if (kvpEntry.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = kvpEntry.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, kvpEntry.Key, "Streams", progressReporter, streamPath + "/Videos", filenameFormat, streamInfo, mediaInfo, streamInfo?.Author, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(kvpEntry.Value, path, kvpEntry.Key, "Streams", progressReporter, streamPath, filenameFormat, streamInfo, mediaInfo, streamInfo?.Author, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Streams Media Already Downloaded: {oldCount} New Streams Media Downloaded: {newCount}"); return new DownloadResult { TotalCount = streams.Streams.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Streams", Success = true }; } /// /// Downloads free posts for a creator. /// /// The creator username. /// The creator user ID. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The posts collection. /// Progress reporter. /// The download result. public async Task DownloadFreePosts(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PostEntities.PostCollection posts, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadFreePosts - {username}"); if (posts.Posts.Count == 0) { Log.Debug("Found 0 Posts"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Posts", Success = true }; } Log.Debug($"Found {posts.Posts.Count} Media from {posts.PostObjects.Count} Posts"); int oldCount = 0, newCount = 0; foreach (KeyValuePair postKvp in posts.Posts) { bool isNew; PostEntities.Medium? mediaInfo = posts.PostMedia.FirstOrDefault(m => m.Id == postKvp.Key); PostEntities.ListItem? postInfo = mediaInfo == null ? null : posts.PostObjects.FirstOrDefault(p => p.Media?.Contains(mediaInfo) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? ""; string postPath = configService.CurrentConfig.FolderPerPost && postInfo != null && postInfo.Id != 0 ? $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}" : "/Posts/Free"; if (postKvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = postKvp.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKvp.Key, "Posts", progressReporter, postPath + "/Videos", filenameFormat, postInfo, mediaInfo, postInfo?.Author, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(postKvp.Value, path, postKvp.Key, "Posts", progressReporter, postPath, filenameFormat, postInfo, mediaInfo, postInfo?.Author, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Posts Media Already Downloaded: {oldCount} New Posts Media Downloaded: {newCount}"); return new DownloadResult { TotalCount = posts.Posts.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Posts", Success = true }; } /// /// Downloads paid posts for a creator. /// /// The creator username. /// The creator user ID. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The paid post collection. /// Progress reporter. /// The download result. public async Task DownloadPaidPosts(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PurchasedEntities.PaidPostCollection purchasedPosts, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadPaidPosts - {username}"); if (purchasedPosts.PaidPosts.Count == 0) { Log.Debug("Found 0 Paid Posts"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Paid Posts", Success = true }; } Log.Debug( $"Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts"); int oldCount = 0, newCount = 0; foreach (KeyValuePair postKvp in purchasedPosts.PaidPosts) { bool isNew; MessageEntities.Medium? mediaInfo = purchasedPosts.PaidPostMedia.FirstOrDefault(m => m.Id == postKvp.Key); PurchasedEntities.ListItem? postInfo = purchasedPosts.PaidPostObjects.FirstOrDefault(p => p.Media?.Any(m => m.Id == postKvp.Key) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PaidPostFileNameFormat ?? ""; string paidPostPath = configService.CurrentConfig.FolderPerPaidPost && postInfo != null && postInfo.Id != 0 && postInfo.PostedAt is not null ? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}" : "/Posts/Paid"; object? postAuthor = postInfo?.FromUser ?? (object?)postInfo?.Author; if (postKvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = postKvp.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKvp.Key, "Posts", progressReporter, paidPostPath + "/Videos", filenameFormat, postInfo, mediaInfo, postAuthor, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(postKvp.Value, path, postKvp.Key, "Posts", progressReporter, paidPostPath, filenameFormat, postInfo, mediaInfo, postAuthor, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Paid Posts Media Already Downloaded: {oldCount} New Paid Posts Media Downloaded: {newCount}"); return new DownloadResult { TotalCount = purchasedPosts.PaidPosts.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Paid Posts", Success = true }; } /// /// Downloads paid posts sourced from the Purchased tab. /// /// The creator username. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The paid post collection. /// Progress reporter. /// The download result. public async Task DownloadPaidPostsPurchasedTab(string username, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PurchasedEntities.PaidPostCollection purchasedPosts, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadPaidPostsPurchasedTab - {username}"); if (purchasedPosts.PaidPosts.Count == 0) { Log.Debug("Found 0 Paid Posts"); return new DownloadResult { TotalCount = 0, MediaType = "Paid Posts", Success = true }; } int oldCount = 0, newCount = 0; foreach (KeyValuePair purchasedPostKvp in purchasedPosts.PaidPosts) { bool isNew; MessageEntities.Medium? mediaInfo = purchasedPosts?.PaidPostMedia?.FirstOrDefault(m => m.Id == purchasedPostKvp.Key); PurchasedEntities.ListItem? postInfo = mediaInfo != null ? purchasedPosts?.PaidPostObjects?.FirstOrDefault(p => p.Media?.Any(m => m.Id == purchasedPostKvp.Key) == true) : null; string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) .PaidPostFileNameFormat ?? ""; string paidPostPath = configService.CurrentConfig.FolderPerPaidPost && postInfo != null && postInfo.Id != 0 && postInfo.PostedAt is not null ? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}" : "/Posts/Paid"; object? postAuthor = postInfo?.FromUser ?? (object?)postInfo?.Author; if (purchasedPostKvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = purchasedPostKvp.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, purchasedPostKvp.Key, "Posts", progressReporter, paidPostPath + "/Videos", filenameFormat, postInfo, mediaInfo, postAuthor, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(purchasedPostKvp.Value, path, purchasedPostKvp.Key, "Posts", progressReporter, paidPostPath, filenameFormat, postInfo, mediaInfo, postAuthor, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Paid Posts Media Already Downloaded: {oldCount} New Paid Posts Media Downloaded: {newCount}"); return new DownloadResult { TotalCount = purchasedPosts?.PaidPosts.Count ?? 0, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Paid Posts", Success = true }; } /// /// Downloads paid messages sourced from the Purchased tab. /// /// The creator username. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The paid message collection. /// Progress reporter. /// The download result. public async Task DownloadPaidMessagesPurchasedTab(string username, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PurchasedEntities.PaidMessageCollection paidMessageCollection, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadPaidMessagesPurchasedTab - {username}"); if (paidMessageCollection.PaidMessages.Count == 0) { Log.Debug("Found 0 Paid Messages"); return new DownloadResult { TotalCount = 0, MediaType = "Paid Messages", Success = true }; } int oldCount = 0, newCount = 0; foreach (KeyValuePair paidMessageKvp in paidMessageCollection.PaidMessages) { bool isNew; MessageEntities.Medium? mediaInfo = paidMessageCollection.PaidMessageMedia.FirstOrDefault(m => m.Id == paidMessageKvp.Key); PurchasedEntities.ListItem? messageInfo = paidMessageCollection.PaidMessageObjects.FirstOrDefault(p => p.Media?.Any(m => m.Id == paidMessageKvp.Key) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) .PaidMessageFileNameFormat ?? ""; string paidMsgPath = configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null && messageInfo.Id != 0 && messageInfo.CreatedAt is not null ? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" : "/Messages/Paid"; object? messageAuthor = messageInfo?.FromUser ?? (object?)messageInfo?.Author; if (paidMessageKvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = paidMessageKvp.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKvp.Key, "Messages", progressReporter, paidMsgPath + "/Videos", filenameFormat, messageInfo, mediaInfo, messageAuthor, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(paidMessageKvp.Value, path, paidMessageKvp.Key, "Messages", progressReporter, paidMsgPath, filenameFormat, messageInfo, mediaInfo, messageAuthor, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Paid Messages Media Already Downloaded: {oldCount} New Paid Messages Media Downloaded: {newCount}"); return new DownloadResult { TotalCount = paidMessageCollection.PaidMessages.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Paid Messages", Success = true }; } /// /// Downloads a single post collection. /// /// The creator username. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The single post collection. /// Progress reporter. /// The download result. public async Task DownloadSinglePost(string username, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PostEntities.SinglePostCollection post, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadSinglePost - {username}"); if (post.SinglePosts.Count == 0) { Log.Debug("Couldn't find post"); return new DownloadResult { TotalCount = 0, MediaType = "Posts", Success = true }; } int oldCount = 0, newCount = 0; bool hasPaidPostMedia = false; foreach (KeyValuePair postKvp in post.SinglePosts) { PostEntities.Medium? mediaInfo = post.SinglePostMedia.FirstOrDefault(m => m.Id == postKvp.Key); PostEntities.SinglePost? postInfo = mediaInfo == null ? null : post.SinglePostObjects.FirstOrDefault(p => p.Media?.Contains(mediaInfo) == true); bool isPaidPost = IsPaidSinglePost(postInfo); if (isPaidPost) { hasPaidPostMedia = true; } string filenameFormat = hasPaidPostMedia ? configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PaidPostFileNameFormat ?? "" : configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? ""; string postPath = hasPaidPostMedia ? configService.CurrentConfig.FolderPerPaidPost && postInfo != null && postInfo.Id != 0 ? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}" : "/Posts/Paid" : configService.CurrentConfig.FolderPerPost && postInfo != null && postInfo.Id != 0 ? $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}" : "/Posts/Free"; bool isNew; if (postKvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = postKvp.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKvp.Key, "Posts", progressReporter, postPath + "/Videos", filenameFormat, postInfo, mediaInfo, postInfo?.Author, users, drmInfo.Value.mpdDurationSeconds); } else { try { isNew = await DownloadMedia(postKvp.Value, path, postKvp.Key, "Posts", progressReporter, postPath, filenameFormat, postInfo, mediaInfo, postInfo?.Author, users); } catch { Log.Warning("Media was null"); continue; } } if (isNew) { newCount++; } else { oldCount++; } } return new DownloadResult { TotalCount = post.SinglePosts.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = hasPaidPostMedia ? "Paid Posts" : "Posts", Success = true }; } private static bool IsPaidSinglePost(PostEntities.SinglePost? postInfo) { if (postInfo == null || !postInfo.IsOpened) { return false; } if (string.IsNullOrWhiteSpace(postInfo.Price)) { return false; } string normalizedPrice = postInfo.Price.Trim(); if (decimal.TryParse(normalizedPrice, NumberStyles.Any, CultureInfo.InvariantCulture, out decimal amount)) { return amount > 0; } return !string.Equals(normalizedPrice, "0", StringComparison.OrdinalIgnoreCase) && !string.Equals(normalizedPrice, "0.0", StringComparison.OrdinalIgnoreCase) && !string.Equals(normalizedPrice, "0.00", StringComparison.OrdinalIgnoreCase); } /// /// Downloads a single paid message collection (including previews). /// /// The creator username. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// The single paid message collection. /// Progress reporter. /// The download result. public async Task DownloadSinglePaidMessage(string username, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection, IProgressReporter progressReporter) { Log.Debug("Calling DownloadSinglePaidMessage - {Username}", username); int totalNew = 0, totalOld = 0; // Download preview messages if (singlePaidMessageCollection.PreviewSingleMessages.Count > 0) { foreach (KeyValuePair paidMessageKvp in singlePaidMessageCollection.PreviewSingleMessages) { MessageEntities.Medium? mediaInfo = singlePaidMessageCollection.PreviewSingleMessageMedia.FirstOrDefault(m => m.Id == paidMessageKvp.Key); MessageEntities.SingleMessage? messageInfo = singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p => p.Media?.Any(m => m.Id == paidMessageKvp.Key) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) .PaidMessageFileNameFormat ?? ""; string previewMsgPath = configService.CurrentConfig.FolderPerMessage && messageInfo != null && messageInfo.Id != 0 && messageInfo.CreatedAt is not null ? $"/Messages/Free/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" : "/Messages/Free"; bool isNew; if (paidMessageKvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = paidMessageKvp.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKvp.Key, "Messages", progressReporter, previewMsgPath + "/Videos", filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(paidMessageKvp.Value, path, paidMessageKvp.Key, "Messages", progressReporter, previewMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users); } if (isNew) { totalNew++; } else { totalOld++; } } } // Download actual paid messages if (singlePaidMessageCollection.SingleMessages.Count > 0) { foreach (KeyValuePair paidMessageKvp in singlePaidMessageCollection.SingleMessages) { MessageEntities.Medium? mediaInfo = singlePaidMessageCollection.SingleMessageMedia.FirstOrDefault(m => m.Id == paidMessageKvp.Key); MessageEntities.SingleMessage? messageInfo = singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p => p.Media?.Any(m => m.Id == paidMessageKvp.Key) == true); string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username) .PaidMessageFileNameFormat ?? ""; string singlePaidMsgPath = configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null && messageInfo.Id != 0 && messageInfo.CreatedAt is not null ? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}" : "/Messages/Paid"; bool isNew; if (paidMessageKvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = paidMessageKvp.Value.Split(','); (string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3], parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing); if (drmInfo == null) { continue; } isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0], drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKvp.Key, "Messages", progressReporter, singlePaidMsgPath + "/Videos", filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users, drmInfo.Value.mpdDurationSeconds); } else { isNew = await DownloadMedia(paidMessageKvp.Value, path, paidMessageKvp.Key, "Messages", progressReporter, singlePaidMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users); } if (isNew) { totalNew++; } else { totalOld++; } } } int totalCount = singlePaidMessageCollection.PreviewSingleMessages.Count + singlePaidMessageCollection.SingleMessages.Count; Log.Debug($"Paid Messages Media Already Downloaded: {totalOld} New Paid Messages Media Downloaded: {totalNew}"); return new DownloadResult { TotalCount = totalCount, NewDownloads = totalNew, ExistingDownloads = totalOld, MediaType = "Paid Messages", Success = true }; } }