using System.Security.Cryptography; using System.Text.RegularExpressions; using System.Xml.Linq; using FFmpeg.NET; using FFmpeg.NET.Events; using OF_DL.Models; using OF_DL.Enumerations; using ArchivedEntities = OF_DL.Models.Entities.Archived; using CommonEntities = OF_DL.Models.Entities.Common; using MessageEntities = OF_DL.Models.Entities.Messages; using PostEntities = OF_DL.Models.Entities.Posts; using PurchasedEntities = OF_DL.Models.Entities.Purchased; using StreamEntities = OF_DL.Models.Entities.Streams; using OF_DL.Utils; using Serilog; using Serilog.Events; namespace OF_DL.Services; public class DownloadService( IAuthService authService, IConfigService configService, IDBService dbService, IFileNameService fileNameService, IAPIService apiService) : IDownloadService { private TaskCompletionSource _completionSource; public async Task DownloadAvatarHeader(string? avatarUrl, string? headerUrl, string folder, string username) { try { string path = "/Profile"; if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(avatarUrl)) { string avatarpath = $"{path}/Avatars"; if (!Directory.Exists(folder + avatarpath)) { Directory.CreateDirectory(folder + avatarpath); } List avatarMD5Hashes = CalculateFolderMD5(folder + avatarpath); Uri uri = new(avatarUrl); string destinationPath = $"{folder}{avatarpath}/"; HttpClient client = new(); HttpRequestMessage request = new() { Method = HttpMethod.Get, RequestUri = uri }; using HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); using MemoryStream memoryStream = new(); await response.Content.CopyToAsync(memoryStream); memoryStream.Seek(0, SeekOrigin.Begin); MD5 md5 = MD5.Create(); byte[] hash = md5.ComputeHash(memoryStream); memoryStream.Seek(0, SeekOrigin.Begin); if (!avatarMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant())) { destinationPath = destinationPath + string.Format("{0} {1}.jpg", username, response.Content.Headers.LastModified.HasValue ? response.Content.Headers.LastModified.Value.LocalDateTime.ToString("dd-MM-yyyy") : DateTime.Now.ToString("dd-MM-yyyy")); using (FileStream fileStream = File.Create(destinationPath)) { await memoryStream.CopyToAsync(fileStream); } File.SetLastWriteTime(destinationPath, response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now); } } if (!string.IsNullOrEmpty(headerUrl)) { string headerpath = $"{path}/Headers"; if (!Directory.Exists(folder + headerpath)) { Directory.CreateDirectory(folder + headerpath); } List headerMD5Hashes = CalculateFolderMD5(folder + headerpath); Uri uri = new(headerUrl); string destinationPath = $"{folder}{headerpath}/"; HttpClient client = new(); HttpRequestMessage request = new() { Method = HttpMethod.Get, RequestUri = uri }; using HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); using MemoryStream memoryStream = new(); await response.Content.CopyToAsync(memoryStream); memoryStream.Seek(0, SeekOrigin.Begin); MD5 md5 = MD5.Create(); byte[] hash = md5.ComputeHash(memoryStream); memoryStream.Seek(0, SeekOrigin.Begin); if (!headerMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant())) { destinationPath = destinationPath + string.Format("{0} {1}.jpg", username, response.Content.Headers.LastModified.HasValue ? response.Content.Headers.LastModified.Value.LocalDateTime.ToString("dd-MM-yyyy") : DateTime.Now.ToString("dd-MM-yyyy")); using (FileStream fileStream = File.Create(destinationPath)) { await memoryStream.CopyToAsync(fileStream); } File.SetLastWriteTime(destinationPath, response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now); } } } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } } #region drm common private async Task DownloadDrmMedia(string user_agent, string policy, string signature, string kvp, string sess, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string customFileName, string filename, string path) { try { _completionSource = new TaskCompletionSource(); int pos1 = decryptionKey.IndexOf(':'); string decKey = ""; if (pos1 >= 0) { decKey = decryptionKey.Substring(pos1 + 1); } int streamIndex = 0; string tempFilename = $"{folder}{path}/{filename}_source.mp4"; //int? streamIndex = await GetVideoStreamIndexFromMpd(url, policy, signature, kvp, downloadConfig.DownloadVideoResolution); //if (streamIndex == null) // throw new Exception($"Could not find video stream for resolution {downloadConfig.DownloadVideoResolution}"); //string tempFilename; //switch (downloadConfig.DownloadVideoResolution) //{ // case VideoResolution.source: // tempFilename = $"{folder}{path}/{filename}_source.mp4"; // break; // case VideoResolution._240: // tempFilename = $"{folder}{path}/{filename}_240.mp4"; // break; // case VideoResolution._720: // tempFilename = $"{folder}{path}/{filename}_720.mp4"; // break; // default: // tempFilename = $"{folder}{path}/{filename}_source.mp4"; // break; //} // Configure ffmpeg log level and optional report file location bool ffmpegDebugLogging = Log.IsEnabled(LogEventLevel.Debug); string logLevelArgs = ffmpegDebugLogging || configService.CurrentConfig.LoggingLevel is LoggingLevel.Verbose or LoggingLevel.Debug ? "-loglevel debug -report" : configService.CurrentConfig.LoggingLevel switch { LoggingLevel.Information => "-loglevel info", LoggingLevel.Warning => "-loglevel warning", LoggingLevel.Error => "-loglevel error", LoggingLevel.Fatal => "-loglevel fatal", _ => "" }; if (logLevelArgs.Contains("-report", StringComparison.OrdinalIgnoreCase)) { // Direct ffmpeg report files into the same logs directory Serilog uses (relative to current working directory) string logDir = Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, "logs")); Directory.CreateDirectory(logDir); string ffReportPath = Path.Combine(logDir, "ffmpeg-%p-%t.log"); // ffmpeg will replace %p/%t Environment.SetEnvironmentVariable("FFREPORT", $"file={ffReportPath}:level=32"); Log.Debug("FFREPORT enabled at: {FFREPORT} (cwd: {Cwd})", Environment.GetEnvironmentVariable("FFREPORT"), Environment.CurrentDirectory); } else { Environment.SetEnvironmentVariable("FFREPORT", null); Log.Debug("FFREPORT disabled (cwd: {Cwd})", Environment.CurrentDirectory); } string cookieHeader = "Cookie: " + $"CloudFront-Policy={policy}; " + $"CloudFront-Signature={signature}; " + $"CloudFront-Key-Pair-Id={kvp}; " + $"{sess}"; string parameters = $"{logLevelArgs} " + $"-cenc_decryption_key {decKey} " + $"-headers \"{cookieHeader}\" " + $"-user_agent \"{user_agent}\" " + "-referer \"https://onlyfans.com\" " + "-rw_timeout 20000000 " + "-reconnect 1 -reconnect_streamed 1 -reconnect_on_network_error 1 -reconnect_delay_max 10 " + "-y " + $"-i \"{url}\" " + $"-map 0:v:{streamIndex} -map 0:a? " + "-c copy " + $"\"{tempFilename}\""; Log.Debug($"Calling FFMPEG with Parameters: {parameters}"); Engine ffmpeg = new(configService.CurrentConfig.FFmpegPath); ffmpeg.Error += OnError; ffmpeg.Complete += async (sender, args) => { _completionSource.TrySetResult(true); await OnFFMPEGDownloadComplete(tempFilename, lastModified, folder, path, customFileName, filename, media_id, api_type, progressReporter); }; await ffmpeg.ExecuteAsync(parameters, CancellationToken.None); return await _completionSource.Task; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } #endregion private async Task OnFFMPEGDownloadComplete(string tempFilename, DateTime lastModified, string folder, string path, string customFileName, string filename, long media_id, string api_type, IProgressReporter progressReporter) { try { if (File.Exists(tempFilename)) { File.SetLastWriteTime(tempFilename, lastModified); } if (!string.IsNullOrEmpty(customFileName)) { File.Move(tempFilename, $"{folder + path + "/" + customFileName + ".mp4"}"); } // Cleanup Files long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : tempFilename).Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + ".mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } } private void OnError(object sender, ConversionErrorEventArgs e) { // Guard all fields to avoid NullReference exceptions from FFmpeg.NET string input = e?.Input?.Name ?? ""; string output = e?.Output?.Name ?? ""; string exitCode = e?.Exception?.ExitCode.ToString() ?? ""; string message = e?.Exception?.Message ?? ""; string inner = e?.Exception?.InnerException?.Message ?? ""; Log.Error("FFmpeg failed. Input={Input} Output={Output} ExitCode={ExitCode} Message={Message} Inner={Inner}", input, output, exitCode, message, inner); _completionSource?.TrySetResult(false); } private async Task GetVideoStreamIndexFromMpd(string mpdUrl, string policy, string signature, string kvp, VideoResolution resolution) { HttpClient client = new(); HttpRequestMessage request = new(HttpMethod.Get, mpdUrl); request.Headers.Add("user-agent", authService.CurrentAuth.USER_AGENT); request.Headers.Add("Accept", "*/*"); request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {authService.CurrentAuth.COOKIE};"); using (HttpResponseMessage response = await client.SendAsync(request)) { response.EnsureSuccessStatusCode(); string body = await response.Content.ReadAsStringAsync(); XDocument doc = XDocument.Parse(body); XNamespace ns = "urn:mpeg:dash:schema:mpd:2011"; XNamespace cenc = "urn:mpeg:cenc:2013"; XElement? videoAdaptationSet = doc .Descendants(ns + "AdaptationSet") .FirstOrDefault(e => (string)e.Attribute("mimeType") == "video/mp4"); if (videoAdaptationSet == null) { return null; } string targetHeight = resolution switch { VideoResolution._240 => "240", VideoResolution._720 => "720", VideoResolution.source => "1280", _ => throw new ArgumentOutOfRangeException(nameof(resolution)) }; List representations = videoAdaptationSet.Elements(ns + "Representation").ToList(); for (int i = 0; i < representations.Count; i++) { if ((string)representations[i].Attribute("height") == targetHeight) { return i; // this is the index FFmpeg will use for `-map 0:v:{i}` } } } return null; } private static List CalculateFolderMD5(string folder) { List md5Hashes = new(); if (Directory.Exists(folder)) { string[] files = Directory.GetFiles(folder); foreach (string file in files) { md5Hashes.Add(CalculateMD5(file)); } } return md5Hashes; } private static string CalculateMD5(string filePath) { using (MD5 md5 = MD5.Create()) { using (FileStream stream = File.OpenRead(filePath)) { byte[] hash = md5.ComputeHash(stream); return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); } } } #region common /// /// /// /// /// /// /// /// /// /// /// /// /// protected async Task CreateDirectoriesAndDownloadMedia(string path, string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string serverFileName, string resolvedFileName) { try { string customFileName = ""; if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } string extension = Path.GetExtension(url.Split("?")[0]); path = UpdatePathBasedOnExtension(folder, path, extension); return await ProcessMediaDownload(folder, media_id, api_type, url, path, serverFileName, resolvedFileName, extension, progressReporter); } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } /// /// Updates the given path based on the file extension. /// /// The parent folder. /// The initial relative path. /// The file extension. /// A string that represents the updated path based on the file extension. private string UpdatePathBasedOnExtension(string folder, string path, string extension) { string subdirectory = ""; switch (extension.ToLower()) { case ".jpg": case ".jpeg": case ".png": subdirectory = "/Images"; break; case ".mp4": case ".avi": case ".wmv": case ".gif": case ".mov": subdirectory = "/Videos"; break; case ".mp3": case ".wav": case ".ogg": subdirectory = "/Audios"; break; } if (!string.IsNullOrEmpty(subdirectory)) { path += subdirectory; string fullPath = folder + path; if (!Directory.Exists(fullPath)) { Directory.CreateDirectory(fullPath); } } return path; } /// /// Generates a custom filename based on the given format and properties. /// /// The format string for the filename. /// General information about the post. /// Media associated with the post. /// Author of the post. /// Dictionary containing user-related data. /// Helper class for filename operations. /// A Task resulting in a string that represents the custom filename. private async Task GenerateCustomFileName(string filename, string? filenameFormat, object? postInfo, object? postMedia, object? author, string username, Dictionary users, IFileNameService fileNameService, CustomFileNameOption option) { if (string.IsNullOrEmpty(filenameFormat) || postInfo == null || postMedia == null || author == null) { return option switch { CustomFileNameOption.ReturnOriginal => filename, CustomFileNameOption.ReturnEmpty => "", _ => filename }; } List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); properties.AddRange(matches.Select(match => match.Groups[1].Value)); Dictionary values = await fileNameService.GetFilename(postInfo, postMedia, author, properties, username, users); return await fileNameService.BuildFilename(filenameFormat, values); } private async Task GetFileSizeAsync(string url) { long fileSize = 0; try { Uri uri = new(url); if (uri.Host == "cdn3.onlyfans.com" && uri.LocalPath.Contains("/dash/files")) { string[] messageUrlParsed = url.Split(','); string mpdURL = messageUrlParsed[0]; string policy = messageUrlParsed[1]; string signature = messageUrlParsed[2]; string kvp = messageUrlParsed[3]; mpdURL = mpdURL.Replace(".mpd", "_source.mp4"); using HttpClient client = new(); client.DefaultRequestHeaders.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {authService.CurrentAuth.COOKIE}"); client.DefaultRequestHeaders.Add("User-Agent", authService.CurrentAuth.USER_AGENT); using HttpResponseMessage response = await client.GetAsync(mpdURL, HttpCompletionOption.ResponseHeadersRead); if (response.IsSuccessStatusCode) { fileSize = response.Content.Headers.ContentLength ?? 0; } } else { using HttpClient client = new(); client.DefaultRequestHeaders.Add("User-Agent", authService.CurrentAuth.USER_AGENT); using HttpResponseMessage response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead); if (response.IsSuccessStatusCode) { fileSize = response.Content.Headers.ContentLength ?? 0; } } } catch (Exception ex) { Console.WriteLine($"Error getting file size for URL '{url}': {ex.Message}"); } return fileSize; } public static async Task GetDRMVideoLastModified(string url, Auth auth) { Uri uri = new(url); string[] messageUrlParsed = url.Split(','); string mpdURL = messageUrlParsed[0]; string policy = messageUrlParsed[1]; string signature = messageUrlParsed[2]; string kvp = messageUrlParsed[3]; mpdURL = mpdURL.Replace(".mpd", "_source.mp4"); using HttpClient client = new(); client.DefaultRequestHeaders.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.COOKIE}"); client.DefaultRequestHeaders.Add("User-Agent", auth.USER_AGENT); using HttpResponseMessage response = await client.GetAsync(mpdURL, HttpCompletionOption.ResponseHeadersRead); if (response.IsSuccessStatusCode) { return response.Content.Headers.LastModified.Value.DateTime; } return DateTime.Now; } public static async Task GetMediaLastModified(string url) { using HttpClient client = new(); using HttpResponseMessage response = await client.GetAsync(url, HttpCompletionOption.ResponseHeadersRead); if (response.IsSuccessStatusCode) { return response.Content.Headers.LastModified.Value.DateTime; } return DateTime.Now; } /// /// Processes the download and database update of media. /// /// The folder where the media is stored. /// The ID of the media. /// The full path to the media. /// The URL from where to download the media. /// The relative path to the media. /// The filename after any required manipulations. /// The file extension. /// The task object for tracking progress. /// A Task resulting in a boolean indicating whether the media is newly downloaded or not. public async Task ProcessMediaDownload(string folder, long media_id, string api_type, string url, string path, string serverFilename, string resolvedFilename, string extension, IProgressReporter progressReporter) { try { if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { return await HandleNewMedia(folder, media_id, api_type, url, path, serverFilename, resolvedFilename, extension, progressReporter); } bool status = await HandlePreviouslyDownloadedMediaAsync(folder, media_id, api_type, progressReporter); if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && serverFilename != resolvedFilename) { await HandleRenamingOfExistingFilesAsync(folder, media_id, api_type, path, serverFilename, resolvedFilename, extension); } return status; } catch (Exception ex) { // Handle exception (e.g., log it) Console.WriteLine($"An error occurred: {ex.Message}"); return false; } } private async Task HandleRenamingOfExistingFilesAsync(string folder, long media_id, string api_type, string path, string serverFilename, string resolvedFilename, string extension) { string fullPathWithTheServerFileName = $"{folder}{path}/{serverFilename}{extension}"; string fullPathWithTheNewFileName = $"{folder}{path}/{resolvedFilename}{extension}"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); DateTime lastModified = File.GetLastWriteTime(fullPathWithTheNewFileName); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, resolvedFilename + extension, size, true, lastModified); return true; } /// /// Handles new media by downloading and updating the database. /// /// /// /// /// /// /// /// /// /// A Task resulting in a boolean indicating whether the media is newly downloaded or not. private async Task HandleNewMedia(string folder, long media_id, string api_type, string url, string path, string serverFilename, string resolvedFilename, string extension, IProgressReporter progressReporter) { long fileSizeInBytes; DateTime lastModified; bool status; string fullPathWithTheServerFileName = $"{folder}{path}/{serverFilename}{extension}"; string fullPathWithTheNewFileName = $"{folder}{path}/{resolvedFilename}{extension}"; //there are a few possibilities here. //1.file has been downloaded in the past but it has the server filename // in that case it should be set as existing and it should be renamed //2.file has been downloaded in the past but it has custom filename. // it should be set as existing and nothing else. // of coures 1 and 2 depends in the fact that there may be a difference in the resolved file name // (ie user has selected a custom format. If he doesn't then the resolved name will be the same as the server filename //3.file doesn't exist and it should be downloaded. // Handle the case where the file has been downloaded in the past with the server filename //but it has downloaded outsite of this application so it doesn't exist in the database if (File.Exists(fullPathWithTheServerFileName)) { string finalPath; if (fullPathWithTheServerFileName != fullPathWithTheNewFileName) { finalPath = fullPathWithTheNewFileName; //rename. try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); } } else { finalPath = fullPathWithTheServerFileName; } fileSizeInBytes = GetLocalFileSize(finalPath); lastModified = File.GetLastWriteTime(finalPath); if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } status = false; } // Handle the case where the file has been downloaded in the past with a custom filename. //but it has downloaded outsite of this application so it doesn't exist in the database // this is a bit improbable but we should check for that. else if (File.Exists(fullPathWithTheNewFileName)) { fileSizeInBytes = GetLocalFileSize(fullPathWithTheNewFileName); lastModified = File.GetLastWriteTime(fullPathWithTheNewFileName); if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } status = false; } else //file doesn't exist and we should download it. { lastModified = await DownloadFile(url, fullPathWithTheNewFileName, progressReporter); fileSizeInBytes = GetLocalFileSize(fullPathWithTheNewFileName); status = true; } //finaly check which filename we should use. Custom or the server one. //if a custom is used, then the servefilename will be different from the resolved filename. string finalName = serverFilename == resolvedFilename ? serverFilename : resolvedFilename; await dbService.UpdateMedia(folder, media_id, api_type, folder + path, finalName + extension, fileSizeInBytes, true, lastModified); return status; } /// /// Handles media that has been previously downloaded and updates the task accordingly. /// /// /// /// /// /// A boolean indicating whether the media is newly downloaded or not. private async Task HandlePreviouslyDownloadedMediaAsync(string folder, long media_id, string api_type, IProgressReporter progressReporter) { if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } return false; } /// /// Gets the file size of the media. /// /// The path to the file. /// The file size in bytes. private long GetLocalFileSize(string filePath) => new FileInfo(filePath).Length; /// /// Downloads a file from the given URL and saves it to the specified destination path. /// /// The URL to download the file from. /// The path where the downloaded file will be saved. /// Progress tracking object. /// A Task resulting in a DateTime indicating the last modified date of the downloaded file. private async Task DownloadFile(string url, string destinationPath, IProgressReporter progressReporter) { using HttpClient client = new(); HttpRequestMessage request = new() { Method = HttpMethod.Get, RequestUri = new Uri(url) }; using HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); Stream body = await response.Content.ReadAsStreamAsync(); // Wrap the body stream with the ThrottledStream to limit read rate. using (ThrottledStream throttledStream = new(body, configService.CurrentConfig.DownloadLimitInMbPerSec * 1_000_000, configService.CurrentConfig.LimitDownloadRate)) { using FileStream fileStream = new(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, 16384, true); byte[] buffer = new byte[16384]; int read; while ((read = await throttledStream.ReadAsync(buffer, CancellationToken.None)) > 0) { if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(read); } await fileStream.WriteAsync(buffer.AsMemory(0, read), CancellationToken.None); } } File.SetLastWriteTime(destinationPath, response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now); if (!configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(1); } return response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now; } public async Task CalculateTotalFileSize(List urls) { long totalFileSize = 0; if (urls.Count > 250) { int batchSize = 250; List> tasks = new(); for (int i = 0; i < urls.Count; i += batchSize) { List batchUrls = urls.Skip(i).Take(batchSize).ToList(); IEnumerable> batchTasks = batchUrls.Select(GetFileSizeAsync); tasks.AddRange(batchTasks); await Task.WhenAll(batchTasks); await Task.Delay(5000); } long[] fileSizes = await Task.WhenAll(tasks); foreach (long fileSize in fileSizes) { totalFileSize += fileSize; } } else { List> tasks = new(); foreach (string url in urls) { tasks.Add(GetFileSizeAsync(url)); } long[] fileSizes = await Task.WhenAll(tasks); foreach (long fileSize in fileSizes) { totalFileSize += fileSize; } } return totalFileSize; } #endregion #region normal posts public async Task DownloadPostMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, PostEntities.ListItem? postInfo, PostEntities.Medium? postMedia, CommonEntities.Author? author, Dictionary users) { string path; if (configService.CurrentConfig.FolderPerPost && postInfo != null && postInfo?.Id is not null && postInfo?.PostedAt is not null) { path = $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}"; } else { path = "/Posts/Free"; } Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, postInfo, postMedia, author, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, resolvedFilename); } public async Task DownloadPostMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, PostEntities.SinglePost? postInfo, PostEntities.Medium? postMedia, CommonEntities.Author? author, Dictionary users) { string path; if (configService.CurrentConfig.FolderPerPost && postInfo != null && postInfo?.Id is not null && postInfo?.PostedAt is not null) { path = $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}"; } else { path = "/Posts/Free"; } Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, postInfo, postMedia, author, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, resolvedFilename); } public async Task DownloadStreamMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, StreamEntities.ListItem? streamInfo, StreamEntities.Medium? streamMedia, CommonEntities.Author? author, Dictionary users) { string path; if (configService.CurrentConfig.FolderPerPost && streamInfo != null && streamInfo?.Id is not null && streamInfo?.PostedAt is not null) { path = $"/Posts/Free/{streamInfo.Id} {streamInfo.PostedAt:yyyy-MM-dd HH-mm-ss}"; } else { path = "/Posts/Free"; } Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, streamInfo, streamMedia, author, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, resolvedFilename); } public async Task DownloadMessageMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, MessageEntities.ListItem? messageInfo, MessageEntities.Medium? messageMedia, CommonEntities.FromUser? fromUser, Dictionary users) { string path; if (configService.CurrentConfig.FolderPerMessage && messageInfo != null && messageInfo?.Id is not null && messageInfo?.CreatedAt is not null) { path = $"/Messages/Free/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}"; } else { path = "/Messages/Free"; } Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, messageInfo, messageMedia, fromUser, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, resolvedFilename); } public async Task DownloadMessagePreviewMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, MessageEntities.SingleMessage? messageInfo, MessageEntities.Medium? messageMedia, CommonEntities.FromUser? fromUser, Dictionary users) { string path; if (configService.CurrentConfig.FolderPerMessage && messageInfo != null && messageInfo?.Id is not null && messageInfo?.CreatedAt is not null) { path = $"/Messages/Free/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}"; } else { path = "/Messages/Free"; } Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, messageInfo, messageMedia, fromUser, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, resolvedFilename); } public async Task DownloadArchivedMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, ArchivedEntities.ListItem? messageInfo, ArchivedEntities.Medium? messageMedia, CommonEntities.Author? author, Dictionary users) { string path = "/Archived/Posts/Free"; Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, messageInfo, messageMedia, author, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, resolvedFilename); } public async Task DownloadStoryMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter) { string path = "/Stories/Free"; Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, filename); } public async Task DownloadPurchasedMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, PurchasedEntities.ListItem? messageInfo, MessageEntities.Medium? messageMedia, CommonEntities.FromUser? fromUser, Dictionary users) { string path; if (configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null && messageInfo?.Id is not null && messageInfo?.CreatedAt is not null) { path = $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}"; } else { path = "/Messages/Paid"; } Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, messageInfo, messageMedia, fromUser, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, resolvedFilename); } public async Task DownloadSinglePurchasedMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, MessageEntities.SingleMessage? messageInfo, MessageEntities.Medium? messageMedia, CommonEntities.FromUser? fromUser, Dictionary users) { string path; if (configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null && messageInfo?.Id is not null && messageInfo?.CreatedAt is not null) { path = $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}"; } else { path = "/Messages/Paid"; } Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, messageInfo, messageMedia, fromUser, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, resolvedFilename); } public async Task DownloadPurchasedPostMedia(string url, string folder, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, PurchasedEntities.ListItem? messageInfo, MessageEntities.Medium? messageMedia, CommonEntities.FromUser? fromUser, Dictionary users) { string path; if (configService.CurrentConfig.FolderPerPaidPost && messageInfo != null && messageInfo?.Id is not null && messageInfo?.PostedAt is not null) { path = $"/Posts/Paid/{messageInfo.Id} {messageInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}"; } else { path = "/Posts/Paid"; } Uri uri = new(url); string filename = Path.GetFileNameWithoutExtension(uri.LocalPath); string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, messageInfo, messageMedia, fromUser, folder.Split("/")[^1], users, fileNameService, CustomFileNameOption.ReturnOriginal); return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, progressReporter, filename, resolvedFilename); } #endregion #region drm posts public async Task DownloadMessageDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, MessageEntities.ListItem? messageInfo, MessageEntities.Medium? messageMedia, CommonEntities.FromUser? fromUser, Dictionary users) { try { string customFileName = ""; string path; Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; if (configService.CurrentConfig.FolderPerMessage && messageInfo != null && messageInfo?.Id is not null && messageInfo?.CreatedAt is not null) { path = $"/Messages/Free/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}/Videos"; } else { path = "/Messages/Free/Videos"; } if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(filenameFormat) && messageInfo != null && messageMedia != null) { List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); foreach (Match match in matches) { properties.Add(match.Groups[1].Value); } Dictionary values = await fileNameService.GetFilename(messageInfo, messageMedia, fromUser, properties, folder.Split("/")[^1], users); customFileName = await fileNameService.BuildFilename(filenameFormat, values); } if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.USER_AGENT, policy, signature, kvp, authService.CurrentAuth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, progressReporter, customFileName, filename, path); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); } } if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } } return false; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } public async Task DownloadSingleMessagePreviewDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, MessageEntities.SingleMessage? messageInfo, MessageEntities.Medium? messageMedia, CommonEntities.FromUser? fromUser, Dictionary users) { try { string customFileName = ""; string path; Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; if (configService.CurrentConfig.FolderPerMessage && messageInfo != null && messageInfo?.Id is not null && messageInfo?.CreatedAt is not null) { path = $"/Messages/Free/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}/Videos"; } else { path = "/Messages/Free/Videos"; } if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(filenameFormat) && messageInfo != null && messageMedia != null) { List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); foreach (Match match in matches) { properties.Add(match.Groups[1].Value); } Dictionary values = await fileNameService.GetFilename(messageInfo, messageMedia, fromUser, properties, folder.Split("/")[^1], users); customFileName = await fileNameService.BuildFilename(filenameFormat, values); } if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.USER_AGENT, policy, signature, kvp, authService.CurrentAuth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, progressReporter, customFileName, filename, path); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); } } if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } } return false; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } public async Task DownloadPurchasedMessageDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, PurchasedEntities.ListItem? messageInfo, MessageEntities.Medium? messageMedia, CommonEntities.FromUser? fromUser, Dictionary users) { try { string customFileName = ""; string path; Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; if (configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null && messageInfo?.Id is not null && messageInfo?.CreatedAt is not null) { path = $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}/Videos"; } else { path = "/Messages/Paid/Videos"; } if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(filenameFormat) && messageInfo != null && messageMedia != null) { List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); foreach (Match match in matches) { properties.Add(match.Groups[1].Value); } Dictionary values = await fileNameService.GetFilename(messageInfo, messageMedia, fromUser, properties, folder.Split("/")[^1], users); customFileName = await fileNameService.BuildFilename(filenameFormat, values); } if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.USER_AGENT, policy, signature, kvp, authService.CurrentAuth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, progressReporter, customFileName, filename, path); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); } } if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } } return false; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } public async Task DownloadSinglePurchasedMessageDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, MessageEntities.SingleMessage? messageInfo, MessageEntities.Medium? messageMedia, CommonEntities.FromUser? fromUser, Dictionary users) { try { string customFileName = ""; string path; Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; if (configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null && messageInfo?.Id is not null && messageInfo?.CreatedAt is not null) { path = $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}/Videos"; } else { path = "/Messages/Paid/Videos"; } if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(filenameFormat) && messageInfo != null && messageMedia != null) { List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); foreach (Match match in matches) { properties.Add(match.Groups[1].Value); } Dictionary values = await fileNameService.GetFilename(messageInfo, messageMedia, fromUser, properties, folder.Split("/")[^1], users); customFileName = await fileNameService.BuildFilename(filenameFormat, values); } if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.USER_AGENT, policy, signature, kvp, authService.CurrentAuth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, progressReporter, customFileName, filename, path); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); } } if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } } return false; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } public async Task DownloadPostDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, PostEntities.ListItem? postInfo, PostEntities.Medium? postMedia, CommonEntities.Author? author, Dictionary users) { try { string customFileName = ""; string path; Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; if (configService.CurrentConfig.FolderPerPost && postInfo != null && postInfo?.Id is not null && postInfo?.PostedAt is not null) { path = $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}/Videos"; } else { path = "/Posts/Free/Videos"; } if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(filenameFormat) && postInfo != null && postMedia != null) { List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); foreach (Match match in matches) { properties.Add(match.Groups[1].Value); } Dictionary values = await fileNameService.GetFilename(postInfo, postMedia, author, properties, folder.Split("/")[^1], users); customFileName = await fileNameService.BuildFilename(filenameFormat, values); } if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.USER_AGENT, policy, signature, kvp, authService.CurrentAuth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, progressReporter, customFileName, filename, path); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); } } if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } } return false; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } public async Task DownloadPostDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string filenameFormat, PostEntities.SinglePost postInfo, PostEntities.Medium postMedia, CommonEntities.Author author, Dictionary users) { try { string customFileName = ""; string path; Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; if (configService.CurrentConfig.FolderPerPost && postInfo != null && postInfo?.Id is not null && postInfo?.PostedAt is not null) { path = $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}/Videos"; } else { path = "/Posts/Free/Videos"; } if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(filenameFormat) && postInfo != null && postMedia != null) { List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); foreach (Match match in matches) { properties.Add(match.Groups[1].Value); } Dictionary values = await fileNameService.GetFilename(postInfo, postMedia, author, properties, folder.Split("/")[^1], users); customFileName = await fileNameService.BuildFilename(filenameFormat, values); } if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.USER_AGENT, policy, signature, kvp, authService.CurrentAuth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, progressReporter, customFileName, filename, path); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); } } if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } } return false; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } public async Task DownloadStreamsDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, StreamEntities.ListItem? streamInfo, StreamEntities.Medium? streamMedia, CommonEntities.Author? author, Dictionary users) { try { string customFileName = ""; string path; Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; if (configService.CurrentConfig.FolderPerPost && streamInfo != null && streamInfo?.Id is not null && streamInfo?.PostedAt is not null) { path = $"/Posts/Free/{streamInfo.Id} {streamInfo.PostedAt:yyyy-MM-dd HH-mm-ss}/Videos"; } else { path = "/Posts/Free/Videos"; } if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(filenameFormat) && streamInfo != null && streamMedia != null) { List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); foreach (Match match in matches) { properties.Add(match.Groups[1].Value); } Dictionary values = await fileNameService.GetFilename(streamInfo, streamMedia, author, properties, folder.Split("/")[^1], users); customFileName = await fileNameService.BuildFilename(filenameFormat, values); } if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.USER_AGENT, policy, signature, kvp, authService.CurrentAuth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, progressReporter, customFileName, filename, path); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); } } if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } } return false; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } public async Task DownloadPurchasedPostDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, PurchasedEntities.ListItem? postInfo, MessageEntities.Medium? postMedia, CommonEntities.FromUser? fromUser, Dictionary users) { try { string customFileName = ""; string path; Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; if (configService.CurrentConfig.FolderPerPaidPost && postInfo != null && postInfo?.Id is not null && postInfo?.PostedAt is not null) { path = $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}/Videos"; } else { path = "/Posts/Paid/Videos"; } if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(filenameFormat) && postInfo != null && postMedia != null) { List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); foreach (Match match in matches) { properties.Add(match.Groups[1].Value); } Dictionary values = await fileNameService.GetFilename(postInfo, postMedia, fromUser, properties, folder.Split("/")[^1], users); customFileName = await fileNameService.BuildFilename(filenameFormat, values); } if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.USER_AGENT, policy, signature, kvp, authService.CurrentAuth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, progressReporter, customFileName, filename, path); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); } } if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } } return false; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } public async Task DownloadArchivedPostDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, IProgressReporter progressReporter, string? filenameFormat, ArchivedEntities.ListItem? postInfo, ArchivedEntities.Medium? postMedia, CommonEntities.Author? author, Dictionary users) { try { string customFileName = ""; Uri uri = new(url); string filename = Path.GetFileName(uri.LocalPath).Split(".")[0]; string path = "/Archived/Posts/Free/Videos"; if (!Directory.Exists(folder + path)) { Directory.CreateDirectory(folder + path); } if (!string.IsNullOrEmpty(filenameFormat) && postInfo != null && postMedia != null) { List properties = new(); string pattern = @"\{(.*?)\}"; MatchCollection matches = Regex.Matches(filenameFormat, pattern); foreach (Match match in matches) { properties.Add(match.Groups[1].Value); } Dictionary values = await fileNameService.GetFilename(postInfo, postMedia, author, properties, folder.Split("/")[^1], users); customFileName = await fileNameService.BuildFilename(filenameFormat, values); } if (!await dbService.CheckDownloaded(folder, media_id, api_type)) { if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) { return await DownloadDrmMedia(authService.CurrentAuth.USER_AGENT, policy, signature, kvp, authService.CurrentAuth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, progressReporter, customFileName, filename, path); } long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; if (configService.CurrentConfig.ShowScrapeSize) { progressReporter.ReportProgress(fileSizeInBytes); } else { progressReporter.ReportProgress(1); } await dbService.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); } else { if (!string.IsNullOrEmpty(customFileName)) { if (configService.CurrentConfig.RenameExistingFilesWhenCustomFormatIsSelected && filename + "_source" != customFileName) { string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; if (!File.Exists(fullPathWithTheServerFileName)) { return false; } try { File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); } catch (Exception ex) { Console.WriteLine($"An error occurred: {ex.Message}"); return false; } long size = await dbService.GetStoredFileSize(folder, media_id, api_type); await dbService.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); } } if (configService.CurrentConfig.ShowScrapeSize) { long size = await dbService.GetStoredFileSize(folder, media_id, api_type); progressReporter.ReportProgress(size); } else { progressReporter.ReportProgress(1); } } return false; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return false; } #endregion #region Collection Download Methods public async Task DownloadHighlights(string username, long userId, string path, HashSet paidPostIds, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadHighlights - {username}"); Dictionary highlights = await apiService.GetMedia(MediaType.Highlights, $"/users/{userId}/stories/highlights", null, path, paidPostIds.ToList()); if (highlights == null || highlights.Count == 0) { Log.Debug("Found 0 Highlights"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Highlights", Success = true }; } Log.Debug($"Found {highlights.Count} Highlights"); int oldHighlightsCount = 0; int newHighlightsCount = 0; foreach (KeyValuePair highlightKVP in highlights) { bool isNew = await DownloadStoryMedia(highlightKVP.Value, path, highlightKVP.Key, "Stories", progressReporter); if (isNew) { newHighlightsCount++; } else { oldHighlightsCount++; } } Log.Debug( $"Highlights Already Downloaded: {oldHighlightsCount} New Highlights Downloaded: {newHighlightsCount}"); return new DownloadResult { TotalCount = highlights.Count, NewDownloads = newHighlightsCount, ExistingDownloads = oldHighlightsCount, MediaType = "Highlights", Success = true }; } public async Task DownloadStories(string username, long userId, string path, HashSet paidPostIds, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadStories - {username}"); Dictionary stories = await apiService.GetMedia(MediaType.Stories, $"/users/{userId}/stories", null, path, paidPostIds.ToList()); if (stories == null || stories.Count == 0) { Log.Debug("Found 0 Stories"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Stories", Success = true }; } Log.Debug($"Found {stories.Count} Stories"); int oldStoriesCount = 0; int newStoriesCount = 0; foreach (KeyValuePair storyKVP in stories) { bool isNew = await DownloadStoryMedia(storyKVP.Value, path, storyKVP.Key, "Stories", progressReporter); if (isNew) { newStoriesCount++; } else { oldStoriesCount++; } } Log.Debug($"Stories Already Downloaded: {oldStoriesCount} New Stories Downloaded: {newStoriesCount}"); return new DownloadResult { TotalCount = stories.Count, NewDownloads = newStoriesCount, ExistingDownloads = oldStoriesCount, MediaType = "Stories", Success = true }; } public async Task DownloadArchived(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, ArchivedEntities.ArchivedCollection archived, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadArchived - {username}"); if (archived == null || archived.ArchivedPosts.Count == 0) { Log.Debug("Found 0 Archived Posts"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Archived Posts", Success = true }; } Log.Debug( $"Found {archived.ArchivedPosts.Count} Media from {archived.ArchivedPostObjects.Count} Archived Posts"); int oldArchivedCount = 0; int newArchivedCount = 0; foreach (KeyValuePair archivedKVP in archived.ArchivedPosts) { bool isNew; if (archivedKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] messageUrlParsed = archivedKVP.Value.Split(','); string mpdURL = messageUrlParsed[0]; string policy = messageUrlParsed[1]; string signature = messageUrlParsed[2]; string kvp = messageUrlParsed[3]; string mediaId = messageUrlParsed[4]; string postId = messageUrlParsed[5]; string? pssh = await apiService.GetDRMMPDPSSH(mpdURL, policy, signature, kvp); if (pssh != null) { DateTime lastModified = await apiService.GetDRMMPDLastModified(mpdURL, policy, signature, kvp); Dictionary drmHeaders = apiService.GetDynamicHeaders($"/api2/v2/users/media/{mediaId}/drm/post/{postId}", "?type=widevine"); string decryptionKey; if (clientIdBlobMissing || devicePrivateKeyMissing) { decryptionKey = await apiService.GetDecryptionKeyOFDL(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/post/{postId}?type=widevine", pssh); } else { decryptionKey = await apiService.GetDecryptionKeyCDM(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/post/{postId}?type=widevine", pssh); } ArchivedEntities.Medium? mediaInfo = archived.ArchivedPostMedia.FirstOrDefault(m => m.Id == archivedKVP.Key); ArchivedEntities.ListItem? postInfo = archived.ArchivedPostObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); isNew = await DownloadArchivedPostDRMVideo( policy, signature, kvp, mpdURL, decryptionKey, path, lastModified, archivedKVP.Key, "Posts", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "", postInfo, mediaInfo, postInfo?.Author, users); } else { continue; } } else { ArchivedEntities.Medium? mediaInfo = archived.ArchivedPostMedia.FirstOrDefault(m => m.Id == archivedKVP.Key); ArchivedEntities.ListItem? postInfo = archived.ArchivedPostObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); isNew = await DownloadArchivedMedia( archivedKVP.Value, path, archivedKVP.Key, "Posts", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "", postInfo, mediaInfo, postInfo?.Author, users); } if (isNew) { newArchivedCount++; } else { oldArchivedCount++; } } Log.Debug( $"Archived Posts Already Downloaded: {oldArchivedCount} New Archived Posts Downloaded: {newArchivedCount}"); return new DownloadResult { TotalCount = archived.ArchivedPosts.Count, NewDownloads = newArchivedCount, ExistingDownloads = oldArchivedCount, MediaType = "Archived Posts", Success = true }; } public async Task DownloadMessages(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, MessageEntities.MessageCollection messages, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadMessages - {username}"); if (messages == null || messages.Messages.Count == 0) { Log.Debug("Found 0 Messages"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Messages", Success = true }; } Log.Debug($"Found {messages.Messages.Count} Media from {messages.MessageObjects.Count} Messages"); int oldMessagesCount = 0; int newMessagesCount = 0; foreach (KeyValuePair messageKVP in messages.Messages) { bool isNew; if (messageKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] messageUrlParsed = messageKVP.Value.Split(','); string mpdURL = messageUrlParsed[0]; string policy = messageUrlParsed[1]; string signature = messageUrlParsed[2]; string kvp = messageUrlParsed[3]; string mediaId = messageUrlParsed[4]; string messageId = messageUrlParsed[5]; string? pssh = await apiService.GetDRMMPDPSSH(mpdURL, policy, signature, kvp); if (pssh != null) { DateTime lastModified = await apiService.GetDRMMPDLastModified(mpdURL, policy, signature, kvp); Dictionary drmHeaders = apiService.GetDynamicHeaders($"/api2/v2/users/media/{mediaId}/drm/message/{messageId}", "?type=widevine"); string decryptionKey; if (clientIdBlobMissing || devicePrivateKeyMissing) { decryptionKey = await apiService.GetDecryptionKeyOFDL(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine", pssh); } else { decryptionKey = await apiService.GetDecryptionKeyCDM(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine", pssh); } MessageEntities.Medium? mediaInfo = messages.MessageMedia.FirstOrDefault(m => m.Id == messageKVP.Key); MessageEntities.ListItem? messageInfo = messages.MessageObjects.FirstOrDefault(p => p?.Media?.Any(m => m.Id == messageKVP.Key) == true); isNew = await DownloadMessageDRMVideo( policy, signature, kvp, mpdURL, decryptionKey, path, lastModified, messageKVP.Key, "Messages", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).MessageFileNameFormat ?? "", messageInfo, mediaInfo, messageInfo?.FromUser, users); } else { continue; } } else { MessageEntities.Medium? mediaInfo = messages.MessageMedia.FirstOrDefault(m => m.Id == messageKVP.Key); MessageEntities.ListItem? messageInfo = messages.MessageObjects.FirstOrDefault(p => p?.Media?.Any(m => m.Id == messageKVP.Key) == true); isNew = await DownloadMessageMedia( messageKVP.Value, path, messageKVP.Key, "Messages", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).MessageFileNameFormat ?? "", messageInfo, mediaInfo, messageInfo?.FromUser, users); } if (isNew) { newMessagesCount++; } else { oldMessagesCount++; } } Log.Debug($"Messages Already Downloaded: {oldMessagesCount} New Messages Downloaded: {newMessagesCount}"); return new DownloadResult { TotalCount = messages.Messages.Count, NewDownloads = newMessagesCount, ExistingDownloads = oldMessagesCount, MediaType = "Messages", Success = true }; } public async Task DownloadPaidMessages(string username, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PurchasedEntities.PaidMessageCollection paidMessageCollection, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadPaidMessages - {username}"); if (paidMessageCollection == null || paidMessageCollection.PaidMessages.Count == 0) { Log.Debug("Found 0 Paid Messages"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Paid Messages", Success = true }; } Log.Debug( $"Found {paidMessageCollection.PaidMessages.Count} Media from {paidMessageCollection.PaidMessageObjects.Count} Paid Messages"); int oldCount = 0; int newCount = 0; foreach (KeyValuePair kvp in paidMessageCollection.PaidMessages) { bool isNew; if (kvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = kvp.Value.Split(','); string? pssh = await apiService.GetDRMMPDPSSH(parsed[0], parsed[1], parsed[2], parsed[3]); if (pssh != null) { DateTime lastModified = await apiService.GetDRMMPDLastModified(parsed[0], parsed[1], parsed[2], parsed[3]); Dictionary drmHeaders = apiService.GetDynamicHeaders($"/api2/v2/users/media/{parsed[4]}/drm/message/{parsed[5]}", "?type=widevine"); string decryptionKey = clientIdBlobMissing || devicePrivateKeyMissing ? await apiService.GetDecryptionKeyOFDL(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{parsed[4]}/drm/message/{parsed[5]}?type=widevine", pssh) : await apiService.GetDecryptionKeyCDM(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{parsed[4]}/drm/message/{parsed[5]}?type=widevine", pssh); MessageEntities.Medium? mediaInfo = paidMessageCollection.PaidMessageMedia.FirstOrDefault(m => m.Id == kvp.Key); PurchasedEntities.ListItem? messageInfo = paidMessageCollection.PaidMessageObjects.FirstOrDefault(p => p?.Media?.Any(m => m.Id == kvp.Key) == true); isNew = await DownloadPurchasedMessageDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], decryptionKey, path, lastModified, kvp.Key, "Messages", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).MessageFileNameFormat ?? "", messageInfo, mediaInfo, messageInfo?.FromUser, users); } else { continue; } } else { MessageEntities.Medium? mediaInfo = paidMessageCollection.PaidMessageMedia.FirstOrDefault(m => m.Id == kvp.Key); PurchasedEntities.ListItem? messageInfo = paidMessageCollection.PaidMessageObjects.FirstOrDefault(p => p?.Media?.Any(m => m.Id == kvp.Key) == true); isNew = await DownloadPurchasedMedia(kvp.Value, path, kvp.Key, "Messages", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).MessageFileNameFormat ?? "", messageInfo, mediaInfo, messageInfo?.FromUser, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Paid Messages Already Downloaded: {oldCount} New Paid Messages Downloaded: {newCount}"); return new DownloadResult { TotalCount = paidMessageCollection.PaidMessages.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Paid Messages", Success = true }; } public async Task DownloadStreams(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, StreamEntities.StreamsCollection streams, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadStreams - {username}"); if (streams == null || streams.Streams.Count == 0) { Log.Debug("Found 0 Streams"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Streams", Success = true }; } Log.Debug($"Found {streams.Streams.Count} Media from {streams.StreamObjects.Count} Streams"); int oldCount = 0; int newCount = 0; foreach (KeyValuePair kvp in streams.Streams) { bool isNew; if (kvp.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = kvp.Value.Split(','); string? pssh = await apiService.GetDRMMPDPSSH(parsed[0], parsed[1], parsed[2], parsed[3]); if (pssh != null) { DateTime lastModified = await apiService.GetDRMMPDLastModified(parsed[0], parsed[1], parsed[2], parsed[3]); Dictionary drmHeaders = apiService.GetDynamicHeaders($"/api2/v2/users/media/{parsed[4]}/drm/post/{parsed[5]}", "?type=widevine"); string decryptionKey = clientIdBlobMissing || devicePrivateKeyMissing ? await apiService.GetDecryptionKeyOFDL(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{parsed[4]}/drm/post/{parsed[5]}?type=widevine", pssh) : await apiService.GetDecryptionKeyCDM(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{parsed[4]}/drm/post/{parsed[5]}?type=widevine", pssh); StreamEntities.Medium? mediaInfo = streams.StreamMedia.FirstOrDefault(m => m.Id == kvp.Key); StreamEntities.ListItem? streamInfo = streams.StreamObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); isNew = await DownloadStreamsDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], decryptionKey, path, lastModified, kvp.Key, "Streams", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "", streamInfo, mediaInfo, streamInfo?.Author, users); } else { continue; } } else { StreamEntities.Medium? mediaInfo = streams.StreamMedia.FirstOrDefault(m => m.Id == kvp.Key); StreamEntities.ListItem? streamInfo = streams.StreamObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); isNew = await DownloadStreamMedia(kvp.Value, path, kvp.Key, "Streams", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "", streamInfo, mediaInfo, streamInfo?.Author, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Streams Already Downloaded: {oldCount} New Streams Downloaded: {newCount}"); return new DownloadResult { TotalCount = streams.Streams.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Streams", Success = true }; } // Add these methods to DownloadService.cs before the #endregion line public async Task DownloadFreePosts(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PostEntities.PostCollection posts, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadFreePosts - {username}"); if (posts == null || posts.Posts.Count == 0) { Log.Debug("Found 0 Posts"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Posts", Success = true }; } Log.Debug($"Found {posts.Posts.Count} Media from {posts.PostObjects.Count} Posts"); int oldCount = 0, newCount = 0; foreach (KeyValuePair postKVP in posts.Posts) { bool isNew; if (postKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = postKVP.Value.Split(','); string? pssh = await apiService.GetDRMMPDPSSH(parsed[0], parsed[1], parsed[2], parsed[3]); if (pssh != null) { DateTime lastModified = await apiService.GetDRMMPDLastModified(parsed[0], parsed[1], parsed[2], parsed[3]); Dictionary drmHeaders = apiService.GetDynamicHeaders($"/api2/v2/users/media/{parsed[4]}/drm/post/{parsed[5]}", "?type=widevine"); string decryptionKey = clientIdBlobMissing || devicePrivateKeyMissing ? await apiService.GetDecryptionKeyOFDL(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{parsed[4]}/drm/post/{parsed[5]}?type=widevine", pssh) : await apiService.GetDecryptionKeyCDM(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{parsed[4]}/drm/post/{parsed[5]}?type=widevine", pssh); PostEntities.Medium mediaInfo = posts.PostMedia.FirstOrDefault(m => m.Id == postKVP.Key); PostEntities.ListItem postInfo = posts.PostObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); isNew = await DownloadPostDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], decryptionKey, path, lastModified, postKVP.Key, "Posts", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "", postInfo, mediaInfo, postInfo?.Author, users); } else { continue; } } else { PostEntities.Medium? mediaInfo = posts.PostMedia.FirstOrDefault(m => m?.Id == postKVP.Key); PostEntities.ListItem? postInfo = posts.PostObjects.FirstOrDefault(p => p?.Media?.Contains(mediaInfo) == true); isNew = await DownloadPostMedia(postKVP.Value, path, postKVP.Key, "Posts", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "", postInfo, mediaInfo, postInfo?.Author, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Posts Already Downloaded: {oldCount} New Posts Downloaded: {newCount}"); return new DownloadResult { TotalCount = posts.Posts.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Posts", Success = true }; } public async Task DownloadPaidPosts(string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, PurchasedEntities.PaidPostCollection purchasedPosts, IProgressReporter progressReporter) { Log.Debug($"Calling DownloadPaidPosts - {username}"); if (purchasedPosts == null || purchasedPosts.PaidPosts.Count == 0) { Log.Debug("Found 0 Paid Posts"); return new DownloadResult { TotalCount = 0, NewDownloads = 0, ExistingDownloads = 0, MediaType = "Paid Posts", Success = true }; } Log.Debug( $"Found {purchasedPosts.PaidPosts.Count} Media from {purchasedPosts.PaidPostObjects.Count} Paid Posts"); int oldCount = 0, newCount = 0; foreach (KeyValuePair postKVP in purchasedPosts.PaidPosts) { bool isNew; if (postKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) { string[] parsed = postKVP.Value.Split(','); string? pssh = await apiService.GetDRMMPDPSSH(parsed[0], parsed[1], parsed[2], parsed[3]); if (pssh != null) { DateTime lastModified = await apiService.GetDRMMPDLastModified(parsed[0], parsed[1], parsed[2], parsed[3]); Dictionary drmHeaders = apiService.GetDynamicHeaders($"/api2/v2/users/media/{parsed[4]}/drm/post/{parsed[5]}", "?type=widevine"); string decryptionKey = clientIdBlobMissing || devicePrivateKeyMissing ? await apiService.GetDecryptionKeyOFDL(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{parsed[4]}/drm/post/{parsed[5]}?type=widevine", pssh) : await apiService.GetDecryptionKeyCDM(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{parsed[4]}/drm/post/{parsed[5]}?type=widevine", pssh); MessageEntities.Medium? mediaInfo = purchasedPosts.PaidPostMedia.FirstOrDefault(m => m.Id == postKVP.Key); PurchasedEntities.ListItem? postInfo = purchasedPosts.PaidPostObjects.FirstOrDefault(p => p?.Media?.Any(m => m.Id == postKVP.Key) == true); isNew = await DownloadPurchasedPostDRMVideo(parsed[1], parsed[2], parsed[3], parsed[0], decryptionKey, path, lastModified, postKVP.Key, "Posts", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "", postInfo, mediaInfo, postInfo?.FromUser, users); } else { continue; } } else { MessageEntities.Medium? mediaInfo = purchasedPosts.PaidPostMedia.FirstOrDefault(m => m.Id == postKVP.Key); PurchasedEntities.ListItem? postInfo = purchasedPosts.PaidPostObjects.FirstOrDefault(p => p?.Media?.Any(m => m.Id == postKVP.Key) == true); isNew = await DownloadPurchasedPostMedia(postKVP.Value, path, postKVP.Key, "Posts", progressReporter, configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "", postInfo, mediaInfo, postInfo?.FromUser, users); } if (isNew) { newCount++; } else { oldCount++; } } Log.Debug($"Paid Posts Already Downloaded: {oldCount} New Paid Posts Downloaded: {newCount}"); return new DownloadResult { TotalCount = purchasedPosts.PaidPosts.Count, NewDownloads = newCount, ExistingDownloads = oldCount, MediaType = "Paid Posts", Success = true }; } #endregion }