diff --git a/Dockerfile b/Dockerfile index 9044f8c..1307df3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.20 AS build +FROM alpine:3.23 AS build ARG VERSION @@ -21,14 +21,14 @@ RUN /src/out/OF\ DL --non-interactive || true && \ mv /src/updated_config.conf /src/config.conf -FROM alpine:3.20 AS final +FROM alpine:3.23 AS final # Install dependencies RUN apk --no-cache --repository community add \ bash \ tini \ dotnet8-runtime \ - ffmpeg \ + ffmpeg7 \ udev \ ttf-freefont \ chromium \ diff --git a/OF DL/Entities/Purchased/SinglePaidMessageCollection.cs b/OF DL/Entities/Purchased/SinglePaidMessageCollection.cs index 703530b..625b6ff 100644 --- a/OF DL/Entities/Purchased/SinglePaidMessageCollection.cs +++ b/OF DL/Entities/Purchased/SinglePaidMessageCollection.cs @@ -14,5 +14,8 @@ namespace OF_DL.Entities.Purchased public Dictionary SingleMessages = new Dictionary(); public List SingleMessageObjects = new List(); public List SingleMessageMedia = new List(); + + public Dictionary PreviewSingleMessages = new Dictionary(); + public List PreviewSingleMessageMedia = new List(); } } diff --git a/OF DL/Helpers/APIHelper.cs b/OF DL/Helpers/APIHelper.cs index 84ce24e..ea209c8 100644 --- a/OF DL/Helpers/APIHelper.cs +++ b/OF DL/Helpers/APIHelper.cs @@ -601,7 +601,7 @@ public class APIHelper : IAPIHelper { continue; } - if (medium.canView && !medium.files.full.url.Contains("upload")) + if (medium.canView) { if (!return_urls.ContainsKey(medium.id)) { @@ -836,7 +836,7 @@ public class APIHelper : IAPIHelper if (previewids.Count > 0) { bool has = previewids.Any(cus => cus.Equals(medium.id)); - if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (!paidPostCollection.PaidPosts.ContainsKey(medium.id)) { @@ -859,7 +859,7 @@ public class APIHelper : IAPIHelper } else { - if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (!paidPostCollection.PaidPosts.ContainsKey(medium.id)) { @@ -1028,7 +1028,7 @@ public class APIHelper : IAPIHelper bool has = paid_post_ids.Any(cus => cus.Equals(medium.id)); if (medium.files!.full != null && !string.IsNullOrEmpty(medium.files!.full.url)) { - if (!has && !medium.files!.full.url.Contains("upload")) + if (!has) { if (!postCollection.Posts.ContainsKey(medium.id)) { @@ -1040,7 +1040,7 @@ public class APIHelper : IAPIHelper } else if (medium.files.preview != null && medium.files!.full == null) { - if (!has && !medium.files.preview.url.Contains("upload")) + if (!has) { if (!postCollection.Posts.ContainsKey(medium.id)) { @@ -1144,14 +1144,11 @@ public class APIHelper : IAPIHelper case VideoResolution.source: if (medium.files!.full != null && !string.IsNullOrEmpty(medium.files!.full.url)) { - if (!medium.files!.full.url.Contains("upload")) + if (!singlePostCollection.SinglePosts.ContainsKey(medium.id)) { - if (!singlePostCollection.SinglePosts.ContainsKey(medium.id)) - { - await m_DBHelper.AddMedia(folder, medium.id, singlePost.id, medium.files!.full.url, null, null, null, "Posts", medium.type == "photo" ? "Images" : (medium.type == "video" || medium.type == "gif" ? "Videos" : (medium.type == "audio" ? "Audios" : null)), postPreviewIds.Contains((long)medium.id) ? true : false, false, null); - singlePostCollection.SinglePosts.Add(medium.id, medium.files!.full.url); - singlePostCollection.SinglePostMedia.Add(medium); - } + await m_DBHelper.AddMedia(folder, medium.id, singlePost.id, medium.files!.full.url, null, null, null, "Posts", medium.type == "photo" ? "Images" : (medium.type == "video" || medium.type == "gif" ? "Videos" : (medium.type == "audio" ? "Audios" : null)), postPreviewIds.Contains((long)medium.id) ? true : false, false, null); + singlePostCollection.SinglePosts.Add(medium.id, medium.files!.full.url); + singlePostCollection.SinglePostMedia.Add(medium); } } break; @@ -1200,14 +1197,11 @@ public class APIHelper : IAPIHelper } else if (medium.files.preview != null && medium.files!.full == null) { - if (!medium.files.preview.url.Contains("upload")) + if (!singlePostCollection.SinglePosts.ContainsKey(medium.id)) { - if (!singlePostCollection.SinglePosts.ContainsKey(medium.id)) - { - await m_DBHelper.AddMedia(folder, medium.id, singlePost.id, medium.files.preview.url, null, null, null, "Posts", medium.type == "photo" ? "Images" : (medium.type == "video" || medium.type == "gif" ? "Videos" : (medium.type == "audio" ? "Audios" : null)), postPreviewIds.Contains((long)medium.id) ? true : false, false, null); - singlePostCollection.SinglePosts.Add(medium.id, medium.files.preview.url); - singlePostCollection.SinglePostMedia.Add(medium); - } + await m_DBHelper.AddMedia(folder, medium.id, singlePost.id, medium.files.preview.url, null, null, null, "Posts", medium.type == "photo" ? "Images" : (medium.type == "video" || medium.type == "gif" ? "Videos" : (medium.type == "audio" ? "Audios" : null)), postPreviewIds.Contains((long)medium.id) ? true : false, false, null); + singlePostCollection.SinglePosts.Add(medium.id, medium.files.preview.url); + singlePostCollection.SinglePostMedia.Add(medium); } } } @@ -1335,7 +1329,7 @@ public class APIHelper : IAPIHelper if (medium.canView && medium.files?.drm == null) { bool has = paid_post_ids.Any(cus => cus.Equals(medium.id)); - if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (!streamsCollection.Streams.ContainsKey(medium.id)) { @@ -1480,7 +1474,7 @@ public class APIHelper : IAPIHelper { continue; } - if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (!archivedCollection.ArchivedPosts.ContainsKey(medium.id)) { @@ -1593,7 +1587,7 @@ public class APIHelper : IAPIHelper { foreach (Messages.Medium medium in list.media) { - if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (medium.type == "photo" && !config.DownloadImages) { @@ -1649,7 +1643,7 @@ public class APIHelper : IAPIHelper { foreach (Messages.Medium medium in list.media) { - if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload") && messagePreviewIds.Contains(medium.id)) + if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && messagePreviewIds.Contains(medium.id)) { if (medium.type == "photo" && !config.DownloadImages) { @@ -1761,7 +1755,7 @@ public class APIHelper : IAPIHelper { foreach (Messages.Medium medium in message.media) { - if (!messagePreviewIds.Contains(medium.id) && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (!messagePreviewIds.Contains(medium.id) && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (medium.type == "photo" && !config.DownloadImages) { @@ -1787,6 +1781,32 @@ public class APIHelper : IAPIHelper singlePaidMessageCollection.SingleMessageMedia.Add(medium); } } + else if (messagePreviewIds.Contains(medium.id) && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) + { + if (medium.type == "photo" && !config.DownloadImages) + { + continue; + } + if (medium.type == "video" && !config.DownloadVideos) + { + continue; + } + if (medium.type == "gif" && !config.DownloadVideos) + { + continue; + } + if (medium.type == "audio" && !config.DownloadAudios) + { + continue; + } + + if (!singlePaidMessageCollection.PreviewSingleMessages.ContainsKey(medium.id)) + { + await m_DBHelper.AddMedia(folder, medium.id, message.id, medium.files.full.url, null, null, null, "Messages", medium.type == "photo" ? "Images" : (medium.type == "video" || medium.type == "gif" ? "Videos" : (medium.type == "audio" ? "Audios" : null)), messagePreviewIds.Contains(medium.id) ? true : false, false, null); + singlePaidMessageCollection.PreviewSingleMessages.Add(medium.id, medium.files.full.url.ToString()); + singlePaidMessageCollection.PreviewSingleMessageMedia.Add(medium); + } + } else if (!messagePreviewIds.Contains(medium.id) && medium.canView && medium.files != null && medium.files.drm != null) { if (medium.type == "photo" && !config.DownloadImages) @@ -1813,6 +1833,32 @@ public class APIHelper : IAPIHelper singlePaidMessageCollection.SingleMessageMedia.Add(medium); } } + else if (messagePreviewIds.Contains(medium.id) && medium.canView && medium.files != null && medium.files.drm != null) + { + if (medium.type == "photo" && !config.DownloadImages) + { + continue; + } + if (medium.type == "video" && !config.DownloadVideos) + { + continue; + } + if (medium.type == "gif" && !config.DownloadVideos) + { + continue; + } + if (medium.type == "audio" && !config.DownloadAudios) + { + continue; + } + + if (!singlePaidMessageCollection.PreviewSingleMessages.ContainsKey(medium.id)) + { + await m_DBHelper.AddMedia(folder, medium.id, message.id, medium.files.drm.manifest.dash, null, null, null, "Messages", medium.type == "photo" ? "Images" : (medium.type == "video" || medium.type == "gif" ? "Videos" : (medium.type == "audio" ? "Audios" : null)), messagePreviewIds.Contains(medium.id) ? true : false, false, null); + singlePaidMessageCollection.PreviewSingleMessages.Add(medium.id, $"{medium.files.drm.manifest.dash},{medium.files.drm.signature.dash.CloudFrontPolicy},{medium.files.drm.signature.dash.CloudFrontSignature},{medium.files.drm.signature.dash.CloudFrontKeyPairId},{medium.id},{message.id}"); + singlePaidMessageCollection.PreviewSingleMessageMedia.Add(medium); + } + } } } } @@ -1941,7 +1987,7 @@ public class APIHelper : IAPIHelper if (previewids.Count > 0) { bool has = previewids.Any(cus => cus.Equals(medium.id)); - if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (medium.type == "photo" && !config.DownloadImages) { @@ -1994,7 +2040,7 @@ public class APIHelper : IAPIHelper } else { - if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (medium.type == "photo" && !config.DownloadImages) { @@ -2325,6 +2371,11 @@ public class APIHelper : IAPIHelper { foreach (Purchased.List purchase in user.Value) { + if (purchase.media == null) + { + Log.Warning("PurchasedTab purchase media null, setting empty list | userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt}", user.Key, purchasedTabCollection.Username, purchase.id, purchase.responseType, purchase.createdAt, purchase.postedAt); + purchase.media = new List(); + } switch (purchase.responseType) { case "post": @@ -2378,7 +2429,7 @@ public class APIHelper : IAPIHelper if (previewids.Count > 0) { bool has = previewids.Any(cus => cus.Equals(medium.id)); - if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.id)) @@ -2402,7 +2453,7 @@ public class APIHelper : IAPIHelper } else { - if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.id)) { @@ -2468,7 +2519,7 @@ public class APIHelper : IAPIHelper if (paidMessagePreviewids.Count > 0) { bool has = paidMessagePreviewids.Any(cus => cus.Equals(medium.id)); - if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (medium.type == "photo" && !config.DownloadImages) { @@ -2521,7 +2572,7 @@ public class APIHelper : IAPIHelper } else { - if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url) && !medium.files.full.url.Contains("upload")) + if (medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (medium.type == "photo" && !config.DownloadImages) { diff --git a/OF DL/Helpers/DownloadHelper.cs b/OF DL/Helpers/DownloadHelper.cs index e936b77..f5ad8e2 100644 --- a/OF DL/Helpers/DownloadHelper.cs +++ b/OF DL/Helpers/DownloadHelper.cs @@ -29,6 +29,7 @@ using System.Threading.Tasks; using System.Xml.Linq; using static OF_DL.Entities.Lists.UserList; using static OF_DL.Entities.Messages.Messages; +using FromUser = OF_DL.Entities.Messages.FromUser; namespace OF_DL.Helpers; @@ -630,7 +631,55 @@ public class DownloadHelper : IDownloadHelper // break; //} - string parameters = $"-cenc_decryption_key {decKey} -headers \"Cookie:CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {sess} Origin: https://onlyfans.com Referer: https://onlyfans.com User-Agent: {user_agent}\" -y -i \"{url}\" -map 0:v:{streamIndex} -map 0:a? -codec copy \"{tempFilename}\""; + // Configure ffmpeg log level and optional report file location + bool ffmpegDebugLogging = Log.IsEnabled(Serilog.Events.LogEventLevel.Debug); + + string logLevelArgs = ffmpegDebugLogging || downloadConfig.LoggingLevel is LoggingLevel.Verbose or LoggingLevel.Debug + ? "-loglevel debug -report" + : downloadConfig.LoggingLevel switch + { + LoggingLevel.Information => "-loglevel info", + LoggingLevel.Warning => "-loglevel warning", + LoggingLevel.Error => "-loglevel error", + LoggingLevel.Fatal => "-loglevel fatal", + _ => string.Empty + }; + + if (logLevelArgs.Contains("-report", StringComparison.OrdinalIgnoreCase)) + { + // Direct ffmpeg report files into the same logs directory Serilog uses (relative to current working directory) + string logDir = Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, "logs")); + Directory.CreateDirectory(logDir); + string ffReportPath = Path.Combine(logDir, "ffmpeg-%p-%t.log"); // ffmpeg will replace %p/%t + Environment.SetEnvironmentVariable("FFREPORT", $"file={ffReportPath}:level=32"); + Log.Debug("FFREPORT enabled at: {FFREPORT} (cwd: {Cwd})", Environment.GetEnvironmentVariable("FFREPORT"), Environment.CurrentDirectory); + } + else + { + Environment.SetEnvironmentVariable("FFREPORT", null); + Log.Debug("FFREPORT disabled (cwd: {Cwd})", Environment.CurrentDirectory); + } + + string cookieHeader = + "Cookie: " + + $"CloudFront-Policy={policy}; " + + $"CloudFront-Signature={signature}; " + + $"CloudFront-Key-Pair-Id={kvp}; " + + $"{sess}"; + + string parameters = + $"{logLevelArgs} " + + $"-cenc_decryption_key {decKey} " + + $"-headers \"{cookieHeader}\" " + + $"-user_agent \"{user_agent}\" " + + "-referer \"https://onlyfans.com\" " + + "-rw_timeout 20000000 " + + "-reconnect 1 -reconnect_streamed 1 -reconnect_on_network_error 1 -reconnect_delay_max 10 " + + "-y " + + $"-i \"{url}\" " + + $"-map 0:v:{streamIndex} -map 0:a? " + + "-c copy " + + $"\"{tempFilename}\""; Log.Debug($"Calling FFMPEG with Parameters: {parameters}"); @@ -734,6 +783,23 @@ public class DownloadHelper : IDownloadHelper return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, task, filename, resolvedFilename); } + public async Task DownloadMessagePreviewMedia(string url, string folder, long media_id, string api_type, ProgressTask task, string? filenameFormat, SingleMessage? messageInfo, Messages.Medium? messageMedia, FromUser? fromUser, Dictionary users) + { + string path; + if (downloadConfig.FolderPerMessage && messageInfo != null && messageInfo?.id is not null && messageInfo?.createdAt is not null) + { + path = $"/Messages/Free/{messageInfo.id} {messageInfo.createdAt.Value:yyyy-MM-dd HH-mm-ss}"; + } + else + { + path = "/Messages/Free"; + } + Uri uri = new(url); + string filename = System.IO.Path.GetFileNameWithoutExtension(uri.LocalPath); + string resolvedFilename = await GenerateCustomFileName(filename, filenameFormat, messageInfo, messageMedia, fromUser, folder.Split("/")[^1], users, _FileNameHelper, CustomFileNameOption.ReturnOriginal); + return await CreateDirectoriesAndDownloadMedia(path, url, folder, media_id, api_type, task, filename, resolvedFilename); + } + public async Task DownloadArchivedMedia(string url, string folder, long media_id, string api_type, ProgressTask task, string? filenameFormat, Archived.List? messageInfo, Archived.Medium? messageMedia, Archived.Author? author, Dictionary users) { @@ -965,10 +1031,20 @@ public class DownloadHelper : IDownloadHelper private void OnError(object sender, ConversionErrorEventArgs e) { - Log.Debug("[{0} => {1}]: Error: {2}\n{3}", e.Input.Name, e.Output.Name, e.Exception.ExitCode, e.Exception.InnerException); + // Guard all fields to avoid NullReference exceptions from FFmpeg.NET + var input = e?.Input?.Name ?? ""; + var output = e?.Output?.Name ?? ""; + var exitCode = e?.Exception?.ExitCode.ToString() ?? ""; + var message = e?.Exception?.Message ?? ""; + var inner = e?.Exception?.InnerException?.Message ?? ""; + + Log.Error("FFmpeg failed. Input={Input} Output={Output} ExitCode={ExitCode} Message={Message} Inner={Inner}", + input, output, exitCode, message, inner); + _completionSource?.TrySetResult(false); } + #region drm posts public async Task DownloadMessageDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, ProgressTask task, string? filenameFormat, Messages.List? messageInfo, Messages.Medium? messageMedia, Messages.FromUser? fromUser, Dictionary users) { @@ -1077,6 +1153,113 @@ public class DownloadHelper : IDownloadHelper return false; } + public async Task DownloadSingleMessagePreviewDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, ProgressTask task, string? filenameFormat, SingleMessage? messageInfo, Messages.Medium? messageMedia, FromUser? fromUser, Dictionary users) + { + try + { + string customFileName = string.Empty; + string path; + Uri uri = new(url); + string filename = System.IO.Path.GetFileName(uri.LocalPath).Split(".")[0]; + if (downloadConfig.FolderPerMessage && messageInfo != null && messageInfo?.id is not null && messageInfo?.createdAt is not null) + { + path = $"/Messages/Free/{messageInfo.id} {messageInfo.createdAt.Value:yyyy-MM-dd HH-mm-ss}/Videos"; + } + else + { + path = "/Messages/Free/Videos"; + } + if (!Directory.Exists(folder + path)) + { + Directory.CreateDirectory(folder + path); + } + + + if (!string.IsNullOrEmpty(filenameFormat) && messageInfo != null && messageMedia != null) + { + List properties = new(); + string pattern = @"\{(.*?)\}"; + MatchCollection matches = Regex.Matches(filenameFormat, pattern); + foreach (Match match in matches) + { + properties.Add(match.Groups[1].Value); + } + Dictionary values = await _FileNameHelper.GetFilename(messageInfo, messageMedia, fromUser, properties, folder.Split("/")[^1],users); + customFileName = await _FileNameHelper.BuildFilename(filenameFormat, values); + } + + if (!await m_DBHelper.CheckDownloaded(folder, media_id, api_type)) + { + if (!string.IsNullOrEmpty(customFileName) ? !File.Exists(folder + path + "/" + customFileName + ".mp4") : !File.Exists(folder + path + "/" + filename + "_source.mp4")) + { + return await DownloadDrmMedia(auth.USER_AGENT, policy, signature, kvp, auth.COOKIE, url, decryptionKey, folder, lastModified, media_id, api_type, task, customFileName, filename, path); + } + else + { + long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName) ? folder + path + "/" + customFileName + ".mp4" : folder + path + "/" + filename + "_source.mp4").Length; + if (downloadConfig.ShowScrapeSize) + { + task.Increment(fileSizeInBytes); + } + else + { + task.Increment(1); + } + await m_DBHelper.UpdateMedia(folder, media_id, api_type, folder + path, !string.IsNullOrEmpty(customFileName) ? customFileName + "mp4" : filename + "_source.mp4", fileSizeInBytes, true, lastModified); + } + } + else + { + if (!string.IsNullOrEmpty(customFileName)) + { + if (downloadConfig.RenameExistingFilesWhenCustomFormatIsSelected && (filename + "_source" != customFileName)) + { + string fullPathWithTheServerFileName = $"{folder}{path}/{filename}_source.mp4"; + string fullPathWithTheNewFileName = $"{folder}{path}/{customFileName}.mp4"; + if (!File.Exists(fullPathWithTheServerFileName)) + { + return false; + } + try + { + File.Move(fullPathWithTheServerFileName, fullPathWithTheNewFileName); + } + catch (Exception ex) + { + Console.WriteLine($"An error occurred: {ex.Message}"); + return false; + } + long size = await m_DBHelper.GetStoredFileSize(folder, media_id, api_type); + await m_DBHelper.UpdateMedia(folder, media_id, api_type, folder + path, customFileName + ".mp4", size, true, lastModified); + } + } + + if (downloadConfig.ShowScrapeSize) + { + long size = await m_DBHelper.GetStoredFileSize(folder, media_id, api_type); + task.Increment(size); + } + else + { + task.Increment(1); + } + } + return false; + } + catch (Exception ex) + { + Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); + Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); + if (ex.InnerException != null) + { + Console.WriteLine("\nInner Exception:"); + Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); + Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); + } + } + return false; + } + public async Task DownloadPurchasedMessageDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, ProgressTask task, string? filenameFormat, Purchased.List? messageInfo, Medium? messageMedia, Purchased.FromUser? fromUser, Dictionary users) { diff --git a/OF DL/Helpers/Interfaces/IDownloadHelper.cs b/OF DL/Helpers/Interfaces/IDownloadHelper.cs index 72cc15b..62672ae 100644 --- a/OF DL/Helpers/Interfaces/IDownloadHelper.cs +++ b/OF DL/Helpers/Interfaces/IDownloadHelper.cs @@ -6,6 +6,7 @@ using OF_DL.Entities.Purchased; using OF_DL.Entities.Streams; using Spectre.Console; using static OF_DL.Entities.Messages.Messages; +using FromUser = OF_DL.Entities.Messages.FromUser; namespace OF_DL.Helpers { @@ -32,5 +33,13 @@ namespace OF_DL.Helpers Task DownloadStoryMedia(string url, string folder, long media_id, string api_type, ProgressTask task); Task DownloadStreamMedia(string url, string folder, long media_id, string api_type, ProgressTask task, string? filenameFormat, Streams.List? streamInfo, Streams.Medium? streamMedia, Streams.Author? author, Dictionary users); Task DownloadStreamsDRMVideo(string policy, string signature, string kvp, string url, string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, ProgressTask task, string filenameFormat, Streams.List streamInfo, Streams.Medium streamMedia, Streams.Author author, Dictionary users); + Task DownloadSingleMessagePreviewDRMVideo(string policy, string signature, string kvp, string url, + string decryptionKey, string folder, DateTime lastModified, long media_id, string api_type, + ProgressTask task, string? filenameFormat, SingleMessage? messageInfo, Medium? messageMedia, + FromUser? fromUser, Dictionary users); + + Task DownloadMessagePreviewMedia(string url, string folder, long media_id, string api_type, + ProgressTask task, string? filenameFormat, SingleMessage? messageInfo, Medium? messageMedia, + FromUser? fromUser, Dictionary users); } } diff --git a/OF DL/OF DL.csproj b/OF DL/OF DL.csproj index 20dc207..2ee9f51 100644 --- a/OF DL/OF DL.csproj +++ b/OF DL/OF DL.csproj @@ -20,7 +20,7 @@ - + diff --git a/OF DL/Program.cs b/OF DL/Program.cs index f848436..bd5ff26 100644 --- a/OF DL/Program.cs +++ b/OF DL/Program.cs @@ -131,6 +131,10 @@ public class Program if (jsonConfig != null) { var hoconConfig = new StringBuilder(); + hoconConfig.AppendLine("# Auth"); + hoconConfig.AppendLine("Auth {"); + hoconConfig.AppendLine($" DisableBrowserAuth = {jsonConfig.DisableBrowserAuth.ToString().ToLower()}"); + hoconConfig.AppendLine("}"); hoconConfig.AppendLine("# External Tools"); hoconConfig.AppendLine("External {"); hoconConfig.AppendLine($" FFmpegPath = \"{jsonConfig.FFmpegPath}\""); @@ -255,7 +259,7 @@ public class Program config = new Entities.Config { //Auth - DisableBrowserAuth = hoconConfig.GetBoolean("DisableBrowserAuth"), + DisableBrowserAuth = hoconConfig.GetBoolean("Auth.DisableBrowserAuth"), // FFmpeg Settings FFmpegPath = hoconConfig.GetString("External.FFmpegPath"), @@ -375,7 +379,9 @@ public class Program Entities.Config jsonConfig = new Entities.Config(); var hoconConfig = new StringBuilder(); hoconConfig.AppendLine("# Auth"); - hoconConfig.AppendLine($"DisableBrowserAuth = {jsonConfig.DisableBrowserAuth.ToString().ToLower()}"); + hoconConfig.AppendLine("Auth {"); + hoconConfig.AppendLine($" DisableBrowserAuth = {jsonConfig.DisableBrowserAuth.ToString().ToLower()}"); + hoconConfig.AppendLine("}"); hoconConfig.AppendLine("# External Tools"); hoconConfig.AppendLine("External {"); hoconConfig.AppendLine($" FFmpegPath = \"{jsonConfig.FFmpegPath}\""); @@ -2553,9 +2559,123 @@ public class Program AnsiConsole.Markup($"[red]Getting Paid Message\n[/]"); SinglePaidMessageCollection singlePaidMessageCollection = await downloadContext.ApiHelper.GetPaidMessage($"/messages/{message_id.ToString()}", path, downloadContext.DownloadConfig!); - int oldPaidMessagesCount = 0; + int oldPreviewPaidMessagesCount = 0; + int newPreviewPaidMessagesCount = 0; + int oldPaidMessagesCount = 0; int newPaidMessagesCount = 0; - if (singlePaidMessageCollection != null && singlePaidMessageCollection.SingleMessages.Count > 0) + if (singlePaidMessageCollection != null && singlePaidMessageCollection.PreviewSingleMessages.Count > 0) + { + AnsiConsole.Markup($"[red]Found {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages\n[/]"); + Log.Debug($"Found {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages"); + paidMessagesCount = singlePaidMessageCollection.SingleMessages.Count; + long totalSize = 0; + if (downloadContext.DownloadConfig.ShowScrapeSize) + { + totalSize = await downloadContext.DownloadHelper.CalculateTotalFileSize(singlePaidMessageCollection.PreviewSingleMessages.Values.ToList()); + } + else + { + totalSize = paidMessagesCount; + } + await AnsiConsole.Progress() + .Columns(GetProgressColumns(downloadContext.DownloadConfig.ShowScrapeSize)) + .StartAsync(async ctx => + { + // Define tasks + var task = ctx.AddTask($"[red]Downloading {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Paid Messages[/]", autoStart: false); + Log.Debug($"Downloading {singlePaidMessageCollection.PreviewSingleMessages.Count} Paid Messages"); + task.MaxValue = totalSize; + task.StartTask(); + foreach (KeyValuePair paidMessageKVP in singlePaidMessageCollection.PreviewSingleMessages) + { + bool isNew; + if (paidMessageKVP.Value.Contains("cdn3.onlyfans.com/dash/files")) + { + string[] messageUrlParsed = paidMessageKVP.Value.Split(','); + string mpdURL = messageUrlParsed[0]; + string policy = messageUrlParsed[1]; + string signature = messageUrlParsed[2]; + string kvp = messageUrlParsed[3]; + string mediaId = messageUrlParsed[4]; + string messageId = messageUrlParsed[5]; + string? licenseURL = null; + string? pssh = await downloadContext.ApiHelper.GetDRMMPDPSSH(mpdURL, policy, signature, kvp); + if (pssh != null) + { + DateTime lastModified = await downloadContext.ApiHelper.GetDRMMPDLastModified(mpdURL, policy, signature, kvp); + Dictionary drmHeaders = downloadContext.ApiHelper.GetDynamicHeaders($"/api2/v2/users/media/{mediaId}/drm/message/{messageId}", "?type=widevine"); + string decryptionKey; + if (clientIdBlobMissing || devicePrivateKeyMissing) + { + decryptionKey = await downloadContext.ApiHelper.GetDecryptionKeyOFDL(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine", pssh); + } + else + { + decryptionKey = await downloadContext.ApiHelper.GetDecryptionKeyCDM(drmHeaders, $"https://onlyfans.com/api2/v2/users/media/{mediaId}/drm/message/{messageId}?type=widevine", pssh); + } + + Medium? mediaInfo = singlePaidMessageCollection.PreviewSingleMessageMedia.FirstOrDefault(m => m.id == paidMessageKVP.Key); + SingleMessage? messageInfo = singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p => p?.media?.Contains(mediaInfo) == true); + + isNew = await downloadContext.DownloadHelper.DownloadSingleMessagePreviewDRMVideo( + policy: policy, + signature: signature, + kvp: kvp, + url: mpdURL, + decryptionKey: decryptionKey, + folder: path, + lastModified: lastModified, + media_id: paidMessageKVP.Key, + api_type: "Messages", + task: task, + filenameFormat: downloadContext.FileNameFormatConfig.PaidMessageFileNameFormat ?? string.Empty, + messageInfo: messageInfo, + messageMedia: mediaInfo, + fromUser: messageInfo?.fromUser, + users: hasSelectedUsersKVP.Value); + + if (isNew) + { + newPreviewPaidMessagesCount++; + } + else + { + oldPreviewPaidMessagesCount++; + } + } + } + else + { + Medium? mediaInfo = singlePaidMessageCollection.PreviewSingleMessageMedia.FirstOrDefault(m => m.id == paidMessageKVP.Key); + SingleMessage? messageInfo = singlePaidMessageCollection.SingleMessageObjects.FirstOrDefault(p => p?.media?.Contains(mediaInfo) == true); + + isNew = await downloadContext.DownloadHelper.DownloadMessagePreviewMedia( + url: paidMessageKVP.Value, + folder: path, + media_id: paidMessageKVP.Key, + api_type: "Messages", + task: task, + filenameFormat: downloadContext.FileNameFormatConfig.PaidMessageFileNameFormat ?? string.Empty, + messageInfo: messageInfo, + messageMedia: mediaInfo, + fromUser: messageInfo?.fromUser, + users: hasSelectedUsersKVP.Value); + if (isNew) + { + newPreviewPaidMessagesCount++; + } + else + { + oldPreviewPaidMessagesCount++; + } + } + } + task.StopTask(); + }); + AnsiConsole.Markup($"[red]Preview Paid Messages Already Downloaded: {oldPreviewPaidMessagesCount} New Preview Paid Messages Downloaded: {newPreviewPaidMessagesCount}[/]\n"); + Log.Debug($"Preview Paid Messages Already Downloaded: {oldPreviewPaidMessagesCount} New Preview Paid Messages Downloaded: {newPreviewPaidMessagesCount}"); + } + if (singlePaidMessageCollection != null && singlePaidMessageCollection.SingleMessages.Count > 0) { AnsiConsole.Markup($"[red]Found {singlePaidMessageCollection.SingleMessages.Count} Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages\n[/]"); Log.Debug($"Found {singlePaidMessageCollection.SingleMessages.Count} Media from {singlePaidMessageCollection.SingleMessageObjects.Count} Paid Messages"); @@ -2992,7 +3112,9 @@ public class Program var hoconConfig = new StringBuilder(); hoconConfig.AppendLine("# Auth"); - hoconConfig.AppendLine($"DisableBrowserAuth = {newConfig.DisableBrowserAuth.ToString().ToLower()}"); + hoconConfig.AppendLine("Auth {"); + hoconConfig.AppendLine($" DisableBrowserAuth = {newConfig.DisableBrowserAuth.ToString().ToLower()}"); + hoconConfig.AppendLine("}"); hoconConfig.AppendLine("# External Tools"); hoconConfig.AppendLine("External {"); hoconConfig.AppendLine($" FFmpegPath = \"{newConfig.FFmpegPath}\""); @@ -3152,7 +3274,9 @@ public class Program var hoconConfig = new StringBuilder(); hoconConfig.AppendLine("# Auth"); - hoconConfig.AppendLine($"DisableBrowserAuth = {newConfig.DisableBrowserAuth.ToString().ToLower()}"); + hoconConfig.AppendLine("Auth {"); + hoconConfig.AppendLine($" DisableBrowserAuth = {newConfig.DisableBrowserAuth.ToString().ToLower()}"); + hoconConfig.AppendLine("}"); hoconConfig.AppendLine("# External Tools"); hoconConfig.AppendLine("External {"); hoconConfig.AppendLine($" FFmpegPath = \"{newConfig.FFmpegPath}\"");