diff --git a/OF DL/Entities/Config.cs b/OF DL/Entities/Config.cs index 0d05fbf..0934422 100644 --- a/OF DL/Entities/Config.cs +++ b/OF DL/Entities/Config.cs @@ -103,6 +103,10 @@ namespace OF_DL.Entities [JsonConverter(typeof(StringEnumConverter))] public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source; + + // When enabled, post/message text is stored as-is without XML stripping. + [ToggleableConfig] + public bool DisableTextSanitization { get; set; } = false; } public class CreatorConfig : IFileNameFormatConfig diff --git a/OF DL/Helpers/APIHelper.cs b/OF DL/Helpers/APIHelper.cs index c368794..7daf3f1 100644 --- a/OF DL/Helpers/APIHelper.cs +++ b/OF DL/Helpers/APIHelper.cs @@ -464,7 +464,7 @@ public class APIHelper : IAPIHelper Dictionary getParams = new() { { "offset", offset.ToString() }, - { "limit", "50" } + { "limit", "50" }, }; List users = new(); @@ -540,7 +540,8 @@ public class APIHelper : IAPIHelper getParams = new Dictionary { { "limit", post_limit.ToString() }, - { "order", "publish_date_desc" } + { "order", "publish_date_desc" }, + { "skip_users", "all" } }; break; @@ -548,7 +549,8 @@ public class APIHelper : IAPIHelper getParams = new Dictionary { { "limit", limit.ToString() }, - { "offset", offset.ToString() } + { "offset", offset.ToString() }, + { "skip_users", "all" } }; break; } @@ -738,9 +740,10 @@ public class APIHelper : IAPIHelper Dictionary getParams = new() { { "limit", post_limit.ToString() }, + { "skip_users", "all" }, { "order", "publish_date_desc" }, { "format", "infinite" }, - { "user_id", username } + { "author", username } }; var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); @@ -906,7 +909,8 @@ public class APIHelper : IAPIHelper { { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, - { "format", "infinite" } + { "format", "infinite" }, + { "skip_users", "all" } }; Enumerations.DownloadDateSelection downloadDateSelection = Enumerations.DownloadDateSelection.before; @@ -1237,7 +1241,8 @@ public class APIHelper : IAPIHelper { { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, - { "format", "infinite" } + { "format", "infinite" }, + { "skip_users", "all" } }; Enumerations.DownloadDateSelection downloadDateSelection = Enumerations.DownloadDateSelection.before; @@ -1524,7 +1529,8 @@ public class APIHelper : IAPIHelper Dictionary getParams = new() { { "limit", post_limit.ToString() }, - { "order", "desc" } + { "order", "desc" }, + { "skip_users", "all" } }; var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); @@ -1840,7 +1846,8 @@ public class APIHelper : IAPIHelper { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, - { "user_id", username } + { "author", username }, + { "skip_users", "all" } }; var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); @@ -2805,11 +2812,11 @@ public class APIHelper : IAPIHelper try { - var resp1 = PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 }); + var resp1 = await PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 }); var certDataB64 = Convert.ToBase64String(resp1); var cdm = new CDMApi(); var challenge = cdm.GetChallenge(pssh, certDataB64, false, false); - var resp2 = PostData(licenceURL, drmHeaders, challenge); + var resp2 = await PostData(licenceURL, drmHeaders, challenge); var licenseB64 = Convert.ToBase64String(resp2); Log.Debug($"resp1: {resp1}"); Log.Debug($"certDataB64: {certDataB64}"); diff --git a/OF DL/Helpers/Constants.cs b/OF DL/Helpers/Constants.cs index 857d1e3..01a410d 100644 --- a/OF DL/Helpers/Constants.cs +++ b/OF DL/Helpers/Constants.cs @@ -3,4 +3,7 @@ namespace OF_DL.Helpers; public static class Constants { public const string API_URL = "https://onlyfans.com/api2/v2"; + + public const int WIDEVINE_RETRY_DELAY = 10; + public const int WIDEVINE_MAX_RETRIES = 3; } diff --git a/OF DL/HttpUtil.cs b/OF DL/HttpUtil.cs index 89d1214..25f0ebe 100644 --- a/OF DL/HttpUtil.cs +++ b/OF DL/HttpUtil.cs @@ -1,4 +1,5 @@ -using System; +using OF_DL.Helpers; +using System; using System.Collections.Generic; using System.Net.Http; using System.Text; @@ -13,46 +14,66 @@ namespace WidevineClient //Proxy = null }); - public static byte[] PostData(string URL, Dictionary headers, string postData) + public static async Task PostData(string URL, Dictionary headers, string postData) { var mediaType = postData.StartsWith("{") ? "application/json" : "application/x-www-form-urlencoded"; - StringContent content = new StringContent(postData, Encoding.UTF8, mediaType); - //ByteArrayContent content = new ByteArrayContent(postData); + var response = await PerformOperation(async () => + { + StringContent content = new StringContent(postData, Encoding.UTF8, mediaType); + //ByteArrayContent content = new ByteArrayContent(postData); - HttpResponseMessage response = Post(URL, headers, content); - byte[] bytes = response.Content.ReadAsByteArrayAsync().Result; + return await Post(URL, headers, content); + }); + + byte[] bytes = await response.Content.ReadAsByteArrayAsync(); return bytes; } - public static byte[] PostData(string URL, Dictionary headers, byte[] postData) + public static async Task PostData(string URL, Dictionary headers, byte[] postData) { - ByteArrayContent content = new ByteArrayContent(postData); + var response = await PerformOperation(async () => + { + ByteArrayContent content = new ByteArrayContent(postData); - HttpResponseMessage response = Post(URL, headers, content); - byte[] bytes = response.Content.ReadAsByteArrayAsync().Result; + return await Post(URL, headers, content); + }); + + byte[] bytes = await response.Content.ReadAsByteArrayAsync(); return bytes; } - public static byte[] PostData(string URL, Dictionary headers, Dictionary postData) + public static async Task PostData(string URL, Dictionary headers, Dictionary postData) { - FormUrlEncodedContent content = new FormUrlEncodedContent(postData); + var response = await PerformOperation(async () => + { + FormUrlEncodedContent content = new FormUrlEncodedContent(postData); - HttpResponseMessage response = Post(URL, headers, content); - byte[] bytes = response.Content.ReadAsByteArrayAsync().Result; + return await Post(URL, headers, content); + }); + + byte[] bytes = await response.Content.ReadAsByteArrayAsync(); return bytes; } - public static string GetWebSource(string URL, Dictionary headers = null) + public static async Task GetWebSource(string URL, Dictionary headers = null) { - HttpResponseMessage response = Get(URL, headers); - byte[] bytes = response.Content.ReadAsByteArrayAsync().Result; + var response = await PerformOperation(async () => + { + return await Get(URL, headers); + }); + + byte[] bytes = await response.Content.ReadAsByteArrayAsync(); return Encoding.UTF8.GetString(bytes); } - public static byte[] GetBinary(string URL, Dictionary headers = null) + public static async Task GetBinary(string URL, Dictionary headers = null) { - HttpResponseMessage response = Get(URL, headers); - byte[] bytes = response.Content.ReadAsByteArrayAsync().Result; + var response = await PerformOperation(async () => + { + return await Get(URL, headers); + }); + + byte[] bytes = await response.Content.ReadAsByteArrayAsync(); return bytes; } public static string GetString(byte[] bytes) @@ -60,7 +81,7 @@ namespace WidevineClient return Encoding.UTF8.GetString(bytes); } - static HttpResponseMessage Get(string URL, Dictionary headers = null) + private static async Task Get(string URL, Dictionary headers = null) { HttpRequestMessage request = new HttpRequestMessage() { @@ -72,10 +93,10 @@ namespace WidevineClient foreach (KeyValuePair header in headers) request.Headers.TryAddWithoutValidation(header.Key, header.Value); - return Send(request); + return await Send(request); } - static HttpResponseMessage Post(string URL, Dictionary headers, HttpContent content) + private static async Task Post(string URL, Dictionary headers, HttpContent content) { HttpRequestMessage request = new HttpRequestMessage() { @@ -88,12 +109,41 @@ namespace WidevineClient foreach (KeyValuePair header in headers) request.Headers.TryAddWithoutValidation(header.Key, header.Value); - return Send(request); + return await Send(request); } - static HttpResponseMessage Send(HttpRequestMessage request) + private static async Task Send(HttpRequestMessage request) { - return Client.SendAsync(request).Result; + return await Client.SendAsync(request); + } + + private static async Task PerformOperation(Func> operation) + { + var response = await operation(); + + var retryCount = 0; + + while (retryCount < Constants.WIDEVINE_MAX_RETRIES && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests) + { + // + // We've hit a rate limit, so we should wait before retrying. + // + var retryAfterSeconds = Constants.WIDEVINE_RETRY_DELAY * (retryCount + 1); // Default retry time. Increases with each retry. + if (response.Headers.RetryAfter != null && response.Headers.RetryAfter.Delta.HasValue) + { + if (response.Headers.RetryAfter.Delta.Value.TotalSeconds > 0) + retryAfterSeconds = (int)response.Headers.RetryAfter.Delta.Value.TotalSeconds + 1; // Add 1 second to ensure we wait a bit longer than the suggested time + } + + await Task.Delay(retryAfterSeconds * 1000); // Peform the delay + + response = await operation(); + retryCount++; + } + + response.EnsureSuccessStatusCode(); // Throw an exception if the response is not successful + + return response; } } } diff --git a/OF DL/Program.cs b/OF DL/Program.cs index 2073e23..49bafb9 100644 --- a/OF DL/Program.cs +++ b/OF DL/Program.cs @@ -160,6 +160,7 @@ public class Program hoconConfig.AppendLine($" DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\""); hoconConfig.AppendLine($" CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\""); hoconConfig.AppendLine($" ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}"); + hoconConfig.AppendLine($" DisableTextSanitization = false"); hoconConfig.AppendLine($" DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\""); hoconConfig.AppendLine("}"); @@ -247,7 +248,7 @@ public class Program { string hoconText = File.ReadAllText("config.conf"); - var hoconConfig = ConfigurationFactory.ParseString(hoconText); + var hoconConfig = ConfigurationFactory.ParseString(hoconText); config = new Entities.Config { @@ -279,7 +280,9 @@ public class Program DownloadOnlySpecificDates = hoconConfig.GetBoolean("Download.DownloadOnlySpecificDates"), DownloadDateSelection = Enum.Parse(hoconConfig.GetString("Download.DownloadDateSelection"), true), CustomDate = !string.IsNullOrWhiteSpace(hoconConfig.GetString("Download.CustomDate")) ? DateTime.Parse(hoconConfig.GetString("Download.CustomDate")) : null, - ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"), + ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"), + // Optional flag; default to false when missing + DisableTextSanitization = bool.TryParse(hoconConfig.GetString("Download.DisableTextSanitization", "false"), out var dts) ? dts : false, DownloadVideoResolution = ParseVideoResolution(hoconConfig.GetString("Download.DownloadVideoResolution", "source")), // File Settings @@ -344,7 +347,9 @@ public class Program } } - levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel; //set the logging level based on config + levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel; //set the logging level based on config + // Apply text sanitization preference globally + OF_DL.Utils.XmlUtils.Passthrough = config.DisableTextSanitization; Log.Debug("Configuration:"); string configString = JsonConvert.SerializeObject(config, Formatting.Indented); Log.Debug(configString); @@ -399,9 +404,11 @@ public class Program hoconConfig.AppendLine($" DownloadOnlySpecificDates = {jsonConfig.DownloadOnlySpecificDates.ToString().ToLower()}"); hoconConfig.AppendLine($" DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\""); hoconConfig.AppendLine($" CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\""); - hoconConfig.AppendLine($" ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}"); - hoconConfig.AppendLine($" DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\""); - hoconConfig.AppendLine("}"); + hoconConfig.AppendLine($" ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}"); + // New option defaults to false when converting legacy json + hoconConfig.AppendLine($" DisableTextSanitization = false"); + hoconConfig.AppendLine($" DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\""); + hoconConfig.AppendLine("}"); hoconConfig.AppendLine("# File Settings"); hoconConfig.AppendLine("File {"); @@ -1311,7 +1318,7 @@ public class Program await AnsiConsole.Status() .StartAsync("[red]Getting Paid Messages[/]", async ctx => { - paidMessageCollection = await downloadContext.ApiHelper.GetPaidMessages("/posts/paid", path, user.Key, downloadContext.DownloadConfig!, ctx); + paidMessageCollection = await downloadContext.ApiHelper.GetPaidMessages("/posts/paid/chat", path, user.Key, downloadContext.DownloadConfig!, ctx); }); int oldPaidMessagesCount = 0; int newPaidMessagesCount = 0; @@ -1964,7 +1971,7 @@ public class Program await AnsiConsole.Status() .StartAsync("[red]Getting Paid Posts[/]", async ctx => { - purchasedPosts = await downloadContext.ApiHelper.GetPaidPosts("/posts/paid", path, user.Key, downloadContext.DownloadConfig!, paid_post_ids, ctx); + purchasedPosts = await downloadContext.ApiHelper.GetPaidPosts("/posts/paid/post", path, user.Key, downloadContext.DownloadConfig!, paid_post_ids, ctx); }); int oldPaidPostCount = 0; @@ -2912,6 +2919,7 @@ public class Program hoconConfig.AppendLine($" DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\""); hoconConfig.AppendLine($" CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\""); hoconConfig.AppendLine($" ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}"); + hoconConfig.AppendLine($" DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}"); hoconConfig.AppendLine($" DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\""); hoconConfig.AppendLine("}"); @@ -3071,6 +3079,7 @@ public class Program hoconConfig.AppendLine($" DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\""); hoconConfig.AppendLine($" CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\""); hoconConfig.AppendLine($" ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}"); + hoconConfig.AppendLine($" DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}"); hoconConfig.AppendLine($" DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\""); hoconConfig.AppendLine("}"); diff --git a/OF DL/Utils/XmlUtils.cs b/OF DL/Utils/XmlUtils.cs index d55be10..55a68c6 100644 --- a/OF DL/Utils/XmlUtils.cs +++ b/OF DL/Utils/XmlUtils.cs @@ -9,8 +9,16 @@ namespace OF_DL.Utils { internal static class XmlUtils { + // When true, return original text without parsing/stripping. + public static bool Passthrough { get; set; } = false; + public static string EvaluateInnerText(string xmlValue) { + if (Passthrough) + { + return xmlValue ?? string.Empty; + } + try { var parsedText = XElement.Parse($"{xmlValue}");