Merge pull request 'feat: Add option to store raw post text without sanitization (Fix #52)' (#53) from wer/OF-DL:master into master

Reviewed-on: sim0n00ps/OF-DL#53

OF DL/Entities/Config.cs

@@ -103,6 +103,10 @@ namespace OF_DL.Entities
 
         [JsonConverter(typeof(StringEnumConverter))]
         public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
+
+        // When enabled, post/message text is stored as-is without XML stripping.
+        [ToggleableConfig]
+        public bool DisableTextSanitization { get; set; } = false;
     }
 
     public class CreatorConfig : IFileNameFormatConfig

OF DL/Helpers/APIHelper.cs

@@ -464,7 +464,7 @@ public class APIHelper : IAPIHelper
             Dictionary<string, string> getParams = new()
             {
                 { "offset", offset.ToString() },
-                { "limit", "50" }
+                { "limit", "50" },
             };
             List<string> users = new();
 
@@ -540,7 +540,8 @@ public class APIHelper : IAPIHelper
                     getParams = new Dictionary<string, string>
                     {
                         { "limit", post_limit.ToString() },
-                        { "order", "publish_date_desc" }
+                        { "order", "publish_date_desc" },
+                        { "skip_users", "all" }
                     };
                     break;
 
@@ -548,7 +549,8 @@ public class APIHelper : IAPIHelper
                     getParams = new Dictionary<string, string>
                     {
                         { "limit", limit.ToString() },
-                        { "offset", offset.ToString() }
+                        { "offset", offset.ToString() },
+                        { "skip_users", "all" }
                     };
                     break;
             }
@@ -738,9 +740,10 @@ public class APIHelper : IAPIHelper
             Dictionary<string, string> getParams = new()
             {
                 { "limit", post_limit.ToString() },
+                { "skip_users", "all" },
                 { "order", "publish_date_desc" },
                 { "format", "infinite" },
-                { "user_id", username }
+                { "author", username }
             };
 
             var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
@@ -906,7 +909,8 @@ public class APIHelper : IAPIHelper
             {
                 { "limit", post_limit.ToString() },
                 { "order", "publish_date_desc" },
-                { "format", "infinite" }
+                { "format", "infinite" },
+                { "skip_users", "all" }
             };
 
             Enumerations.DownloadDateSelection downloadDateSelection = Enumerations.DownloadDateSelection.before;
@@ -1237,7 +1241,8 @@ public class APIHelper : IAPIHelper
             {
                 { "limit", post_limit.ToString() },
                 { "order", "publish_date_desc" },
-                { "format", "infinite" }
+                { "format", "infinite" },
+                { "skip_users", "all" }
             };
 
             Enumerations.DownloadDateSelection downloadDateSelection = Enumerations.DownloadDateSelection.before;
@@ -1524,7 +1529,8 @@ public class APIHelper : IAPIHelper
             Dictionary<string, string> getParams = new()
             {
                 { "limit", post_limit.ToString() },
-                { "order", "desc" }
+                { "order", "desc" },
+                { "skip_users", "all" }
             };
 
             var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
@@ -1840,7 +1846,8 @@ public class APIHelper : IAPIHelper
                 { "limit", post_limit.ToString() },
                 { "order", "publish_date_desc" },
                 { "format", "infinite" },
-                { "user_id", username }
+                { "author", username },
+                { "skip_users", "all" }
             };
 
             var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
@@ -2805,11 +2812,11 @@ public class APIHelper : IAPIHelper
 
         try
         {
-            var resp1 = PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 });
+            var resp1 = await PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 });
             var certDataB64 = Convert.ToBase64String(resp1);
             var cdm = new CDMApi();
             var challenge = cdm.GetChallenge(pssh, certDataB64, false, false);
-            var resp2 = PostData(licenceURL, drmHeaders, challenge);
+            var resp2 = await PostData(licenceURL, drmHeaders, challenge);
             var licenseB64 = Convert.ToBase64String(resp2);
             Log.Debug($"resp1: {resp1}");
             Log.Debug($"certDataB64: {certDataB64}");

OF DL/Helpers/Constants.cs

@@ -3,4 +3,7 @@ namespace OF_DL.Helpers;
 public static class Constants
 {
     public const string API_URL = "https://onlyfans.com/api2/v2";
+
+    public const int WIDEVINE_RETRY_DELAY = 10;
+    public const int WIDEVINE_MAX_RETRIES = 3;
 }

OF DL/HttpUtil.cs

@@ -1,4 +1,5 @@
-using System;
+using OF_DL.Helpers;
+using System;
 using System.Collections.Generic;
 using System.Net.Http;
 using System.Text;
@@ -13,46 +14,66 @@ namespace WidevineClient
             //Proxy = null
         });
 
-        public static byte[] PostData(string URL, Dictionary<string, string> headers, string postData)
+        public static async Task<byte[]> PostData(string URL, Dictionary<string, string> headers, string postData)
         {
             var mediaType = postData.StartsWith("{") ? "application/json" : "application/x-www-form-urlencoded";
-            StringContent content = new StringContent(postData, Encoding.UTF8, mediaType);
+            var response = await PerformOperation(async () =>
-            //ByteArrayContent content = new ByteArrayContent(postData);
+            {
+                StringContent content = new StringContent(postData, Encoding.UTF8, mediaType);
+                //ByteArrayContent content = new ByteArrayContent(postData);
 
-            HttpResponseMessage response = Post(URL, headers, content);
+                return await Post(URL, headers, content);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
 
-        public static byte[] PostData(string URL, Dictionary<string, string> headers, byte[] postData)
+        public static async Task<byte[]> PostData(string URL, Dictionary<string, string> headers, byte[] postData)
         {
-            ByteArrayContent content = new ByteArrayContent(postData);
+            var response = await PerformOperation(async () =>
+            {
+                ByteArrayContent content = new ByteArrayContent(postData);
 
-            HttpResponseMessage response = Post(URL, headers, content);
+                return await Post(URL, headers, content);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
 
-        public static byte[] PostData(string URL, Dictionary<string, string> headers, Dictionary<string, string> postData)
+        public static async Task<byte[]> PostData(string URL, Dictionary<string, string> headers, Dictionary<string, string> postData)
         {
-            FormUrlEncodedContent content = new FormUrlEncodedContent(postData);
+            var response = await PerformOperation(async () =>
+            {
+                FormUrlEncodedContent content = new FormUrlEncodedContent(postData);
 
-            HttpResponseMessage response = Post(URL, headers, content);
+                return await Post(URL, headers, content);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
 
-        public static string GetWebSource(string URL, Dictionary<string, string> headers = null)
+        public static async Task<string> GetWebSource(string URL, Dictionary<string, string> headers = null)
         {
-            HttpResponseMessage response = Get(URL, headers);
+            var response = await PerformOperation(async () =>
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            {
+                return await Get(URL, headers);
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return Encoding.UTF8.GetString(bytes);
         }
 
-        public static byte[] GetBinary(string URL, Dictionary<string, string> headers = null)
+        public static async Task<byte[]> GetBinary(string URL, Dictionary<string, string> headers = null)
         {
-            HttpResponseMessage response = Get(URL, headers);
+            var response = await PerformOperation(async () =>
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            {
+                return await Get(URL, headers);
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
         public static string GetString(byte[] bytes)
@@ -60,7 +81,7 @@ namespace WidevineClient
             return Encoding.UTF8.GetString(bytes);
         }
 
-        static HttpResponseMessage Get(string URL, Dictionary<string, string> headers = null)
+        private static async Task<HttpResponseMessage> Get(string URL, Dictionary<string, string> headers = null)
         {
             HttpRequestMessage request = new HttpRequestMessage()
             {
@@ -72,10 +93,10 @@ namespace WidevineClient
                 foreach (KeyValuePair<string, string> header in headers)
                     request.Headers.TryAddWithoutValidation(header.Key, header.Value);
 
-            return Send(request);
+            return await Send(request);
         }
 
-        static HttpResponseMessage Post(string URL, Dictionary<string, string> headers, HttpContent content)
+        private static async Task<HttpResponseMessage> Post(string URL, Dictionary<string, string> headers, HttpContent content)
         {
             HttpRequestMessage request = new HttpRequestMessage()
             {
@@ -88,12 +109,41 @@ namespace WidevineClient
                 foreach (KeyValuePair<string, string> header in headers)
                     request.Headers.TryAddWithoutValidation(header.Key, header.Value);
 
-            return Send(request);
+            return await Send(request);
         }
 
-        static HttpResponseMessage Send(HttpRequestMessage request)
+        private static async Task<HttpResponseMessage> Send(HttpRequestMessage request)
         {
-            return Client.SendAsync(request).Result;
+            return await Client.SendAsync(request);
+        }
+
+        private static async Task<HttpResponseMessage> PerformOperation(Func<Task<HttpResponseMessage>> operation)
+        {
+            var response = await operation();
+
+            var retryCount = 0;
+
+            while (retryCount < Constants.WIDEVINE_MAX_RETRIES && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
+            {
+                //
+                // We've hit a rate limit, so we should wait before retrying.
+                //
+                var retryAfterSeconds = Constants.WIDEVINE_RETRY_DELAY * (retryCount + 1); // Default retry time. Increases with each retry.
+                if (response.Headers.RetryAfter != null && response.Headers.RetryAfter.Delta.HasValue)
+                {
+                    if (response.Headers.RetryAfter.Delta.Value.TotalSeconds > 0)
+                        retryAfterSeconds = (int)response.Headers.RetryAfter.Delta.Value.TotalSeconds + 1; // Add 1 second to ensure we wait a bit longer than the suggested time
+                }
+
+                await Task.Delay(retryAfterSeconds * 1000); // Peform the delay
+
+                response = await operation();
+                retryCount++;
+            }
+
+            response.EnsureSuccessStatusCode(); // Throw an exception if the response is not successful
+
+            return response;
         }
     }
 }

OF DL/Program.cs

@@ -160,6 +160,7 @@ public class Program
                         hoconConfig.AppendLine($"  DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
                         hoconConfig.AppendLine($"  CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
                         hoconConfig.AppendLine($"  ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = false");
                         hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
                         hoconConfig.AppendLine("}");
 
@@ -247,7 +248,7 @@ public class Program
 				{
 					string hoconText = File.ReadAllText("config.conf");
 
-					var hoconConfig = ConfigurationFactory.ParseString(hoconText);
+                    var hoconConfig = ConfigurationFactory.ParseString(hoconText);
 
 					config = new Entities.Config
 					{
@@ -279,7 +280,9 @@ public class Program
 						DownloadOnlySpecificDates = hoconConfig.GetBoolean("Download.DownloadOnlySpecificDates"),
 						DownloadDateSelection = Enum.Parse<DownloadDateSelection>(hoconConfig.GetString("Download.DownloadDateSelection"), true),
 						CustomDate = !string.IsNullOrWhiteSpace(hoconConfig.GetString("Download.CustomDate")) ? DateTime.Parse(hoconConfig.GetString("Download.CustomDate")) : null,
-						ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
+                        ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
+                        // Optional flag; default to false when missing
+                        DisableTextSanitization = bool.TryParse(hoconConfig.GetString("Download.DisableTextSanitization", "false"), out var dts) ? dts : false,
                         DownloadVideoResolution = ParseVideoResolution(hoconConfig.GetString("Download.DownloadVideoResolution", "source")),
 
                         // File Settings
@@ -344,7 +347,9 @@ public class Program
                         }
                     }
 
-					levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel;      //set the logging level based on config
+                    levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel;      //set the logging level based on config
+                    // Apply text sanitization preference globally
+                    OF_DL.Utils.XmlUtils.Passthrough = config.DisableTextSanitization;
 					Log.Debug("Configuration:");
 					string configString = JsonConvert.SerializeObject(config, Formatting.Indented);
 					Log.Debug(configString);
@@ -399,9 +404,11 @@ public class Program
                 hoconConfig.AppendLine($"  DownloadOnlySpecificDates = {jsonConfig.DownloadOnlySpecificDates.ToString().ToLower()}");
                 hoconConfig.AppendLine($"  DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
                 hoconConfig.AppendLine($"  CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
-                hoconConfig.AppendLine($"  ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
-                hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
+                        // New option defaults to false when converting legacy json
-                hoconConfig.AppendLine("}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = false");
+                        hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
+                        hoconConfig.AppendLine("}");
 
                 hoconConfig.AppendLine("# File Settings");
                 hoconConfig.AppendLine("File {");
@@ -1303,7 +1310,7 @@ public class Program
         await AnsiConsole.Status()
         .StartAsync("[red]Getting Paid Messages[/]", async ctx =>
         {
-            paidMessageCollection = await downloadContext.ApiHelper.GetPaidMessages("/posts/paid", path, user.Key, downloadContext.DownloadConfig!, ctx);
+            paidMessageCollection = await downloadContext.ApiHelper.GetPaidMessages("/posts/paid/chat", path, user.Key, downloadContext.DownloadConfig!, ctx);
         });
 		int oldPaidMessagesCount = 0;
 		int newPaidMessagesCount = 0;
@@ -1956,7 +1963,7 @@ public class Program
         await AnsiConsole.Status()
         .StartAsync("[red]Getting Paid Posts[/]", async ctx =>
         {
-            purchasedPosts = await downloadContext.ApiHelper.GetPaidPosts("/posts/paid", path, user.Key, downloadContext.DownloadConfig!, paid_post_ids, ctx);
+            purchasedPosts = await downloadContext.ApiHelper.GetPaidPosts("/posts/paid/post", path, user.Key, downloadContext.DownloadConfig!, paid_post_ids, ctx);
         });
 
 		int oldPaidPostCount = 0;
@@ -2904,6 +2911,7 @@ public class Program
                         hoconConfig.AppendLine($"  DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
                         hoconConfig.AppendLine($"  CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
                         hoconConfig.AppendLine($"  ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
                         hoconConfig.AppendLine("}");
 
@@ -3063,6 +3071,7 @@ public class Program
                         hoconConfig.AppendLine($"  DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
                         hoconConfig.AppendLine($"  CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
                         hoconConfig.AppendLine($"  ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
                         hoconConfig.AppendLine("}");
 

OF DL/Utils/XmlUtils.cs

@@ -9,8 +9,16 @@ namespace OF_DL.Utils
 {
     internal static class XmlUtils
     {
+        // When true, return original text without parsing/stripping.
+        public static bool Passthrough { get; set; } = false;
+
         public static string EvaluateInnerText(string xmlValue)
         {
+            if (Passthrough)
+            {
+                return xmlValue ?? string.Empty;
+            }
+
             try
             {
                 var parsedText = XElement.Parse($"<root>{xmlValue}</root>");

mkdocs.yml

@@ -2,6 +2,11 @@ site_name: OF-DL Docs
 site_url: https://docs.ofdl.tools
 nav:
   - Home: index.md
+  - Installation:
+    - Windows: installation/windows.md
+    - macOS: installation/macos.md
+    - Linux: installation/linux.md
+    - Docker: installation/docker.md
   - Running the Program: running-the-program.md
   - Config:
     - Authentication: config/auth.md
@@ -9,11 +14,6 @@ nav:
     - Configuration: config/configuration.md
     - All Configuration Options: config/all-configuration-options.md
     - Custom Filename Formats: config/custom-filename-formats.md
-  - Installation:
-    - Windows: installation/windows.md
-    - macOS: installation/macos.md
-    - Linux: installation/linux.md
-    - Docker: installation/docker.md
 theme:
   name: material
   features: