Added safe-guard against paid content returned for other models

Tweaked order to fully replace users before updating DB.

OF DL/Entities/Config.cs

@@ -104,6 +104,10 @@ namespace OF_DL.Entities
         [JsonConverter(typeof(StringEnumConverter))]
         public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
 
+        // When enabled, post/message text is stored as-is without XML stripping.
+        [ToggleableConfig]
+        public bool DisableTextSanitization { get; set; } = false;
+
         public string[] NonInteractiveSpecificUsers { get; set; } = [];
         public string[] NonInteractiveSpecificLists { get; set; } = [];
 

OF DL/Helpers/APIHelper.cs

@@ -571,7 +571,7 @@ public class APIHelper : IAPIHelper
             Dictionary<string, string> getParams = new()
             {
                 { "offset", offset.ToString() },
-                { "limit", "50" }
+                { "limit", "50" },
             };
             List<string> users = new();
 
@@ -707,7 +707,8 @@ public class APIHelper : IAPIHelper
                     getParams = new Dictionary<string, string>
                     {
                         { "limit", post_limit.ToString() },
-                        { "order", "publish_date_desc" }
+                        { "order", "publish_date_desc" },
+                        { "skip_users", "all" }
                     };
                     break;
 
@@ -715,7 +716,8 @@ public class APIHelper : IAPIHelper
                     getParams = new Dictionary<string, string>
                     {
                         { "limit", limit.ToString() },
-                        { "offset", offset.ToString() }
+                        { "offset", offset.ToString() },
+                        { "skip_users", "all" }
                     };
                     break;
             }
@@ -908,7 +910,6 @@ public class APIHelper : IAPIHelper
                 { "limit", post_limit.ToString() },
                 { "skip_users", "all" },
                 { "format", "infinite" },
-                { "offset", offset.ToString() },
                 { "author", username },
             };
 
@@ -1078,7 +1079,8 @@ public class APIHelper : IAPIHelper
             {
                 { "limit", post_limit.ToString() },
                 { "order", "publish_date_desc" },
-                { "format", "infinite" }
+                { "format", "infinite" },
+                { "skip_users", "all" }
             };
 
             Enumerations.DownloadDateSelection downloadDateSelection = Enumerations.DownloadDateSelection.before;
@@ -1409,7 +1411,8 @@ public class APIHelper : IAPIHelper
             {
                 { "limit", post_limit.ToString() },
                 { "order", "publish_date_desc" },
-                { "format", "infinite" }
+                { "format", "infinite" },
+                { "skip_users", "all" }
             };
 
             Enumerations.DownloadDateSelection downloadDateSelection = Enumerations.DownloadDateSelection.before;
@@ -3077,11 +3080,11 @@ public class APIHelper : IAPIHelper
 
         try
         {
-            var resp1 = PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 });
+            var resp1 = await PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 });
             var certDataB64 = Convert.ToBase64String(resp1);
             var cdm = new CDMApi();
             var challenge = cdm.GetChallenge(pssh, certDataB64, false, false);
-            var resp2 = PostData(licenceURL, drmHeaders, challenge);
+            var resp2 = await PostData(licenceURL, drmHeaders, challenge);
             var licenseB64 = Convert.ToBase64String(resp2);
             Log.Debug($"resp1: {resp1}");
             Log.Debug($"certDataB64: {certDataB64}");

OF DL/Helpers/Constants.cs

@@ -3,4 +3,7 @@ namespace OF_DL.Helpers;
 public static class Constants
 {
     public const string API_URL = "https://onlyfans.com/api2/v2";
+
+    public const int WIDEVINE_RETRY_DELAY = 10;
+    public const int WIDEVINE_MAX_RETRIES = 3;
 }

OF DL/HttpUtil.cs

@@ -1,4 +1,5 @@
-using System;
+using OF_DL.Helpers;
+using System;
 using System.Collections.Generic;
 using System.Net.Http;
 using System.Text;
@@ -13,46 +14,66 @@ namespace WidevineClient
             //Proxy = null
         });
 
-        public static byte[] PostData(string URL, Dictionary<string, string> headers, string postData)
+        public static async Task<byte[]> PostData(string URL, Dictionary<string, string> headers, string postData)
         {
             var mediaType = postData.StartsWith("{") ? "application/json" : "application/x-www-form-urlencoded";
-            StringContent content = new StringContent(postData, Encoding.UTF8, mediaType);
-            //ByteArrayContent content = new ByteArrayContent(postData);
+            var response = await PerformOperation(async () =>
+            {
+                StringContent content = new StringContent(postData, Encoding.UTF8, mediaType);
+                //ByteArrayContent content = new ByteArrayContent(postData);
 
-            HttpResponseMessage response = Post(URL, headers, content);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+                return await Post(URL, headers, content);
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
 
-        public static byte[] PostData(string URL, Dictionary<string, string> headers, byte[] postData)
+        public static async Task<byte[]> PostData(string URL, Dictionary<string, string> headers, byte[] postData)
         {
-            ByteArrayContent content = new ByteArrayContent(postData);
+            var response = await PerformOperation(async () =>
+            {
+                ByteArrayContent content = new ByteArrayContent(postData);
 
-            HttpResponseMessage response = Post(URL, headers, content);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+                return await Post(URL, headers, content);
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
 
-        public static byte[] PostData(string URL, Dictionary<string, string> headers, Dictionary<string, string> postData)
+        public static async Task<byte[]> PostData(string URL, Dictionary<string, string> headers, Dictionary<string, string> postData)
         {
-            FormUrlEncodedContent content = new FormUrlEncodedContent(postData);
+            var response = await PerformOperation(async () =>
+            {
+                FormUrlEncodedContent content = new FormUrlEncodedContent(postData);
 
-            HttpResponseMessage response = Post(URL, headers, content);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+                return await Post(URL, headers, content);
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
 
-        public static string GetWebSource(string URL, Dictionary<string, string> headers = null)
+        public static async Task<string> GetWebSource(string URL, Dictionary<string, string> headers = null)
         {
-            HttpResponseMessage response = Get(URL, headers);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            var response = await PerformOperation(async () =>
+            {
+                return await Get(URL, headers);
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return Encoding.UTF8.GetString(bytes);
         }
 
-        public static byte[] GetBinary(string URL, Dictionary<string, string> headers = null)
+        public static async Task<byte[]> GetBinary(string URL, Dictionary<string, string> headers = null)
         {
-            HttpResponseMessage response = Get(URL, headers);
-            byte[] bytes = response.Content.ReadAsByteArrayAsync().Result;
+            var response = await PerformOperation(async () =>
+            {
+                return await Get(URL, headers);
+            });
+
+            byte[] bytes = await response.Content.ReadAsByteArrayAsync();
             return bytes;
         }
         public static string GetString(byte[] bytes)
@@ -60,7 +81,7 @@ namespace WidevineClient
             return Encoding.UTF8.GetString(bytes);
         }
 
-        static HttpResponseMessage Get(string URL, Dictionary<string, string> headers = null)
+        private static async Task<HttpResponseMessage> Get(string URL, Dictionary<string, string> headers = null)
         {
             HttpRequestMessage request = new HttpRequestMessage()
             {
@@ -72,10 +93,10 @@ namespace WidevineClient
                 foreach (KeyValuePair<string, string> header in headers)
                     request.Headers.TryAddWithoutValidation(header.Key, header.Value);
 
-            return Send(request);
+            return await Send(request);
         }
 
-        static HttpResponseMessage Post(string URL, Dictionary<string, string> headers, HttpContent content)
+        private static async Task<HttpResponseMessage> Post(string URL, Dictionary<string, string> headers, HttpContent content)
         {
             HttpRequestMessage request = new HttpRequestMessage()
             {
@@ -88,12 +109,41 @@ namespace WidevineClient
                 foreach (KeyValuePair<string, string> header in headers)
                     request.Headers.TryAddWithoutValidation(header.Key, header.Value);
 
-            return Send(request);
+            return await Send(request);
         }
 
-        static HttpResponseMessage Send(HttpRequestMessage request)
+        private static async Task<HttpResponseMessage> Send(HttpRequestMessage request)
         {
-            return Client.SendAsync(request).Result;
+            return await Client.SendAsync(request);
+        }
+
+        private static async Task<HttpResponseMessage> PerformOperation(Func<Task<HttpResponseMessage>> operation)
+        {
+            var response = await operation();
+
+            var retryCount = 0;
+
+            while (retryCount < Constants.WIDEVINE_MAX_RETRIES && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
+            {
+                //
+                // We've hit a rate limit, so we should wait before retrying.
+                //
+                var retryAfterSeconds = Constants.WIDEVINE_RETRY_DELAY * (retryCount + 1); // Default retry time. Increases with each retry.
+                if (response.Headers.RetryAfter != null && response.Headers.RetryAfter.Delta.HasValue)
+                {
+                    if (response.Headers.RetryAfter.Delta.Value.TotalSeconds > 0)
+                        retryAfterSeconds = (int)response.Headers.RetryAfter.Delta.Value.TotalSeconds + 1; // Add 1 second to ensure we wait a bit longer than the suggested time
+                }
+
+                await Task.Delay(retryAfterSeconds * 1000); // Peform the delay
+
+                response = await operation();
+                retryCount++;
+            }
+
+            response.EnsureSuccessStatusCode(); // Throw an exception if the response is not successful
+
+            return response;
         }
     }
 }

OF DL/Program.cs

@@ -162,6 +162,7 @@ public class Program
                         hoconConfig.AppendLine($"  DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
                         hoconConfig.AppendLine($"  CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
                         hoconConfig.AppendLine($"  ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = false");
                         hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
                         hoconConfig.AppendLine("}");
 
@@ -249,7 +250,7 @@ public class Program
 				{
 					string hoconText = await File.ReadAllTextAsync("config.conf");
 
-					var hoconConfig = ConfigurationFactory.ParseString(hoconText);
+                    var hoconConfig = ConfigurationFactory.ParseString(hoconText);
 
 					config = new Entities.Config
 					{
@@ -281,7 +282,9 @@ public class Program
 						DownloadOnlySpecificDates = hoconConfig.GetBoolean("Download.DownloadOnlySpecificDates"),
 						DownloadDateSelection = Enum.Parse<DownloadDateSelection>(hoconConfig.GetString("Download.DownloadDateSelection"), true),
 						CustomDate = !string.IsNullOrWhiteSpace(hoconConfig.GetString("Download.CustomDate")) ? DateTime.Parse(hoconConfig.GetString("Download.CustomDate")) : null,
-						ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
+                        ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
+                        // Optional flag; default to false when missing
+                        DisableTextSanitization = bool.TryParse(hoconConfig.GetString("Download.DisableTextSanitization", "false"), out var dts) ? dts : false,
                         DownloadVideoResolution = ParseVideoResolution(hoconConfig.GetString("Download.DownloadVideoResolution", "source")),
 
                         // File Settings
@@ -346,7 +349,9 @@ public class Program
                         }
                     }
 
-					levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel;      //set the logging level based on config
+                    levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel;      //set the logging level based on config
+                    // Apply text sanitization preference globally
+                    OF_DL.Utils.XmlUtils.Passthrough = config.DisableTextSanitization;
 					Log.Debug("Configuration:");
 					string configString = JsonConvert.SerializeObject(config, Formatting.Indented);
 					Log.Debug(configString);
@@ -401,9 +406,11 @@ public class Program
                 hoconConfig.AppendLine($"  DownloadOnlySpecificDates = {jsonConfig.DownloadOnlySpecificDates.ToString().ToLower()}");
                 hoconConfig.AppendLine($"  DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
                 hoconConfig.AppendLine($"  CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
-                hoconConfig.AppendLine($"  ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
-                hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
-                hoconConfig.AppendLine("}");
+                        hoconConfig.AppendLine($"  ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
+                        // New option defaults to false when converting legacy json
+                        hoconConfig.AppendLine($"  DisableTextSanitization = false");
+                        hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
+                        hoconConfig.AppendLine("}");
 
                 hoconConfig.AppendLine("# File Settings");
                 hoconConfig.AppendLine("File {");
@@ -3039,6 +3046,7 @@ public class Program
                         hoconConfig.AppendLine($"  DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
                         hoconConfig.AppendLine($"  CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
                         hoconConfig.AppendLine($"  ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
                         hoconConfig.AppendLine("}");
 
@@ -3198,6 +3206,7 @@ public class Program
                         hoconConfig.AppendLine($"  DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
                         hoconConfig.AppendLine($"  CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
                         hoconConfig.AppendLine($"  ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
+                        hoconConfig.AppendLine($"  DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
                         hoconConfig.AppendLine($"  DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
                         hoconConfig.AppendLine("}");
 

OF DL/Utils/XmlUtils.cs

@@ -9,8 +9,16 @@ namespace OF_DL.Utils
 {
     internal static class XmlUtils
     {
+        // When true, return original text without parsing/stripping.
+        public static bool Passthrough { get; set; } = false;
+
         public static string EvaluateInnerText(string xmlValue)
         {
+            if (Passthrough)
+            {
+                return xmlValue ?? string.Empty;
+            }
+
             try
             {
                 var parsedText = XElement.Parse($"<root>{xmlValue}</root>");