merge upstream

Reviewed-on: sim0n00ps/OF-DL#84

OF DL/Helpers/APIHelper.cs

@@ -31,6 +31,8 @@ public class APIHelper : IAPIHelper
     private readonly Auth auth;
     private static DateTime? cachedDynamicRulesExpiration;
     private static DynamicRules? cachedDynamicRules;
+    private const int MaxAttempts = 30;
+    private const int DelayBetweenAttempts = 3000;
 
     static APIHelper()
     {
@@ -1181,7 +1183,7 @@ public class APIHelper : IAPIHelper
                                         }
                                     }
                                     break;
-                                
+
                             }
                         }
                         else if (medium.canView && medium.files != null && medium.files.drm != null)
@@ -2078,7 +2080,8 @@ public class APIHelper : IAPIHelper
             {
                 { "limit", post_limit.ToString() },
                 { "order", "publish_date_desc" },
-                { "format", "infinite" }
+                { "format", "infinite" },
+                { "skip_users", "all" }
             };
 
             var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
@@ -2251,7 +2254,8 @@ public class APIHelper : IAPIHelper
             {
                 { "limit", post_limit.ToString() },
                 { "order", "publish_date_desc" },
-                { "format", "infinite" }
+                { "format", "infinite" },
+                { "skip_users", "all" }
             };
 
             var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
@@ -2672,13 +2676,10 @@ public class APIHelper : IAPIHelper
         return DateTime.Now;
     }
 
-
     public async Task<string> GetDecryptionKeyCDRMProject(Dictionary<string, string> drmHeaders, string licenceURL, string pssh)
     {
         Log.Debug("Calling GetDecryptionKey");
 
-        const int maxAttempts = 30;
-        const int delayBetweenAttempts = 3000;
         int attempt = 0;
 
         try
@@ -2699,7 +2700,7 @@ public class APIHelper : IAPIHelper
 
             Log.Debug($"Posting to CDRM Project: {json}");
 
-            while (attempt < maxAttempts)
+            while (attempt < MaxAttempts)
             {
                 attempt++;
 
@@ -2725,19 +2726,19 @@ public class APIHelper : IAPIHelper
                     }
                     else
                     {
-                        Log.Debug($"CDRM response status not successful. Retrying... Attempt {attempt} of {maxAttempts}");
-                        if (attempt < maxAttempts)
+                        Log.Debug($"CDRM response status not successful. Retrying... Attempt {attempt} of {MaxAttempts}");
+                        if (attempt < MaxAttempts)
                         {
-                            await Task.Delay(delayBetweenAttempts);
+                            await Task.Delay(DelayBetweenAttempts);
                         }
                     }
                 }
                 else
                 {
-                    Log.Debug($"Status not in CDRM response. Retrying... Attempt {attempt} of {maxAttempts}");
-                    if (attempt < maxAttempts)
+                    Log.Debug($"Status not in CDRM response. Retrying... Attempt {attempt} of {MaxAttempts}");
+                    if (attempt < MaxAttempts)
                     {
-                        await Task.Delay(delayBetweenAttempts);
+                        await Task.Delay(DelayBetweenAttempts);
                     }
                 }
             }
@@ -2762,11 +2763,10 @@ public class APIHelper : IAPIHelper
     {
         Log.Debug("Calling GetDecryptionOFDL");
 
-
         try
         {
-            string dcValue = string.Empty;
             HttpClient client = new();
+            int attempt = 0;
 
             OFDLRequest ofdlRequest = new OFDLRequest
             {
@@ -2779,17 +2779,27 @@ public class APIHelper : IAPIHelper
 
             Log.Debug($"Posting to ofdl.tools: {json}");
 
-            HttpRequestMessage request = new(HttpMethod.Post, "https://ofdl.tools/WV")
+            while (attempt < MaxAttempts)
             {
-                Content = new StringContent(json, Encoding.UTF8, "application/json")
-            };
+                attempt++;
 
-            using var response = await client.SendAsync(request);
+                HttpRequestMessage request = new(HttpMethod.Post, "https://ofdl.tools/WV")
+                {
+                    Content = new StringContent(json, Encoding.UTF8, "application/json")
+                };
+
+                using var response = await client.SendAsync(request);
+
+                if (!response.IsSuccessStatusCode)
+                    continue;
 
-            if (response.IsSuccessStatusCode)
-            {
                 string body = await response.Content.ReadAsStringAsync();
-                return body;
+
+                if (!body.TrimStart().StartsWith('{'))
+                    return body;
+
+                Log.Debug($"Received JSON object instead of string. Retrying... Attempt {attempt} of {MaxAttempts}");
+                await Task.Delay(DelayBetweenAttempts);
             }
         }
         catch (Exception ex)
@@ -2803,6 +2813,7 @@ public class APIHelper : IAPIHelper
                 Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
             }
         }
+
         return null;
     }
 

OF DL/Helpers/VersionHelper.cs

@@ -6,14 +6,15 @@ namespace OF_DL.Helpers;
 
 public static class VersionHelper
 {
-    public static string? GetLatestReleaseTag()
+    private static readonly HttpClient httpClient = new HttpClient();
+    private const string url = "https://git.ofdl.tools/api/v1/repos/sim0n00ps/OF-DL/releases/latest";
+
+    public static async Task<string?> GetLatestReleaseTag(CancellationToken cancellationToken = default)
     {
         Log.Debug("Calling GetLatestReleaseTag");
         try
         {
-            HttpClient client = new();
-            HttpRequestMessage request = new(HttpMethod.Get, "https://git.ofdl.tools/api/v1/repos/sim0n00ps/OF-DL/releases/latest");
-            using var response = client.Send(request);
+            var response = await httpClient.GetAsync(url, cancellationToken);
 
             if (!response.IsSuccessStatusCode)
             {
@@ -21,7 +22,7 @@ public static class VersionHelper
                 return null;
             }
 
-            var body = response.Content.ReadAsStringAsync().Result;
+            var body = await response.Content.ReadAsStringAsync();
 
             Log.Debug("GetLatestReleaseTag API Response: ");
             Log.Debug(body);
@@ -36,6 +37,10 @@ public static class VersionHelper
 
             return versionCheckResponse.TagName;
         }
+        catch (OperationCanceledException)
+        {
+            throw; // Rethrow timeout exceptions to be handled by the caller
+        }
         catch (Exception ex)
         {
             Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);

OF DL/Program.cs

@@ -526,36 +526,50 @@ public class Program
 				// Only run the version check if not in DEBUG mode
 				#if !DEBUG
 				Version localVersion = Assembly.GetEntryAssembly()?.GetName().Version; //Only tested with numeric values.
-                String? latestReleaseTag = VersionHelper.GetLatestReleaseTag();
 
-                if (latestReleaseTag == null)
-                {
-                    AnsiConsole.Markup("[yellow]Failed to verify that OF-DL is up-to-date.\n[/]");
-                    Log.Error("Failed to get the latest release tag.");
-                }
-                else
-                {
-                    Version latestGiteaRelease = new Version(latestReleaseTag.Replace("OFDLV", ""));
+				// Create a cancellation token with 30 second timeout
+				using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30));
+				String? latestReleaseTag = null;
 
-                    // Compare the Versions
-                    int versionComparison = localVersion.CompareTo(latestGiteaRelease);
-                    if (versionComparison < 0)
-                    {
-                        // The version on GitHub is more up to date than this local release.
-                        AnsiConsole.Markup("[red]You are running OF-DL version " + $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}\n[/]");
-                        AnsiConsole.Markup("[red]Please update to the current release, " + $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}: [link]https://git.ofdl.tools/sim0n00ps/OF-DL/releases[/]\n[/]");
-                        Log.Debug("Detected outdated client running version " + $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}");
-                        Log.Debug("Latest release version " + $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}");
-                    }
-                    else
-                    {
-                        // This local version is greater than the release version on GitHub.
-                        AnsiConsole.Markup("[green]You are running OF-DL version " + $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}\n[/]");
-                        AnsiConsole.Markup("[green]Latest Release version: " + $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}\n[/]");
-                        Log.Debug("Detected client running version " + $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}");
-                        Log.Debug("Latest release version " + $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}");
-                    }
-                }
+				try
+				{
+					latestReleaseTag = await VersionHelper.GetLatestReleaseTag(cts.Token);
+				}
+				catch (OperationCanceledException)
+				{
+					AnsiConsole.Markup("[yellow]Version check timed out after 30 seconds.\n[/]");
+					Log.Warning("Version check timed out after 30 seconds");
+					latestReleaseTag = null;
+				}
+
+				if (latestReleaseTag == null)
+				{
+					AnsiConsole.Markup("[yellow]Failed to verify that OF-DL is up-to-date.\n[/]");
+					Log.Error("Failed to get the latest release tag.");
+				}
+				else
+				{
+					Version latestGiteaRelease = new Version(latestReleaseTag.Replace("OFDLV", ""));
+
+					// Compare the Versions
+					int versionComparison = localVersion.CompareTo(latestGiteaRelease);
+					if (versionComparison < 0)
+					{
+						// The version on GitHub is more up to date than this local release.
+						AnsiConsole.Markup("[red]You are running OF-DL version " + $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}\n[/]");
+						AnsiConsole.Markup("[red]Please update to the current release, " + $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}: [link=https://git.ofdl.tools/sim0n00ps/OF-DL/releases]https://git.ofdl.tools/sim0n00ps/OF-DL/releases[/]\n[/]");
+						Log.Debug("Detected outdated client running version " + $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}");
+						Log.Debug("Latest release version " + $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}");
+					}
+					else
+					{
+						// This local version is greater than the release version on GitHub.
+						AnsiConsole.Markup("[green]You are running OF-DL version " + $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}\n[/]");
+						AnsiConsole.Markup("[green]Latest Release version: " + $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}\n[/]");
+						Log.Debug("Detected client running version " + $"{localVersion.Major}.{localVersion.Minor}.{localVersion.Build}");
+						Log.Debug("Latest release version " + $"{latestGiteaRelease.Major}.{latestGiteaRelease.Minor}.{latestGiteaRelease.Build}");
+					}
+				}
 
 				#else
 				AnsiConsole.Markup("[yellow]Running in Debug/Local mode. Version check skipped.\n[/]");
@@ -748,6 +762,61 @@ public class Program
 				{
 					config.FFmpegPath = config.FFmpegPath.Replace(@"\", @"\\");
 				}
+
+				// Get FFmpeg version
+				try
+				{
+					var processStartInfo = new System.Diagnostics.ProcessStartInfo
+					{
+						FileName = config.FFmpegPath,
+						Arguments = "-version",
+						RedirectStandardOutput = true,
+						RedirectStandardError = true,
+						UseShellExecute = false,
+						CreateNoWindow = true
+					};
+
+					using (var process = System.Diagnostics.Process.Start(processStartInfo))
+					{
+						if (process != null)
+						{
+							string output = await process.StandardOutput.ReadToEndAsync();
+							await process.WaitForExitAsync();
+
+							// Log full output
+							Log.Information("FFmpeg version output:\n{Output}", output);
+
+							// Parse first line for console output
+							string firstLine = output.Split('\n')[0].Trim();
+							if (firstLine.StartsWith("ffmpeg version"))
+							{
+								// Extract version string (text between "ffmpeg version " and " Copyright")
+								int versionStart = "ffmpeg version ".Length;
+								int copyrightIndex = firstLine.IndexOf(" Copyright");
+								if (copyrightIndex > versionStart)
+								{
+									string version = firstLine.Substring(versionStart, copyrightIndex - versionStart);
+									AnsiConsole.Markup($"[green]ffmpeg version detected as {version}[/]\n");
+								}
+								else
+								{
+									// Fallback if Copyright not found
+									string version = firstLine.Substring(versionStart);
+									AnsiConsole.Markup($"[green]ffmpeg version detected as {version}[/]\n");
+								}
+							}
+							else
+							{
+								AnsiConsole.Markup($"[yellow]ffmpeg version could not be parsed[/]\n");
+							}
+						}
+					}
+				}
+				catch (Exception ex)
+				{
+					Log.Warning(ex, "Failed to get FFmpeg version");
+					AnsiConsole.Markup($"[yellow]Could not retrieve ffmpeg version[/]\n");
+				}
 			}
 			else
 			{
@@ -991,7 +1060,7 @@ public class Program
 			}
 			else if (hasSelectedUsersKVP.Key && hasSelectedUsersKVP.Value != null && hasSelectedUsersKVP.Value.ContainsKey("PurchasedTab"))
 			{
-				Dictionary<string, int> purchasedTabUsers = await m_ApiHelper.GetPurchasedTabUsers("/posts/paid", Config, users);
+				Dictionary<string, int> purchasedTabUsers = await m_ApiHelper.GetPurchasedTabUsers("/posts/paid/all", Config, users);
 				AnsiConsole.Markup($"[red]Checking folders for Users in Purchased Tab\n[/]");
 				foreach (KeyValuePair<string, int> user in purchasedTabUsers)
 				{
@@ -1038,7 +1107,7 @@ public class Program
 
 				Log.Debug($"Download path: {p}");
 
-				List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
+				List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid/all", p, Config, users);
 				foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
 				{
 					AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]");

docs/config/all-configuration-options.md

@@ -516,3 +516,23 @@ Allowed values: Any positive integer or `-1`
 
 Description: You won't need to set this, but if you see errors about the configured timeout of 100 seconds elapsing then
 you could set this to be more than 100. It is recommended that you leave this as the default value.
+
+## DisableTextSanitization
+
+Type: `boolean`
+
+Default: `false`
+
+Allowed values: `true`, `false`
+
+Description: When enabled, post/message text is stored as-is without XML stripping.
+
+## DownloadVideoResolution
+
+Type: `string`
+
+Default: `"source"`
+
+Allowed values: `"source"`, `"240"`, `"720"`
+
+Description: This allows you to download videos in alternative resolutions, by default videos are downloaded in source resolution but some people prefer smoother videos at a lower resolution.

docs/config/configuration.md

@@ -20,6 +20,8 @@ information about what it does, its default value, and the allowed values.
     - [DownloadDateSelection](/config/all-configuration-options#downloaddateselection)
     - [CustomDate](/config/all-configuration-options#customdate)
     - [ShowScrapeSize](/config/all-configuration-options#showscrapesize)
+    - [DisableTextSanitization](/config/all-configuration-options#disabletextsanitization)
+    - [DownloadVideoResolution](/config/all-configuration-options#downloadvideoresolution)
     - Media
         - [DownloadAvatarHeaderPhoto](/config/all-configuration-options#downloadavatarheaderphoto)
         - [DownloadPaidPosts](/config/all-configuration-options#downloadpaidposts)