sim0n00ps/OF-DL
37
39
Fork 10
Code Issues 28 Pull Requests 3 Actions Packages Projects Releases 6 Wiki Activity

feat: Add option to store raw post text without sanitization (Fix #52) #53

Open
wer wants to merge 1 commits from wer/OF-DL:master into master
pull from: wer/OF-DL:master
merge into: sim0n00ps:master
Conversation 0 Commits 1 Files Changed 3 +27 -6
3 changed files with 27 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 3e7fd45589 - Show all commits

4
OF DL/Entities/Config.cs
View File

@ -103,6 +103,10 @@ namespace OF_DL.Entities
[JsonConverter(typeof(StringEnumConverter))]
public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
// When enabled, post/message text is stored as-is without XML stripping.
[ToggleableConfig]
public bool DisableTextSanitization { get; set; } = false;
}
public class CreatorConfig : IFileNameFormatConfig

9
OF DL/Program.cs
View File

@ -160,6 +160,7 @@ public class Program
hoconConfig.AppendLine($" DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
hoconConfig.AppendLine($" CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
hoconConfig.AppendLine($" ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
hoconConfig.AppendLine($" DisableTextSanitization = false");
hoconConfig.AppendLine($" DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
hoconConfig.AppendLine("}");
@ -280,6 +281,8 @@ public class Program
DownloadDateSelection = Enum.Parse<DownloadDateSelection>(hoconConfig.GetString("Download.DownloadDateSelection"), true),
CustomDate = !string.IsNullOrWhiteSpace(hoconConfig.GetString("Download.CustomDate")) ? DateTime.Parse(hoconConfig.GetString("Download.CustomDate")) : null,
ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
// Optional flag; default to false when missing
DisableTextSanitization = bool.TryParse(hoconConfig.GetString("Download.DisableTextSanitization", "false"), out var dts) ? dts : false,
DownloadVideoResolution = ParseVideoResolution(hoconConfig.GetString("Download.DownloadVideoResolution", "source")),
// File Settings
@ -345,6 +348,8 @@ public class Program
}
levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel; //set the logging level based on config
// Apply text sanitization preference globally
OF_DL.Utils.XmlUtils.Passthrough = config.DisableTextSanitization;
Log.Debug("Configuration:");
string configString = JsonConvert.SerializeObject(config, Formatting.Indented);
Log.Debug(configString);
@ -400,6 +405,8 @@ public class Program
hoconConfig.AppendLine($" DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
hoconConfig.AppendLine($" CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
hoconConfig.AppendLine($" ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
// New option defaults to false when converting legacy json
hoconConfig.AppendLine($" DisableTextSanitization = false");
hoconConfig.AppendLine($" DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
hoconConfig.AppendLine("}");
@ -2904,6 +2911,7 @@ public class Program
hoconConfig.AppendLine($" DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
hoconConfig.AppendLine($" CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
hoconConfig.AppendLine($" ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
hoconConfig.AppendLine($" DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
hoconConfig.AppendLine($" DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
hoconConfig.AppendLine("}");
@ -3063,6 +3071,7 @@ public class Program
hoconConfig.AppendLine($" DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
hoconConfig.AppendLine($" CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
hoconConfig.AppendLine($" ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
hoconConfig.AppendLine($" DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
hoconConfig.AppendLine($" DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
hoconConfig.AppendLine("}");

8
OF DL/Utils/XmlUtils.cs
View File

@ -9,8 +9,16 @@ namespace OF_DL.Utils
{
internal static class XmlUtils
{
// When true, return original text without parsing/stripping.
public static bool Passthrough { get; set; } = false;
public static string EvaluateInnerText(string xmlValue)
{
if (Passthrough)
{
return xmlValue ?? string.Empty;
}
try
{
var parsedText = XElement.Parse($"<root>{xmlValue}</root>");