sim0n00ps/OF-DL
37
39
Fork 10
Code Issues 28 Pull Requests 3 Actions Packages Projects Releases 6 Wiki Activity

feat: Add option to store raw post text without sanitization (Fix #52) #53

Open
wer wants to merge 1 commits from wer/OF-DL:master into master
pull from: wer/OF-DL:master
merge into: sim0n00ps:master
Conversation 0 Commits 1 Files Changed 3 +27 -6
3 changed files with 27 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
OF DL/Entities/Config.cs
View File

@ -103,6 +103,10 @@ namespace OF_DL.Entities
[JsonConverter(typeof(StringEnumConverter))] [JsonConverter(typeof(StringEnumConverter))]
public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source; public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
// When enabled, post/message text is stored as-is without XML stripping.
[ToggleableConfig]
public bool DisableTextSanitization { get; set; } = false;
} }
public class CreatorConfig : IFileNameFormatConfig public class CreatorConfig : IFileNameFormatConfig

21
OF DL/Program.cs
View File

@ -160,6 +160,7 @@ public class Program
hoconConfig.AppendLine($" DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\""); hoconConfig.AppendLine($" DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
hoconConfig.AppendLine($" CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\""); hoconConfig.AppendLine($" CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
hoconConfig.AppendLine($" ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}"); hoconConfig.AppendLine($" ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
hoconConfig.AppendLine($" DisableTextSanitization = false");
hoconConfig.AppendLine($" DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\""); hoconConfig.AppendLine($" DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
hoconConfig.AppendLine("}"); hoconConfig.AppendLine("}");
@ -247,7 +248,7 @@ public class Program
{ {
string hoconText = File.ReadAllText("config.conf"); string hoconText = File.ReadAllText("config.conf");
var hoconConfig = ConfigurationFactory.ParseString(hoconText); var hoconConfig = ConfigurationFactory.ParseString(hoconText);
config = new Entities.Config config = new Entities.Config
{ {
@ -279,7 +280,9 @@ public class Program
DownloadOnlySpecificDates = hoconConfig.GetBoolean("Download.DownloadOnlySpecificDates"), DownloadOnlySpecificDates = hoconConfig.GetBoolean("Download.DownloadOnlySpecificDates"),
DownloadDateSelection = Enum.Parse<DownloadDateSelection>(hoconConfig.GetString("Download.DownloadDateSelection"), true), DownloadDateSelection = Enum.Parse<DownloadDateSelection>(hoconConfig.GetString("Download.DownloadDateSelection"), true),
CustomDate = !string.IsNullOrWhiteSpace(hoconConfig.GetString("Download.CustomDate")) ? DateTime.Parse(hoconConfig.GetString("Download.CustomDate")) : null, CustomDate = !string.IsNullOrWhiteSpace(hoconConfig.GetString("Download.CustomDate")) ? DateTime.Parse(hoconConfig.GetString("Download.CustomDate")) : null,
ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"), ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
// Optional flag; default to false when missing
DisableTextSanitization = bool.TryParse(hoconConfig.GetString("Download.DisableTextSanitization", "false"), out var dts) ? dts : false,
DownloadVideoResolution = ParseVideoResolution(hoconConfig.GetString("Download.DownloadVideoResolution", "source")), DownloadVideoResolution = ParseVideoResolution(hoconConfig.GetString("Download.DownloadVideoResolution", "source")),
// File Settings // File Settings
@ -344,7 +347,9 @@ public class Program
} }
} }
levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel; //set the logging level based on config levelSwitch.MinimumLevel = (LogEventLevel)config.LoggingLevel; //set the logging level based on config
// Apply text sanitization preference globally
OF_DL.Utils.XmlUtils.Passthrough = config.DisableTextSanitization;
Log.Debug("Configuration:"); Log.Debug("Configuration:");
string configString = JsonConvert.SerializeObject(config, Formatting.Indented); string configString = JsonConvert.SerializeObject(config, Formatting.Indented);
Log.Debug(configString); Log.Debug(configString);
@ -399,9 +404,11 @@ public class Program
hoconConfig.AppendLine($" DownloadOnlySpecificDates = {jsonConfig.DownloadOnlySpecificDates.ToString().ToLower()}"); hoconConfig.AppendLine($" DownloadOnlySpecificDates = {jsonConfig.DownloadOnlySpecificDates.ToString().ToLower()}");
hoconConfig.AppendLine($" DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\""); hoconConfig.AppendLine($" DownloadDateSelection = \"{jsonConfig.DownloadDateSelection.ToString().ToLower()}\"");
hoconConfig.AppendLine($" CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\""); hoconConfig.AppendLine($" CustomDate = \"{jsonConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
hoconConfig.AppendLine($" ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}"); hoconConfig.AppendLine($" ShowScrapeSize = {jsonConfig.ShowScrapeSize.ToString().ToLower()}");
hoconConfig.AppendLine($" DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\""); // New option defaults to false when converting legacy json
hoconConfig.AppendLine("}"); hoconConfig.AppendLine($" DisableTextSanitization = false");
hoconConfig.AppendLine($" DownloadVideoResolution = \"{(jsonConfig.DownloadVideoResolution == VideoResolution.source ? "source" : jsonConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
hoconConfig.AppendLine("}");
hoconConfig.AppendLine("# File Settings"); hoconConfig.AppendLine("# File Settings");
hoconConfig.AppendLine("File {"); hoconConfig.AppendLine("File {");
@ -2904,6 +2911,7 @@ public class Program
hoconConfig.AppendLine($" DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\""); hoconConfig.AppendLine($" DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
hoconConfig.AppendLine($" CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\""); hoconConfig.AppendLine($" CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
hoconConfig.AppendLine($" ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}"); hoconConfig.AppendLine($" ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
hoconConfig.AppendLine($" DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
hoconConfig.AppendLine($" DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\""); hoconConfig.AppendLine($" DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
hoconConfig.AppendLine("}"); hoconConfig.AppendLine("}");
@ -3063,6 +3071,7 @@ public class Program
hoconConfig.AppendLine($" DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\""); hoconConfig.AppendLine($" DownloadDateSelection = \"{newConfig.DownloadDateSelection.ToString().ToLower()}\"");
hoconConfig.AppendLine($" CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\""); hoconConfig.AppendLine($" CustomDate = \"{newConfig.CustomDate?.ToString("yyyy-MM-dd")}\"");
hoconConfig.AppendLine($" ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}"); hoconConfig.AppendLine($" ShowScrapeSize = {newConfig.ShowScrapeSize.ToString().ToLower()}");
hoconConfig.AppendLine($" DisableTextSanitization = {newConfig.DisableTextSanitization.ToString().ToLower()}");
hoconConfig.AppendLine($" DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\""); hoconConfig.AppendLine($" DownloadVideoResolution = \"{(newConfig.DownloadVideoResolution == VideoResolution.source ? "source" : newConfig.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
hoconConfig.AppendLine("}"); hoconConfig.AppendLine("}");

8
OF DL/Utils/XmlUtils.cs
View File

@ -9,8 +9,16 @@ namespace OF_DL.Utils
{ {
internal static class XmlUtils internal static class XmlUtils
{ {
// When true, return original text without parsing/stripping.
public static bool Passthrough { get; set; } = false;
public static string EvaluateInnerText(string xmlValue) public static string EvaluateInnerText(string xmlValue)
{ {
if (Passthrough)
{
return xmlValue ?? string.Empty;
}
try try
{ {
var parsedText = XElement.Parse($"<root>{xmlValue}</root>"); var parsedText = XElement.Parse($"<root>{xmlValue}</root>");