forked from sim0n00ps/OF-DL
Compare commits
53 Commits
e698ea2313
...
96aaebd72d
| Author | SHA1 | Date | |
|---|---|---|---|
| 96aaebd72d | |||
| 8d783b512c | |||
| 21efebe74f | |||
| a1dce61b33 | |||
| c170bba96c | |||
| 7e9eba54e6 | |||
| ea5261bc62 | |||
| d098355587 | |||
| b3538e5e70 | |||
| a1c4792e61 | |||
| 0ceb0c1cc8 | |||
| 5921b164bd | |||
| 6c7a171f0e | |||
| 9df2490acf | |||
| 4a039b3de4 | |||
| 5c59e09a68 | |||
| 7754bb2252 | |||
| 51f42bddff | |||
| a96f9f3da6 | |||
| 452540bfd1 | |||
| 8d5cc39722 | |||
| 8eaa14ae03 | |||
| baddcfda4e | |||
| 7a4b145641 | |||
| 16096c5815 | |||
| 3c8231d329 | |||
| ba0347f86f | |||
| 22ad1c005b | |||
| 40a7687606 | |||
| ccb990675a | |||
| 77bd5f7ed9 | |||
| 70f69fb502 | |||
| e22d2b63a2 | |||
| 4b0bd4d676 | |||
| dce7e7a6bd | |||
| 378a82548b | |||
| 03dd66a842 | |||
| edc3d771d1 | |||
| b4aac13bc6 | |||
| 15a5a1d5f1 | |||
| c2ab3dd79f | |||
| de97336f6c | |||
| e106fa2242 | |||
| 568a071658 | |||
| 7bd5971695 | |||
| 5c57178f5b | |||
| 0572844ca8 | |||
| e4eb6c0507 | |||
| d79733ec24 | |||
| a4d8676f2e | |||
| f501a7e806 | |||
| 2b2206a0b4 | |||
| 3ef7895007 |
@ -52,12 +52,13 @@ jobs:
|
|||||||
echo "➤ Creating folder for CDM"
|
echo "➤ Creating folder for CDM"
|
||||||
mkdir -p cdm/devices/chrome_1610
|
mkdir -p cdm/devices/chrome_1610
|
||||||
|
|
||||||
echo "➤ Copying ffmpeg from user folder"
|
echo "➤ Copying ffmpeg and ffprobe from user folder"
|
||||||
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
|
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
|
||||||
|
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffprobe.exe .
|
||||||
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
|
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
|
||||||
|
|
||||||
echo "➤ Creating release zip"
|
echo "➤ Creating release zip"
|
||||||
zip ../OFDLV${{ steps.version.outputs.version }}.zip OF\ DL.exe e_sqlite3.dll rules.json config.conf cdm ffmpeg.exe LICENSE.ffmpeg
|
zip ../OFDLV${{ steps.version.outputs.version }}.zip OF\ DL.exe e_sqlite3.dll rules.json config.conf cdm chromium-scripts ffmpeg.exe ffprobe.exe LICENSE.ffmpeg
|
||||||
cd ..
|
cd ..
|
||||||
|
|
||||||
- name: Create release and upload artifact
|
- name: Create release and upload artifact
|
||||||
|
|||||||
4
.gitignore
vendored
4
.gitignore
vendored
@ -374,3 +374,7 @@ venv/
|
|||||||
|
|
||||||
# Generated docs
|
# Generated docs
|
||||||
/site
|
/site
|
||||||
|
|
||||||
|
# Cajetan
|
||||||
|
.dev/
|
||||||
|
*Publish/
|
||||||
8
Cajetan.OF-DL.slnx
Normal file
8
Cajetan.OF-DL.slnx
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
<Solution>
|
||||||
|
<Folder Name="/Cajetan/">
|
||||||
|
<Project Path="Cajetan.OF-DL/Cajetan.OF-DL.csproj" />
|
||||||
|
</Folder>
|
||||||
|
<Project Path="OF DL.Core/OF DL.Core.csproj" />
|
||||||
|
<Project Path="OF DL.Tests/OF DL.Tests.csproj" />
|
||||||
|
<Project Path="OF DL/OF DL.csproj" />
|
||||||
|
</Solution>
|
||||||
45
Cajetan.OF-DL/CLI/CajetanDownloadEventHandler.cs
Normal file
45
Cajetan.OF-DL/CLI/CajetanDownloadEventHandler.cs
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
using OF_DL.Models.Downloads;
|
||||||
|
|
||||||
|
namespace OF_DL.CLI;
|
||||||
|
|
||||||
|
public interface ICajetanDownloadEventHandler : IDownloadEventHandler
|
||||||
|
{
|
||||||
|
void OnMessage(string message, string color);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class CajetanDownloadEventHandler : ICajetanDownloadEventHandler
|
||||||
|
{
|
||||||
|
private readonly SpectreDownloadEventHandler _eventHandler = new();
|
||||||
|
|
||||||
|
public void OnContentFound(string contentType, int mediaCount, int objectCount)
|
||||||
|
=> _eventHandler.OnContentFound(contentType, mediaCount, objectCount);
|
||||||
|
|
||||||
|
public void OnDownloadComplete(string contentType, DownloadResult result)
|
||||||
|
=> _eventHandler.OnDownloadComplete(contentType, result);
|
||||||
|
|
||||||
|
public void OnMessage(string message)
|
||||||
|
=> _eventHandler.OnMessage(message);
|
||||||
|
|
||||||
|
public void OnMessage(string message, string color)
|
||||||
|
=> AnsiConsole.MarkupLine($"[{color.ToLowerInvariant()}]{Markup.Escape(message)}[/]");
|
||||||
|
|
||||||
|
public void OnNoContentFound(string contentType)
|
||||||
|
=> _eventHandler.OnNoContentFound(contentType);
|
||||||
|
|
||||||
|
public void OnPurchasedTabUserComplete(string username, int paidPostCount, int paidMessagesCount)
|
||||||
|
=> _eventHandler.OnPurchasedTabUserComplete(username, paidPostCount, paidMessagesCount);
|
||||||
|
|
||||||
|
public void OnScrapeComplete(TimeSpan elapsed)
|
||||||
|
=> _eventHandler.OnScrapeComplete(elapsed);
|
||||||
|
|
||||||
|
public void OnUserComplete(string username, CreatorDownloadResult result)
|
||||||
|
=> _eventHandler.OnUserComplete(username, result);
|
||||||
|
|
||||||
|
public void OnUserStarting(string username) { }
|
||||||
|
|
||||||
|
public Task<T> WithProgressAsync<T>(string description, long maxValue, bool showSize, Func<IProgressReporter, Task<T>> work)
|
||||||
|
=> _eventHandler.WithProgressAsync(description, maxValue, showSize, work);
|
||||||
|
|
||||||
|
public Task<T> WithStatusAsync<T>(string statusMessage, Func<IStatusReporter, Task<T>> work)
|
||||||
|
=> _eventHandler.WithStatusAsync(statusMessage, work);
|
||||||
|
}
|
||||||
50
Cajetan.OF-DL/Cajetan.OF-DL.csproj
Normal file
50
Cajetan.OF-DL/Cajetan.OF-DL.csproj
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<OutputType>Exe</OutputType>
|
||||||
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
|
<RootNamespace>OF_DL</RootNamespace>
|
||||||
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
|
<Nullable>enable</Nullable>
|
||||||
|
<ApplicationIcon>Icon\download.ico</ApplicationIcon>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<PropertyGroup>
|
||||||
|
<SelfContained>false</SelfContained>
|
||||||
|
<PublishSingleFile>true</PublishSingleFile>
|
||||||
|
<DebugType>embedded</DebugType>
|
||||||
|
</PropertyGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<Content Include="Icon\download.ico" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<Reference Include="Spectre.Console">
|
||||||
|
<HintPath>..\OF DL\References\Spectre.Console.dll</HintPath>
|
||||||
|
</Reference>
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<ProjectReference Include="..\OF DL\OF DL.csproj" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<PackageReference Include="Serilog.Sinks.Seq" Version="7.0.1" />
|
||||||
|
<PackageReference Include="Serilog.Sinks.Debug" Version="3.0.0" />
|
||||||
|
<PackageReference Include="Akka" Version="1.5.60"/>
|
||||||
|
<PackageReference Include="BouncyCastle.NetCore" Version="2.2.1"/>
|
||||||
|
<PackageReference Include="HtmlAgilityPack" Version="1.12.4"/>
|
||||||
|
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.3"/>
|
||||||
|
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.3"/>
|
||||||
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.4"/>
|
||||||
|
<PackageReference Include="protobuf-net" Version="3.2.56"/>
|
||||||
|
<PackageReference Include="PuppeteerSharp" Version="20.2.6"/>
|
||||||
|
<PackageReference Include="Serilog" Version="4.3.1"/>
|
||||||
|
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1"/>
|
||||||
|
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0"/>
|
||||||
|
<PackageReference Include="System.Reactive" Version="6.1.0"/>
|
||||||
|
<PackageReference Include="xFFmpeg.NET" Version="7.2.0"/>
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
|
</Project>
|
||||||
6
Cajetan.OF-DL/Exceptions/ExitCodeException.cs
Normal file
6
Cajetan.OF-DL/Exceptions/ExitCodeException.cs
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
namespace OF_DL.Exceptions;
|
||||||
|
|
||||||
|
public sealed class ExitCodeException(int exitCode) : Exception
|
||||||
|
{
|
||||||
|
public int ExitCode { get; } = exitCode;
|
||||||
|
}
|
||||||
BIN
Cajetan.OF-DL/Icon/download.ico
Normal file
BIN
Cajetan.OF-DL/Icon/download.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 1.1 KiB |
9
Cajetan.OF-DL/Models/CajetanConfig.cs
Normal file
9
Cajetan.OF-DL/Models/CajetanConfig.cs
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
namespace OF_DL.Models;
|
||||||
|
|
||||||
|
public class CajetanConfig
|
||||||
|
{
|
||||||
|
public string[] NonInteractiveSpecificLists { get; set; } = [];
|
||||||
|
public string[] NonInteractiveSpecificUsers { get; set; } = [];
|
||||||
|
|
||||||
|
public EMode Mode { get; set; } = EMode.DownloadCreatorContent;
|
||||||
|
}
|
||||||
8
Cajetan.OF-DL/Models/Dtos/Lists/ListUsersDto.cs
Normal file
8
Cajetan.OF-DL/Models/Dtos/Lists/ListUsersDto.cs
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
namespace OF_DL.Models.Dtos.Lists;
|
||||||
|
|
||||||
|
public class ListUsersDto
|
||||||
|
{
|
||||||
|
[JsonProperty("list")] public List<UsersListDto> List { get; set; } = [];
|
||||||
|
[JsonProperty("hasMore")] public bool? HasMore { get; set; }
|
||||||
|
[JsonProperty("nextOffset")] public int NextOffset { get; set; }
|
||||||
|
}
|
||||||
19
Cajetan.OF-DL/Models/Dtos/Messages/ChatsDto.cs
Normal file
19
Cajetan.OF-DL/Models/Dtos/Messages/ChatsDto.cs
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
namespace OF_DL.Models.Dtos.Messages;
|
||||||
|
|
||||||
|
public class ChatsDto
|
||||||
|
{
|
||||||
|
[JsonProperty("list")] public List<ChatItemDto> List { get; set; } = [];
|
||||||
|
[JsonProperty("hasMore")] public bool HasMore { get; set; }
|
||||||
|
[JsonProperty("nextOffset")] public int NextOffset { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ChatItemDto
|
||||||
|
{
|
||||||
|
[JsonProperty("withUser")] public ChatUserDto WithUser { get; set; } = new();
|
||||||
|
[JsonProperty("unreadMessagesCount")] public int UnreadMessagesCount { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public class ChatUserDto
|
||||||
|
{
|
||||||
|
[JsonProperty("id")] public long Id { get; set; }
|
||||||
|
}
|
||||||
8
Cajetan.OF-DL/Models/EMode.cs
Normal file
8
Cajetan.OF-DL/Models/EMode.cs
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
namespace OF_DL.Models;
|
||||||
|
|
||||||
|
public enum EMode
|
||||||
|
{
|
||||||
|
DownloadCreatorContent,
|
||||||
|
OutputBlockedUsers,
|
||||||
|
UpdateAllUserInfo
|
||||||
|
}
|
||||||
19
Cajetan.OF-DL/Models/Extensions.cs
Normal file
19
Cajetan.OF-DL/Models/Extensions.cs
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
namespace OF_DL.Models.Downloads;
|
||||||
|
|
||||||
|
internal static class DownloadsExtensions
|
||||||
|
{
|
||||||
|
extension(CreatorDownloadResult result)
|
||||||
|
{
|
||||||
|
public void Add(CreatorDownloadResult other)
|
||||||
|
{
|
||||||
|
result.PaidPostCount += other.PaidPostCount;
|
||||||
|
result.PostCount += other.PostCount;
|
||||||
|
result.ArchivedCount += other.ArchivedCount;
|
||||||
|
result.StreamsCount += other.StreamsCount;
|
||||||
|
result.StoriesCount += other.StoriesCount;
|
||||||
|
result.HighlightsCount += other.HighlightsCount;
|
||||||
|
result.MessagesCount += other.MessagesCount;
|
||||||
|
result.PaidMessagesCount += other.PaidMessagesCount;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
184
Cajetan.OF-DL/ProgramCajetan.cs
Normal file
184
Cajetan.OF-DL/ProgramCajetan.cs
Normal file
@ -0,0 +1,184 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using System.Reflection;
|
||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
|
||||||
|
AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
|
||||||
|
await RunAsync(args);
|
||||||
|
|
||||||
|
static async Task RunAsync(string[] args)
|
||||||
|
{
|
||||||
|
ServiceCollection services = await ConfigureServices(args);
|
||||||
|
ServiceProvider serviceProvider = services.BuildServiceProvider();
|
||||||
|
|
||||||
|
ExitIfOtherProcess(serviceProvider);
|
||||||
|
|
||||||
|
Worker worker = serviceProvider.GetRequiredService<Worker>();
|
||||||
|
await worker.RunAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
static async Task<ServiceCollection> ConfigureServices(string[] args)
|
||||||
|
{
|
||||||
|
// Set up dependency injection with LoggingService and ConfigService
|
||||||
|
ServiceCollection services = new();
|
||||||
|
services.AddSingleton<ILoggingService, CajetanLoggingService>();
|
||||||
|
services.AddSingleton<IConfigService, ConfigService>();
|
||||||
|
services.AddSingleton(new ExitHelper(new SpectreDownloadEventHandler()));
|
||||||
|
ServiceProvider tempServiceProvider = services.BuildServiceProvider();
|
||||||
|
|
||||||
|
ILoggingService loggingService = tempServiceProvider.GetRequiredService<ILoggingService>();
|
||||||
|
IConfigService configService = tempServiceProvider.GetRequiredService<IConfigService>();
|
||||||
|
ExitHelper exitHelper = tempServiceProvider.GetRequiredService<ExitHelper>();
|
||||||
|
|
||||||
|
if (!await configService.LoadConfigurationAsync(args))
|
||||||
|
{
|
||||||
|
AnsiConsole.MarkupLine("\n[red]config.conf is not valid, check your syntax![/]\n");
|
||||||
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
||||||
|
Console.ReadKey();
|
||||||
|
|
||||||
|
exitHelper.ExitWithCode(3);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (configService.CurrentConfig.NonInteractiveMode is false)
|
||||||
|
{
|
||||||
|
AnsiConsole.MarkupLine("\n[red]Cannot run in Interactive Mode![/]\n");
|
||||||
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
||||||
|
Console.ReadKey();
|
||||||
|
|
||||||
|
exitHelper.ExitWithCode(3);
|
||||||
|
}
|
||||||
|
|
||||||
|
AnsiConsole.Markup("[green]config.conf located successfully!\n[/]");
|
||||||
|
|
||||||
|
CajetanConfig cajetanConfig = ParseCommandlineArgs(args, configService.CurrentConfig);
|
||||||
|
|
||||||
|
// Set up full dependency injection with loaded config
|
||||||
|
services = [];
|
||||||
|
services.AddSingleton(loggingService);
|
||||||
|
services.AddSingleton(configService);
|
||||||
|
services.AddSingleton(exitHelper);
|
||||||
|
services.AddSingleton(cajetanConfig);
|
||||||
|
|
||||||
|
services.AddSingleton<IAuthService, AuthService>();
|
||||||
|
services.AddSingleton<IStartupService, StartupService>();
|
||||||
|
services.AddSingleton<IFileNameService, FileNameService>();
|
||||||
|
|
||||||
|
services.AddSingleton<ICajetanDownloadService, CajetanDownloadService>();
|
||||||
|
services.AddSingleton<IDownloadService>(sp => sp.GetRequiredService<ICajetanDownloadService>());
|
||||||
|
|
||||||
|
services.AddSingleton<ICajetanApiService, CajetanApiService>();
|
||||||
|
services.AddSingleton<IApiService>(sp => sp.GetRequiredService<ICajetanApiService>());
|
||||||
|
|
||||||
|
services.AddSingleton<ICajetanDbService, CajetanDbService>();
|
||||||
|
services.AddSingleton<IDbService>(sp => sp.GetRequiredService<ICajetanDbService>());
|
||||||
|
|
||||||
|
|
||||||
|
services.AddSingleton<ICajetanDownloadOrchestrationService, CajetanDownloadOrchestrationService>();
|
||||||
|
services.AddSingleton<IDownloadOrchestrationService>(sp => sp.GetRequiredService<ICajetanDownloadOrchestrationService>());
|
||||||
|
|
||||||
|
services.AddSingleton<ICajetanDownloadEventHandler, CajetanDownloadEventHandler>();
|
||||||
|
services.AddSingleton<IDownloadEventHandler>(sp => sp.GetRequiredService<ICajetanDownloadEventHandler>());
|
||||||
|
|
||||||
|
services.AddSingleton<Worker>();
|
||||||
|
|
||||||
|
return services;
|
||||||
|
}
|
||||||
|
|
||||||
|
static CajetanConfig ParseCommandlineArgs(string[] args, Config currentConfig)
|
||||||
|
{
|
||||||
|
const string SPECIFIC_LISTS_ARG = "--specific-lists";
|
||||||
|
const string SPECIFIC_USERS_ARG = "--specific-users";
|
||||||
|
const string OUTPUT_BLOCKED_USERS_ARG = "--output-blocked";
|
||||||
|
const string UPDATE_ALL_USER_INFO_ARG = "--update-userinfo";
|
||||||
|
|
||||||
|
CajetanConfig parsedConfig = new();
|
||||||
|
|
||||||
|
if (ParseListAndUserArguments(ref parsedConfig))
|
||||||
|
return parsedConfig;
|
||||||
|
|
||||||
|
if (ParseFlagArgument(OUTPUT_BLOCKED_USERS_ARG, EMode.OutputBlockedUsers, ref parsedConfig))
|
||||||
|
return parsedConfig;
|
||||||
|
|
||||||
|
if (ParseFlagArgument(UPDATE_ALL_USER_INFO_ARG, EMode.UpdateAllUserInfo, ref parsedConfig))
|
||||||
|
return parsedConfig;
|
||||||
|
|
||||||
|
// Will process all active subscriptions
|
||||||
|
return parsedConfig;
|
||||||
|
|
||||||
|
bool ParseListAndUserArguments(ref CajetanConfig parsedConfig)
|
||||||
|
{
|
||||||
|
bool hasSpecificListsArg = ParseCommaSeparatedListArgument(SPECIFIC_LISTS_ARG, ref parsedConfig, (c, v) =>
|
||||||
|
{
|
||||||
|
c.NonInteractiveSpecificLists = v;
|
||||||
|
Log.Logger = Log.Logger.ForContext(nameof(CajetanConfig.NonInteractiveSpecificLists), string.Join(",", v));
|
||||||
|
});
|
||||||
|
|
||||||
|
if (hasSpecificListsArg)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
|
||||||
|
bool hasSpecificUsersArg = ParseCommaSeparatedListArgument(SPECIFIC_USERS_ARG, ref parsedConfig, (c, v) =>
|
||||||
|
{
|
||||||
|
c.NonInteractiveSpecificUsers = v;
|
||||||
|
Log.Logger = Log.Logger.ForContext(nameof(CajetanConfig.NonInteractiveSpecificUsers), string.Join(",", v));
|
||||||
|
});
|
||||||
|
|
||||||
|
if (hasSpecificUsersArg)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ParseCommaSeparatedListArgument(string argName, ref CajetanConfig parsedConfig, Action<CajetanConfig, string[]> assignmentFunc)
|
||||||
|
{
|
||||||
|
char[] separator = [','];
|
||||||
|
int indexOfArg = Array.FindIndex(args, a => argName.Equals(a, StringComparison.OrdinalIgnoreCase));
|
||||||
|
|
||||||
|
if (indexOfArg < 0)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
int indexOfListValues = indexOfArg + 1;
|
||||||
|
string[] strListValues = args.ElementAtOrDefault(indexOfListValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
|
||||||
|
|
||||||
|
if (strListValues.Length == 0)
|
||||||
|
return false;
|
||||||
|
|
||||||
|
assignmentFunc(parsedConfig, strListValues);
|
||||||
|
parsedConfig.Mode = EMode.DownloadCreatorContent;
|
||||||
|
|
||||||
|
Log.Logger = Log.Logger.ForContext("Mode", $"{EMode.DownloadCreatorContent}");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool ParseFlagArgument(string argName, EMode modeIfArgIsSet, ref CajetanConfig parsedConfig)
|
||||||
|
{
|
||||||
|
if (!args.Any(a => argName.Equals(a, StringComparison.OrdinalIgnoreCase)))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
currentConfig.NonInteractiveMode = true;
|
||||||
|
parsedConfig.Mode = modeIfArgIsSet;
|
||||||
|
|
||||||
|
Log.Logger = Log.Logger.ForContext("Mode", $"{modeIfArgIsSet}");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static void ExitIfOtherProcess(ServiceProvider serviceProvider)
|
||||||
|
{
|
||||||
|
Assembly? entryAssembly = Assembly.GetEntryAssembly();
|
||||||
|
AssemblyName? entryAssemblyName = entryAssembly?.GetName();
|
||||||
|
|
||||||
|
if (entryAssemblyName?.Name is null)
|
||||||
|
return;
|
||||||
|
|
||||||
|
Process thisProcess = Process.GetCurrentProcess();
|
||||||
|
Process[] otherProcesses = [.. Process.GetProcessesByName(entryAssemblyName.Name).Where(p => p.Id != thisProcess.Id)];
|
||||||
|
|
||||||
|
if (otherProcesses.Length <= 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]");
|
||||||
|
Log.Warning("Other OF DL process detected, exiting..");
|
||||||
|
|
||||||
|
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(0);
|
||||||
|
}
|
||||||
|
|
||||||
19
Cajetan.OF-DL/Properties/GlobalUsings.cs
Normal file
19
Cajetan.OF-DL/Properties/GlobalUsings.cs
Normal file
@ -0,0 +1,19 @@
|
|||||||
|
global using Newtonsoft.Json;
|
||||||
|
global using OF_DL;
|
||||||
|
global using OF_DL.CLI;
|
||||||
|
global using OF_DL.Enumerations;
|
||||||
|
global using OF_DL.Exceptions;
|
||||||
|
global using OF_DL.Helpers;
|
||||||
|
global using OF_DL.Models;
|
||||||
|
global using OF_DL.Models.Config;
|
||||||
|
global using OF_DL.Models.Downloads;
|
||||||
|
global using OF_DL.Services;
|
||||||
|
global using Serilog;
|
||||||
|
global using Serilog.Context;
|
||||||
|
global using Spectre.Console;
|
||||||
|
global using ListsDtos = OF_DL.Models.Dtos.Lists;
|
||||||
|
global using MessageDtos = OF_DL.Models.Dtos.Messages;
|
||||||
|
global using MessageEntities = OF_DL.Models.Entities.Messages;
|
||||||
|
global using SubscriptionsDtos = OF_DL.Models.Dtos.Subscriptions;
|
||||||
|
global using UserDtos = OF_DL.Models.Dtos.Users;
|
||||||
|
global using UserEntities = OF_DL.Models.Entities.Users;
|
||||||
10
Cajetan.OF-DL/Properties/launchSettings.json
Normal file
10
Cajetan.OF-DL/Properties/launchSettings.json
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
{
|
||||||
|
"profiles": {
|
||||||
|
"Cajetan.OF-DL": {
|
||||||
|
"commandName": "Project",
|
||||||
|
"workingDirectory": "..\\.dev\\",
|
||||||
|
//"commandLineArgs": "--non-interactive --specific-lists Capture"
|
||||||
|
"commandLineArgs": "--non-interactive --specific-users amyboz"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
462
Cajetan.OF-DL/Services/CajetanApiService.cs
Normal file
462
Cajetan.OF-DL/Services/CajetanApiService.cs
Normal file
@ -0,0 +1,462 @@
|
|||||||
|
namespace OF_DL.Services;
|
||||||
|
|
||||||
|
public class CajetanApiService(IAuthService authService, IConfigService configService, ICajetanDbService dbService, ICajetanDownloadEventHandler eventHandler)
|
||||||
|
: ApiService(authService, configService, dbService), ICajetanApiService
|
||||||
|
{
|
||||||
|
private readonly ICajetanDownloadEventHandler _eventHandler = eventHandler;
|
||||||
|
|
||||||
|
public new async Task<UserEntities.User?> GetUserInfo(string endpoint)
|
||||||
|
{
|
||||||
|
UserEntities.UserInfo? userInfo = await GetDetailedUserInfoAsync(endpoint);
|
||||||
|
|
||||||
|
if (userInfo is not null && !endpoint.EndsWith("/me"))
|
||||||
|
await dbService.UpdateUserInfoAsync(userInfo);
|
||||||
|
|
||||||
|
return userInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
public new Task<Dictionary<string, long>?> GetActiveSubscriptions(string endpoint, bool includeRestricted)
|
||||||
|
{
|
||||||
|
Log.Debug("Calling GetActiveSubscriptions");
|
||||||
|
return GetAllSubscriptions(endpoint, includeRestricted, "active");
|
||||||
|
}
|
||||||
|
|
||||||
|
public new Task<Dictionary<string, long>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted)
|
||||||
|
{
|
||||||
|
Log.Debug("Calling GetExpiredSubscriptions");
|
||||||
|
return GetAllSubscriptions(endpoint, includeRestricted, "expired");
|
||||||
|
}
|
||||||
|
|
||||||
|
public new async Task<MessageEntities.MessageCollection> GetMessages(string endpoint, string folder, IStatusReporter statusReporter)
|
||||||
|
{
|
||||||
|
(bool couldExtract, long userId) = ExtractUserId(endpoint);
|
||||||
|
|
||||||
|
_eventHandler.OnMessage("Getting Unread Chats", "grey");
|
||||||
|
HashSet<long> usersWithUnread = couldExtract ? await GetUsersWithUnreadMessagesAsync() : [];
|
||||||
|
|
||||||
|
MessageEntities.MessageCollection messages = await base.GetMessages(endpoint, folder, statusReporter);
|
||||||
|
|
||||||
|
if (usersWithUnread.Contains(userId))
|
||||||
|
{
|
||||||
|
_eventHandler.OnMessage("Restoring unread state", "grey");
|
||||||
|
await MarkAsUnreadAsync($"/chats/{userId}/mark-as-read");
|
||||||
|
}
|
||||||
|
|
||||||
|
return messages;
|
||||||
|
|
||||||
|
static (bool couldExtract, long userId) ExtractUserId(string endpoint)
|
||||||
|
{
|
||||||
|
string withoutChatsAndMessages = endpoint
|
||||||
|
.Replace("chats", "", StringComparison.OrdinalIgnoreCase)
|
||||||
|
.Replace("messages", "", StringComparison.OrdinalIgnoreCase);
|
||||||
|
string trimmed = withoutChatsAndMessages.Trim(' ', '/', '\\');
|
||||||
|
|
||||||
|
if (long.TryParse(trimmed, out long userId))
|
||||||
|
return (true, userId);
|
||||||
|
|
||||||
|
return (false, default);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public new async Task<Dictionary<string, long>?> GetListUsers(string endpoint)
|
||||||
|
{
|
||||||
|
if (!HasSignedRequestAuth())
|
||||||
|
return null;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Dictionary<string, long> users = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
Log.Debug($"Calling GetListUsers - {endpoint}");
|
||||||
|
|
||||||
|
const int limit = 50;
|
||||||
|
int offset = 0;
|
||||||
|
|
||||||
|
Dictionary<string, string> getParams = new()
|
||||||
|
{
|
||||||
|
["format"] = "infinite",
|
||||||
|
["limit"] = limit.ToString()
|
||||||
|
};
|
||||||
|
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
getParams["offset"] = offset.ToString();
|
||||||
|
|
||||||
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(body))
|
||||||
|
break;
|
||||||
|
|
||||||
|
ListsDtos.ListUsersDto? listUsers = DeserializeJson<ListsDtos.ListUsersDto>(body, s_mJsonSerializerSettings);
|
||||||
|
|
||||||
|
if (listUsers?.List is null)
|
||||||
|
break;
|
||||||
|
|
||||||
|
foreach (ListsDtos.UsersListDto item in listUsers.List)
|
||||||
|
{
|
||||||
|
if (item.Id is null)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
users.TryAdd(item.Username, item.Id.Value);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (listUsers.HasMore is false)
|
||||||
|
break;
|
||||||
|
|
||||||
|
offset = listUsers.NextOffset;
|
||||||
|
}
|
||||||
|
|
||||||
|
return users;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<UserEntities.UserInfo?> GetDetailedUserInfoAsync(string endpoint)
|
||||||
|
{
|
||||||
|
Log.Debug($"Calling GetDetailedUserInfo: {endpoint}");
|
||||||
|
|
||||||
|
if (!HasSignedRequestAuth())
|
||||||
|
return null;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
UserEntities.UserInfo userInfo = new();
|
||||||
|
Dictionary<string, string> getParams = new()
|
||||||
|
{
|
||||||
|
{ "limit", Constants.ApiPageSize.ToString() }, { "order", "publish_date_asc" }
|
||||||
|
};
|
||||||
|
|
||||||
|
HttpClient client = new();
|
||||||
|
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
|
||||||
|
|
||||||
|
using HttpResponseMessage response = await client.SendAsync(request);
|
||||||
|
|
||||||
|
if (!response.IsSuccessStatusCode)
|
||||||
|
return userInfo;
|
||||||
|
|
||||||
|
response.EnsureSuccessStatusCode();
|
||||||
|
string body = await response.Content.ReadAsStringAsync();
|
||||||
|
UserDtos.UserDto? userDto = JsonConvert.DeserializeObject<UserDtos.UserDto>(body, s_mJsonSerializerSettings);
|
||||||
|
userInfo = FromDto(userDto);
|
||||||
|
return userInfo;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<Dictionary<string, long>> GetUsersWithProgressAsync(string typeDisplay, string endpoint, string? typeParam, bool offsetByCount)
|
||||||
|
{
|
||||||
|
Dictionary<string, long> usersOfType = await _eventHandler.WithStatusAsync(
|
||||||
|
statusMessage: $"Getting {typeDisplay} Users",
|
||||||
|
work: FetchAsync
|
||||||
|
);
|
||||||
|
|
||||||
|
return usersOfType;
|
||||||
|
|
||||||
|
async Task<Dictionary<string, long>> FetchAsync(IStatusReporter statusReporter)
|
||||||
|
{
|
||||||
|
Dictionary<string, long> users = [];
|
||||||
|
|
||||||
|
int limit = 50;
|
||||||
|
int offset = 0;
|
||||||
|
bool includeRestricted = true;
|
||||||
|
|
||||||
|
Dictionary<string, string> getParams = new()
|
||||||
|
{
|
||||||
|
["format"] = "infinite",
|
||||||
|
["limit"] = limit.ToString(),
|
||||||
|
["offset"] = offset.ToString()
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(typeParam))
|
||||||
|
getParams["type"] = typeParam;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Log.Debug("Calling GetUsersWithProgress");
|
||||||
|
|
||||||
|
HttpClient client = GetHttpClient();
|
||||||
|
|
||||||
|
bool isLastLoop = false;
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, client);
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(body))
|
||||||
|
break;
|
||||||
|
|
||||||
|
SubscriptionsDtos.SubscriptionsDto? subscriptions = DeserializeJson<SubscriptionsDtos.SubscriptionsDto>(body, s_mJsonSerializerSettings);
|
||||||
|
|
||||||
|
if (subscriptions?.List is null)
|
||||||
|
break;
|
||||||
|
|
||||||
|
foreach (SubscriptionsDtos.ListItemDto item in subscriptions.List)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(item?.Username))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (users.ContainsKey(item.Username))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
bool isRestricted = item.IsRestricted ?? false;
|
||||||
|
bool isRestrictedButAllowed = isRestricted && includeRestricted;
|
||||||
|
|
||||||
|
if (!isRestricted || isRestrictedButAllowed)
|
||||||
|
users.Add(item.Username, item.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
statusReporter.ReportStatus($"[blue]Getting {typeDisplay} Users\n[/] [blue]Found {users.Count}[/]");
|
||||||
|
|
||||||
|
if (isLastLoop)
|
||||||
|
break;
|
||||||
|
|
||||||
|
if (!subscriptions.HasMore || subscriptions.List.Count == 0)
|
||||||
|
isLastLoop = true;
|
||||||
|
|
||||||
|
offset += offsetByCount
|
||||||
|
? subscriptions.List.Count
|
||||||
|
: limit;
|
||||||
|
|
||||||
|
getParams["offset"] = offset.ToString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
return users;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<HashSet<long>> GetUsersWithUnreadMessagesAsync()
|
||||||
|
{
|
||||||
|
MessageDtos.ChatsDto unreadChats = await GetChatsAsync("/chats", onlyUnread: true);
|
||||||
|
HashSet<long> userWithUnread = [];
|
||||||
|
|
||||||
|
foreach (MessageDtos.ChatItemDto chatItem in unreadChats.List)
|
||||||
|
{
|
||||||
|
if (chatItem?.WithUser?.Id is null)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
if (chatItem.UnreadMessagesCount <= 0)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
userWithUnread.Add(chatItem.WithUser.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
return userWithUnread;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task MarkAsUnreadAsync(string endpoint)
|
||||||
|
{
|
||||||
|
Log.Debug($"Calling MarkAsUnread - {endpoint}");
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var result = new { success = false };
|
||||||
|
|
||||||
|
string? body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(), HttpMethod.Delete);
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(body))
|
||||||
|
result = JsonConvert.DeserializeAnonymousType(body, result);
|
||||||
|
|
||||||
|
if (result?.success != true)
|
||||||
|
_eventHandler.OnMessage($"Failed to mark chat as unread! Endpoint: {endpoint}", "yellow");
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task SortBlockedAsync(string endpoint, string order = "recent", string direction = "desc")
|
||||||
|
{
|
||||||
|
Log.Debug($"Calling SortBlocked - {endpoint}");
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var reqBody = new { order, direction };
|
||||||
|
var result = new { success = false, canAddFriends = false };
|
||||||
|
|
||||||
|
string? body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(), HttpMethod.Post, reqBody);
|
||||||
|
|
||||||
|
if (!string.IsNullOrWhiteSpace(body))
|
||||||
|
result = JsonConvert.DeserializeAnonymousType(body, result);
|
||||||
|
|
||||||
|
if (result?.success != true)
|
||||||
|
_eventHandler.OnMessage($"Failed to sort blocked (order: {order}, direction; {direction})! Endpoint: {endpoint}", "yellow");
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<Dictionary<string, long>?> GetAllSubscriptions(string endpoint, bool includeRestricted, string type)
|
||||||
|
{
|
||||||
|
if (!HasSignedRequestAuth())
|
||||||
|
return null;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Dictionary<string, long> users = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
Log.Debug("Calling GetAllSubscrptions");
|
||||||
|
|
||||||
|
const int limit = 50;
|
||||||
|
int offset = 0;
|
||||||
|
|
||||||
|
Dictionary<string, string> getParams = new()
|
||||||
|
{
|
||||||
|
["type"] = type,
|
||||||
|
["format"] = "infinite",
|
||||||
|
["limit"] = limit.ToString(),
|
||||||
|
};
|
||||||
|
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
getParams["offset"] = offset.ToString();
|
||||||
|
|
||||||
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(body))
|
||||||
|
break;
|
||||||
|
|
||||||
|
SubscriptionsDtos.SubscriptionsDto? subscriptionsDto = DeserializeJson<SubscriptionsDtos.SubscriptionsDto>(body, s_mJsonSerializerSettings);
|
||||||
|
|
||||||
|
if (subscriptionsDto?.List is null)
|
||||||
|
break;
|
||||||
|
|
||||||
|
foreach (SubscriptionsDtos.ListItemDto item in subscriptionsDto.List)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(item.Username))
|
||||||
|
{
|
||||||
|
Log.Warning("Found '{Type:l}' subscription user with empty username (id: {Id})", type, item.Id);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (users.ContainsKey(item.Username))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
bool isRestricted = item.IsRestricted ?? false;
|
||||||
|
bool isRestrictedButAllowed = isRestricted && includeRestricted;
|
||||||
|
|
||||||
|
if (!isRestricted || isRestrictedButAllowed)
|
||||||
|
users.Add(item.Username, item.Id);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (subscriptionsDto.HasMore is false)
|
||||||
|
break;
|
||||||
|
|
||||||
|
offset += limit;
|
||||||
|
}
|
||||||
|
|
||||||
|
return users;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<MessageDtos.ChatsDto> GetChatsAsync(string endpoint, bool onlyUnread)
|
||||||
|
{
|
||||||
|
Log.Debug($"Calling GetChats - {endpoint}");
|
||||||
|
|
||||||
|
MessageDtos.ChatsDto allChats = new();
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
const int limit = 60;
|
||||||
|
int offset = 0;
|
||||||
|
|
||||||
|
Dictionary<string, string> getParams = new()
|
||||||
|
{
|
||||||
|
["order"] = "recent",
|
||||||
|
["skip_users"] = "all",
|
||||||
|
["filter"] = "unread",
|
||||||
|
["limit"] = $"{limit}",
|
||||||
|
};
|
||||||
|
|
||||||
|
if (onlyUnread is false)
|
||||||
|
getParams.Remove("filter");
|
||||||
|
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
getParams["offset"] = $"{offset}";
|
||||||
|
|
||||||
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(body))
|
||||||
|
break;
|
||||||
|
|
||||||
|
MessageDtos.ChatsDto? chats = DeserializeJson<MessageDtos.ChatsDto>(body, s_mJsonSerializerSettings);
|
||||||
|
|
||||||
|
if (chats?.List is null)
|
||||||
|
break;
|
||||||
|
|
||||||
|
allChats.List.AddRange(chats.List);
|
||||||
|
|
||||||
|
if (chats.HasMore is false)
|
||||||
|
break;
|
||||||
|
|
||||||
|
offset = chats.NextOffset;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
|
}
|
||||||
|
|
||||||
|
return allChats;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static UserEntities.UserInfo FromDto(UserDtos.UserDto? userDto)
|
||||||
|
{
|
||||||
|
if (userDto is null)
|
||||||
|
return new();
|
||||||
|
|
||||||
|
return new()
|
||||||
|
{
|
||||||
|
Id = userDto.Id,
|
||||||
|
|
||||||
|
Avatar = userDto.Avatar,
|
||||||
|
Header = userDto.Header,
|
||||||
|
Name = userDto.Name,
|
||||||
|
Username = userDto.Username,
|
||||||
|
|
||||||
|
SubscribePrice = userDto.SubscribePrice,
|
||||||
|
CurrentSubscribePrice = userDto.CurrentSubscribePrice,
|
||||||
|
IsPaywallRequired = userDto.IsPaywallRequired,
|
||||||
|
IsRestricted = userDto.IsRestricted,
|
||||||
|
SubscribedBy = userDto.SubscribedBy,
|
||||||
|
SubscribedByExpire = userDto.SubscribedByExpire,
|
||||||
|
SubscribedByExpireDate = userDto.SubscribedByExpireDate,
|
||||||
|
SubscribedByAutoprolong = userDto.SubscribedByAutoprolong,
|
||||||
|
SubscribedIsExpiredNow = userDto.SubscribedIsExpiredNow,
|
||||||
|
SubscribedOn = userDto.SubscribedOn,
|
||||||
|
SubscribedOnExpiredNow = userDto.SubscribedOnExpiredNow,
|
||||||
|
SubscribedOnDuration = userDto.SubscribedOnDuration,
|
||||||
|
About = userDto.About,
|
||||||
|
PostsCount = userDto.PostsCount,
|
||||||
|
ArchivedPostsCount = userDto.ArchivedPostsCount,
|
||||||
|
PrivateArchivedPostsCount = userDto.PrivateArchivedPostsCount,
|
||||||
|
PhotosCount = userDto.PhotosCount,
|
||||||
|
VideosCount = userDto.VideosCount,
|
||||||
|
AudiosCount = userDto.AudiosCount,
|
||||||
|
MediasCount = userDto.MediasCount,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
145
Cajetan.OF-DL/Services/CajetanDbService.cs
Normal file
145
Cajetan.OF-DL/Services/CajetanDbService.cs
Normal file
@ -0,0 +1,145 @@
|
|||||||
|
using Microsoft.Data.Sqlite;
|
||||||
|
|
||||||
|
namespace OF_DL.Services;
|
||||||
|
|
||||||
|
public class CajetanDbService(IConfigService configService)
|
||||||
|
: DbService(configService), ICajetanDbService
|
||||||
|
{
|
||||||
|
public async Task InitializeUserInfoTablesAsync()
|
||||||
|
{
|
||||||
|
await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
|
||||||
|
await connection.OpenAsync();
|
||||||
|
|
||||||
|
using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, about VARCHAR NULL, expires_on TIMESTAMP NULL, photo_count INT NOT NULL, video_count INT NOT NULL, PRIMARY KEY(user_id));", connection))
|
||||||
|
{
|
||||||
|
await cmdInfo.ExecuteNonQueryAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info_blob (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, blob TEXT NULL, PRIMARY KEY(user_id));", connection))
|
||||||
|
{
|
||||||
|
await cmdInfo.ExecuteNonQueryAsync();
|
||||||
|
}
|
||||||
|
|
||||||
|
using (SqliteCommand cmdNonNude = new("CREATE TABLE IF NOT EXISTS user_non_nude (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, PRIMARY KEY(user_id));", connection))
|
||||||
|
{
|
||||||
|
await cmdNonNude.ExecuteNonQueryAsync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task<Dictionary<string, long>> GetUsersAsync()
|
||||||
|
{
|
||||||
|
await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
|
||||||
|
await connection.OpenAsync();
|
||||||
|
|
||||||
|
using SqliteCommand cmd = new("SELECT user_id, username FROM users", connection);
|
||||||
|
using SqliteDataReader reader = cmd.ExecuteReader();
|
||||||
|
|
||||||
|
Dictionary<string, long> result = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
while (reader.Read())
|
||||||
|
{
|
||||||
|
long userId = reader.GetInt64(0);
|
||||||
|
string username = reader.GetString(1);
|
||||||
|
|
||||||
|
result[username] = userId;
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task UpdateUserInfoAsync(UserEntities.UserInfo? userInfo)
|
||||||
|
{
|
||||||
|
if (userInfo?.Id is null || userInfo?.Username is null)
|
||||||
|
return;
|
||||||
|
|
||||||
|
await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
|
||||||
|
await connection.OpenAsync();
|
||||||
|
|
||||||
|
Log.Debug("Database data source: " + connection.DataSource);
|
||||||
|
|
||||||
|
await UpdateAsync();
|
||||||
|
await UpdateBlobAsync();
|
||||||
|
|
||||||
|
async Task UpdateAsync()
|
||||||
|
{
|
||||||
|
using SqliteCommand cmdInfo = new(
|
||||||
|
"INSERT OR REPLACE INTO user_info (user_id, name, about, expires_on, photo_count, video_count) " +
|
||||||
|
"VALUES (@userId, @name, @about, @expiresOn, @photoCount, @videoCount);",
|
||||||
|
connection
|
||||||
|
);
|
||||||
|
|
||||||
|
cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
|
||||||
|
cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
|
||||||
|
cmdInfo.Parameters.AddWithValue("@about", userInfo.About);
|
||||||
|
cmdInfo.Parameters.AddWithValue("@expiresOn", userInfo.SubscribedByExpireDate);
|
||||||
|
cmdInfo.Parameters.AddWithValue("@photoCount", userInfo.PhotosCount ?? 0);
|
||||||
|
cmdInfo.Parameters.AddWithValue("@videoCount", userInfo.VideosCount ?? 0);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await cmdInfo.ExecuteNonQueryAsync();
|
||||||
|
Log.Debug("Inserted or updated creator info: {Username:l}", userInfo.Username);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Log.Warning(ex, "Failed to update User Info for: {Username:l}", userInfo.Username);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task UpdateBlobAsync()
|
||||||
|
{
|
||||||
|
using SqliteCommand cmdInfo = new(
|
||||||
|
"INSERT OR REPLACE INTO user_info_blob (user_id, name, blob) " +
|
||||||
|
"VALUES (@userId, @name, @blob);",
|
||||||
|
connection
|
||||||
|
);
|
||||||
|
|
||||||
|
cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
|
||||||
|
cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
|
||||||
|
cmdInfo.Parameters.AddWithValue("@blob", JsonConvert.SerializeObject(userInfo));
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await cmdInfo.ExecuteNonQueryAsync();
|
||||||
|
Log.Debug("Inserted or updated creator blob: {Username:l}", userInfo.Username);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Log.Warning(ex, "Failed to update User Info Blob for: {Username:l}", userInfo.Username);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public async Task UpdateNonNudeCollectionAsync(Dictionary<string, long> usersInNonNudeLists)
|
||||||
|
{
|
||||||
|
if (usersInNonNudeLists.Count == 0)
|
||||||
|
return;
|
||||||
|
|
||||||
|
await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
|
||||||
|
await connection.OpenAsync();
|
||||||
|
|
||||||
|
Log.Debug("Database data source: " + connection.DataSource);
|
||||||
|
|
||||||
|
foreach ((string username, long userId) in usersInNonNudeLists)
|
||||||
|
{
|
||||||
|
using SqliteCommand cmdInfo = new(
|
||||||
|
"INSERT OR REPLACE INTO user_non_nude (user_id, name) " +
|
||||||
|
"VALUES (@userId, @name);",
|
||||||
|
connection
|
||||||
|
);
|
||||||
|
|
||||||
|
cmdInfo.Parameters.AddWithValue("@userId", userId);
|
||||||
|
cmdInfo.Parameters.AddWithValue("@name", username);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await cmdInfo.ExecuteNonQueryAsync();
|
||||||
|
Log.Debug("Updating Non-Nude collection with: {Username:l}", username);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Log.Warning(ex, "Failed to update Non-Nude collection with: {Username:l}", username);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
namespace OF_DL.Services;
|
||||||
|
|
||||||
|
public class CajetanDownloadOrchestrationService(ICajetanApiService apiService, IConfigService configService, ICajetanDownloadService downloadService, ICajetanDbService dbService)
|
||||||
|
: DownloadOrchestrationService(apiService, configService, downloadService, dbService), ICajetanDownloadOrchestrationService
|
||||||
|
{
|
||||||
|
}
|
||||||
6
Cajetan.OF-DL/Services/CajetanDownloadService.cs
Normal file
6
Cajetan.OF-DL/Services/CajetanDownloadService.cs
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
namespace OF_DL.Services;
|
||||||
|
|
||||||
|
public class CajetanDownloadService(IAuthService authService, IConfigService configService, ICajetanDbService dbService, IFileNameService fileNameService, ICajetanApiService apiService)
|
||||||
|
: DownloadService(authService, configService, dbService, fileNameService, apiService), ICajetanDownloadService
|
||||||
|
{
|
||||||
|
}
|
||||||
54
Cajetan.OF-DL/Services/CajetanLoggingService.cs
Normal file
54
Cajetan.OF-DL/Services/CajetanLoggingService.cs
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
using Serilog.Core;
|
||||||
|
using Serilog.Events;
|
||||||
|
|
||||||
|
namespace OF_DL.Services;
|
||||||
|
|
||||||
|
public class CajetanLoggingService : ILoggingService
|
||||||
|
{
|
||||||
|
public CajetanLoggingService()
|
||||||
|
{
|
||||||
|
LevelSwitch = new LoggingLevelSwitch();
|
||||||
|
InitializeLoggerWithSeq();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Gets the level switch that controls runtime logging verbosity.
|
||||||
|
/// </summary>
|
||||||
|
public LoggingLevelSwitch LevelSwitch { get; }
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Updates the minimum logging level at runtime.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="newLevel">The new minimum log level.</param>
|
||||||
|
public void UpdateLoggingLevel(LoggingLevel newLevel)
|
||||||
|
{
|
||||||
|
LevelSwitch.MinimumLevel = (LogEventLevel)newLevel;
|
||||||
|
Log.Debug("Logging level updated to: {LoggingLevel}", newLevel);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Returns the current minimum logging level.
|
||||||
|
/// </summary>
|
||||||
|
public LoggingLevel GetCurrentLoggingLevel() => (LoggingLevel)LevelSwitch.MinimumLevel;
|
||||||
|
|
||||||
|
private void InitializeLoggerWithSeq()
|
||||||
|
{
|
||||||
|
LevelSwitch.MinimumLevel = LogEventLevel.Warning;
|
||||||
|
|
||||||
|
LoggerConfiguration loggerConfig = new LoggerConfiguration()
|
||||||
|
.Enrich.FromLogContext()
|
||||||
|
.Enrich.WithProperty("Application", "OF_DL")
|
||||||
|
.Enrich.WithProperty("StartTime", $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} ")
|
||||||
|
.Enrich.WithProperty("MachineName", Environment.MachineName)
|
||||||
|
.MinimumLevel.Verbose()
|
||||||
|
.WriteTo.File("logs/OFDL.txt", rollingInterval: RollingInterval.Day, restrictedToMinimumLevel: LogEventLevel.Error, levelSwitch: LevelSwitch)
|
||||||
|
.WriteTo.Seq("https://seq.cajetan.dk", controlLevelSwitch: LevelSwitch);
|
||||||
|
|
||||||
|
if (System.Diagnostics.Debugger.IsAttached)
|
||||||
|
loggerConfig.WriteTo.Debug(restrictedToMinimumLevel: LogEventLevel.Debug);
|
||||||
|
|
||||||
|
Log.Logger = loggerConfig.CreateLogger();
|
||||||
|
|
||||||
|
Log.Debug("Logging service initialized");
|
||||||
|
}
|
||||||
|
}
|
||||||
12
Cajetan.OF-DL/Services/ICajetanApiService.cs
Normal file
12
Cajetan.OF-DL/Services/ICajetanApiService.cs
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
namespace OF_DL.Services;
|
||||||
|
|
||||||
|
public interface ICajetanApiService : IApiService
|
||||||
|
{
|
||||||
|
new Task<Dictionary<string, long>?> GetListUsers(string endpoint);
|
||||||
|
|
||||||
|
Task<UserEntities.UserInfo?> GetDetailedUserInfoAsync(string endpoint);
|
||||||
|
Task<Dictionary<string, long>> GetUsersWithProgressAsync(string typeDisplay, string endpoint, string? typeParam, bool offsetByCount);
|
||||||
|
Task<HashSet<long>> GetUsersWithUnreadMessagesAsync();
|
||||||
|
Task MarkAsUnreadAsync(string endpoint);
|
||||||
|
Task SortBlockedAsync(string endpoint, string order = "recent", string direction = "desc");
|
||||||
|
}
|
||||||
10
Cajetan.OF-DL/Services/ICajetanDbService.cs
Normal file
10
Cajetan.OF-DL/Services/ICajetanDbService.cs
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
namespace OF_DL.Services;
|
||||||
|
|
||||||
|
public interface ICajetanDbService : IDbService
|
||||||
|
{
|
||||||
|
Task InitializeUserInfoTablesAsync();
|
||||||
|
|
||||||
|
Task<Dictionary<string, long>> GetUsersAsync();
|
||||||
|
Task UpdateUserInfoAsync(UserEntities.UserInfo? userInfo);
|
||||||
|
Task UpdateNonNudeCollectionAsync(Dictionary<string, long> usersInNonNudeLists);
|
||||||
|
}
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
namespace OF_DL.Services;
|
||||||
|
|
||||||
|
public interface ICajetanDownloadOrchestrationService : IDownloadOrchestrationService
|
||||||
|
{
|
||||||
|
|
||||||
|
}
|
||||||
6
Cajetan.OF-DL/Services/ICajetanDownloadService.cs
Normal file
6
Cajetan.OF-DL/Services/ICajetanDownloadService.cs
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
namespace OF_DL.Services;
|
||||||
|
|
||||||
|
public interface ICajetanDownloadService : IDownloadService
|
||||||
|
{
|
||||||
|
|
||||||
|
}
|
||||||
454
Cajetan.OF-DL/Worker.cs
Normal file
454
Cajetan.OF-DL/Worker.cs
Normal file
@ -0,0 +1,454 @@
|
|||||||
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
|
||||||
|
namespace OF_DL;
|
||||||
|
|
||||||
|
internal class Worker(IServiceProvider serviceProvider)
|
||||||
|
{
|
||||||
|
private readonly IConfigService _configService = serviceProvider.GetRequiredService<IConfigService>();
|
||||||
|
private readonly IAuthService _authService = serviceProvider.GetRequiredService<IAuthService>();
|
||||||
|
private readonly IStartupService _startupService = serviceProvider.GetRequiredService<IStartupService>();
|
||||||
|
private readonly IDownloadOrchestrationService _orchestrationService = serviceProvider.GetRequiredService<IDownloadOrchestrationService>();
|
||||||
|
private readonly ICajetanDbService _dbService = serviceProvider.GetRequiredService<ICajetanDbService>();
|
||||||
|
private readonly ICajetanApiService _apiService = serviceProvider.GetRequiredService<ICajetanApiService>();
|
||||||
|
private readonly ExitHelper _exitHelper = serviceProvider.GetRequiredService<ExitHelper>();
|
||||||
|
|
||||||
|
private readonly CajetanConfig _cajetanConfig = serviceProvider.GetRequiredService<CajetanConfig>();
|
||||||
|
private bool _clientIdBlobMissing = true;
|
||||||
|
private bool _devicePrivateKeyMissing = true;
|
||||||
|
|
||||||
|
public async Task RunAsync()
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await InitializeAsync();
|
||||||
|
|
||||||
|
Task tMode = _cajetanConfig.Mode switch
|
||||||
|
{
|
||||||
|
EMode.DownloadCreatorContent => DownloadCreatorContentAsync(),
|
||||||
|
EMode.OutputBlockedUsers => OutputBlockedUsersAsync(),
|
||||||
|
EMode.UpdateAllUserInfo => UpdateUserInfoAsync(),
|
||||||
|
_ => Task.CompletedTask
|
||||||
|
};
|
||||||
|
|
||||||
|
await tMode;
|
||||||
|
}
|
||||||
|
catch (ExitCodeException ex)
|
||||||
|
{
|
||||||
|
_exitHelper.ExitWithCode(ex.ExitCode);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Log.Fatal(ex, "Unhandled Exception! {ExceptionMessage:l}", ex.Message);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
_exitHelper.ExitWithCode(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task InitializeAsync()
|
||||||
|
{
|
||||||
|
StartupResult startupResult = await _startupService.ValidateEnvironmentAsync();
|
||||||
|
|
||||||
|
if (startupResult.IsWindowsVersionValid is false)
|
||||||
|
throw new ExitCodeException(1);
|
||||||
|
|
||||||
|
if (startupResult.FfmpegFound is false)
|
||||||
|
throw new ExitCodeException(4);
|
||||||
|
|
||||||
|
if (startupResult.ClientIdBlobMissing || startupResult.DevicePrivateKeyMissing)
|
||||||
|
AnsiConsole.Markup("[yellow]device_client_id_blob and/or device_private_key missing, https://ofdl.tools/ or https://cdrm-project.com/ will be used instead for DRM protected videos\n[/]");
|
||||||
|
|
||||||
|
_clientIdBlobMissing = startupResult.ClientIdBlobMissing;
|
||||||
|
_devicePrivateKeyMissing = startupResult.DevicePrivateKeyMissing;
|
||||||
|
|
||||||
|
if (IsRulesJsonValid() is false)
|
||||||
|
{
|
||||||
|
AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]");
|
||||||
|
Console.ReadKey();
|
||||||
|
throw new ExitCodeException(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (await IsAuthorizedAsync() is false)
|
||||||
|
{
|
||||||
|
AnsiConsole.MarkupLine("\n[red]Press any key to exit.[/]");
|
||||||
|
Console.ReadKey();
|
||||||
|
throw new ExitCodeException(2);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool IsRulesJsonValid()
|
||||||
|
{
|
||||||
|
if (startupResult.RulesJsonExists is false)
|
||||||
|
return true;
|
||||||
|
|
||||||
|
if (startupResult.RulesJsonValid)
|
||||||
|
{
|
||||||
|
AnsiConsole.Markup("[green]rules.json located successfully!\n[/]");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
AnsiConsole.MarkupLine("\n[red]rules.json is not valid, check your JSON syntax![/]\n");
|
||||||
|
Log.Error("processing of rules.json failed: {Error:l}", startupResult.RulesJsonError);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task<bool> IsAuthorizedAsync()
|
||||||
|
{
|
||||||
|
if (await _authService.LoadFromFileAsync() is false)
|
||||||
|
{
|
||||||
|
if (File.Exists("auth.json"))
|
||||||
|
{
|
||||||
|
Log.Error("Auth file was found but could not be deserialized!");
|
||||||
|
AnsiConsole.MarkupLine("\n[red]auth.json could not be deserialized.[/]");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
Log.Error("Auth file was not found!");
|
||||||
|
AnsiConsole.MarkupLine("\n[red]auth.json is missing.[/]");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
AnsiConsole.Markup("[green]auth.json located successfully!\n[/]");
|
||||||
|
|
||||||
|
// Validate cookie string
|
||||||
|
_authService.ValidateCookieString();
|
||||||
|
|
||||||
|
UserEntities.User? user = await _authService.ValidateAuthAsync();
|
||||||
|
if (user is null || (user.Name is null && user.Username is null))
|
||||||
|
{
|
||||||
|
Log.Error("Auth failed");
|
||||||
|
_authService.CurrentAuth = null;
|
||||||
|
AnsiConsole.MarkupLine("\n[red]Auth failed. Please try again or use other authentication methods detailed here:[/]\n");
|
||||||
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth[/]\n");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
string displayName = $"{user.Name} {user.Username}".Trim();
|
||||||
|
AnsiConsole.MarkupLine($"[green]Logged In successfully as {displayName}\n[/]");
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task DownloadCreatorContentAsync()
|
||||||
|
{
|
||||||
|
DateTime startTime = DateTime.Now;
|
||||||
|
|
||||||
|
UserListResult allUsersAndLists = await GetAvailableUsersAsync();
|
||||||
|
Dictionary<string, long> usersToDownload = allUsersAndLists.Users;
|
||||||
|
|
||||||
|
if (_cajetanConfig.NonInteractiveSpecificLists is not null && _cajetanConfig.NonInteractiveSpecificLists.Length > 0)
|
||||||
|
usersToDownload = await GetUsersFromSpecificListsAsync(allUsersAndLists, _cajetanConfig.NonInteractiveSpecificLists);
|
||||||
|
|
||||||
|
else if (_cajetanConfig.NonInteractiveSpecificUsers is not null && _cajetanConfig.NonInteractiveSpecificUsers.Length > 0)
|
||||||
|
usersToDownload = GetUsersFromSpecificUsernames(allUsersAndLists, [.. _cajetanConfig.NonInteractiveSpecificUsers]);
|
||||||
|
|
||||||
|
int userNum = 0;
|
||||||
|
int userCount = usersToDownload.Count;
|
||||||
|
CajetanDownloadEventHandler eventHandler = new();
|
||||||
|
CreatorDownloadResult totalResults = new();
|
||||||
|
|
||||||
|
LoggerWithConfigContext(_configService.CurrentConfig, _cajetanConfig)
|
||||||
|
.Information("Scraping Data for {UserCount} user(s)", usersToDownload.Count);
|
||||||
|
|
||||||
|
eventHandler.OnMessage(
|
||||||
|
$"\nScraping Data for {usersToDownload.Count} user(s)\n" +
|
||||||
|
$"{"======================================================================================================"}\n"
|
||||||
|
);
|
||||||
|
|
||||||
|
foreach ((string username, long userId) in usersToDownload)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
DateTime userStartTime = DateTime.Now;
|
||||||
|
|
||||||
|
Log.Information("Scraping Data for '{Username:l}' ({UserNum} of {UserCount})", username, ++userNum, userCount);
|
||||||
|
eventHandler.OnMessage($"\nScraping Data for {Markup.Escape(username)} ({userNum} of {userCount})\n");
|
||||||
|
|
||||||
|
string path = _orchestrationService.ResolveDownloadPath(username);
|
||||||
|
Log.Debug($"Download path: {path}");
|
||||||
|
|
||||||
|
CreatorDownloadResult results = await _orchestrationService.DownloadCreatorContentAsync(
|
||||||
|
username: username,
|
||||||
|
userId: userId,
|
||||||
|
path: path,
|
||||||
|
users: usersToDownload,
|
||||||
|
clientIdBlobMissing: _clientIdBlobMissing,
|
||||||
|
devicePrivateKeyMissing: _devicePrivateKeyMissing,
|
||||||
|
eventHandler: eventHandler
|
||||||
|
);
|
||||||
|
CreatorDownloadResult newResults = results.NewDownloads ?? results;
|
||||||
|
|
||||||
|
totalResults.Add(newResults);
|
||||||
|
|
||||||
|
DateTime userEndTime = DateTime.Now;
|
||||||
|
TimeSpan userTotalTime = userEndTime - userStartTime;
|
||||||
|
|
||||||
|
Log.ForContext("Posts", newResults.PostCount)
|
||||||
|
.ForContext("PaidPosts", newResults.PaidPostCount)
|
||||||
|
.ForContext("AllPosts", newResults.PostCount + newResults.PaidPostCount)
|
||||||
|
.ForContext("Archived", newResults.ArchivedCount)
|
||||||
|
.ForContext("Streams", newResults.StreamsCount)
|
||||||
|
.ForContext("Stories", newResults.StoriesCount)
|
||||||
|
.ForContext("Highlights", newResults.HighlightsCount)
|
||||||
|
.ForContext("Messages", newResults.MessagesCount)
|
||||||
|
.ForContext("PaidMessages", newResults.PaidMessagesCount)
|
||||||
|
.ForContext("AllMessages", newResults.MessagesCount + newResults.PaidMessagesCount)
|
||||||
|
.ForContext("Username", username)
|
||||||
|
.ForContext("TotalMinutes", userTotalTime.TotalMinutes)
|
||||||
|
.Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes [P: {AllPosts}] [M: {AllMessages}]");
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Log.ForContext("Username", username)
|
||||||
|
.ForContext("UserNum", userNum)
|
||||||
|
.ForContext("UserCount", userCount)
|
||||||
|
.ForContext("ExceptionMessage", ex.Message)
|
||||||
|
.Error(ex, "Scrape for '{Username:l}' ({UserNum} of {UserCount}) failed! {ExceptionMessage:l}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
DateTime endTime = DateTime.Now;
|
||||||
|
TimeSpan totalTime = endTime - startTime;
|
||||||
|
|
||||||
|
eventHandler.OnScrapeComplete(totalTime);
|
||||||
|
|
||||||
|
Log.ForContext("Posts", totalResults.PostCount)
|
||||||
|
.ForContext("PaidPosts", totalResults.PaidPostCount)
|
||||||
|
.ForContext("AllPosts", totalResults.PostCount + totalResults.PaidPostCount)
|
||||||
|
.ForContext("Archived", totalResults.ArchivedCount)
|
||||||
|
.ForContext("Streams", totalResults.StreamsCount)
|
||||||
|
.ForContext("Stories", totalResults.StoriesCount)
|
||||||
|
.ForContext("Highlights", totalResults.HighlightsCount)
|
||||||
|
.ForContext("Messages", totalResults.MessagesCount)
|
||||||
|
.ForContext("PaidMessages", totalResults.PaidMessagesCount)
|
||||||
|
.ForContext("AllMessages", totalResults.MessagesCount + totalResults.PaidMessagesCount)
|
||||||
|
.ForContext("TotalMinutes", totalTime.TotalMinutes)
|
||||||
|
.Information("Scrape Completed in {TotalMinutes:0.00} minutes [P: {AllPosts}] [M: {AllMessages}]");
|
||||||
|
|
||||||
|
await Task.Delay(2000);
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task OutputBlockedUsersAsync()
|
||||||
|
{
|
||||||
|
const string OUTPUT_FILE_BLOCKED = "blocked-users.json";
|
||||||
|
const string OUTPUT_FILE_EXPIRED = "expired-users.json";
|
||||||
|
|
||||||
|
await _apiService.SortBlockedAsync("/lists/blocked/sort");
|
||||||
|
await GetUsersAsync("Blocked", "/users/blocked", OUTPUT_FILE_BLOCKED);
|
||||||
|
|
||||||
|
await GetUsersAsync("Expired", "/subscriptions/subscribes", OUTPUT_FILE_EXPIRED, typeParam: "expired", offsetByCount: false);
|
||||||
|
|
||||||
|
async Task GetUsersAsync(string typeDisplay, string uri, string outputFile, string? typeParam = null, bool offsetByCount = true)
|
||||||
|
{
|
||||||
|
Dictionary<string, long> users = await _apiService.GetUsersWithProgressAsync(typeDisplay, uri, typeParam, offsetByCount);
|
||||||
|
|
||||||
|
Console.WriteLine();
|
||||||
|
|
||||||
|
if (users is null || users.Count == 0)
|
||||||
|
{
|
||||||
|
AnsiConsole.Markup($"[green]No {typeDisplay} Users found.\n[/]");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
AnsiConsole.Markup($"[green]Found {users.Count} {typeDisplay} Users, saving to '{outputFile}'\n[/]");
|
||||||
|
string json = JsonConvert.SerializeObject(users, Formatting.Indented);
|
||||||
|
await File.WriteAllTextAsync(outputFile, json);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task UpdateUserInfoAsync()
|
||||||
|
{
|
||||||
|
await _dbService.CreateUsersDb([]);
|
||||||
|
await _dbService.InitializeUserInfoTablesAsync();
|
||||||
|
|
||||||
|
Dictionary<string, long> users = await _dbService.GetUsersAsync();
|
||||||
|
Console.WriteLine();
|
||||||
|
|
||||||
|
Log.Information("Updating User Info for '{UserCount}' users", users.Count);
|
||||||
|
AnsiConsole.Markup($"[green]Updating User Info for '{users.Count}' users\n[/]");
|
||||||
|
|
||||||
|
await AnsiConsole.Progress()
|
||||||
|
.Columns(new ProgressBarColumn(), new PercentageColumn(), new TaskDescriptionColumn { Alignment = Justify.Left })
|
||||||
|
.StartAsync(RunUpdateAsync);
|
||||||
|
|
||||||
|
async Task RunUpdateAsync(ProgressContext context)
|
||||||
|
{
|
||||||
|
ProgressTask? updateTask = null;
|
||||||
|
|
||||||
|
int maxUsernameLength = users.Keys.Max(s => s.Length);
|
||||||
|
|
||||||
|
foreach ((string username, long userId) in users)
|
||||||
|
{
|
||||||
|
string description = $"Updating '{username}'".PadRight(11 + maxUsernameLength);
|
||||||
|
double prevValue = updateTask?.Value ?? 0;
|
||||||
|
|
||||||
|
updateTask = context.AddTask(description, true, users.Count);
|
||||||
|
updateTask.Value = prevValue;
|
||||||
|
|
||||||
|
using (LogContext.PushProperty("Username", username))
|
||||||
|
using (LogContext.PushProperty("UserId", userId))
|
||||||
|
using (LogContext.PushProperty("UserNum", prevValue + 1))
|
||||||
|
using (LogContext.PushProperty("UserTotal", users.Count))
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
Log.Information("[{UserNum:0} of {UserTotal}] Updating User Info for for: {Username:l}");
|
||||||
|
UserEntities.UserInfo? userInfo = await _apiService.GetDetailedUserInfoAsync($"/users/{username}");
|
||||||
|
await _dbService.UpdateUserInfoAsync(userInfo);
|
||||||
|
|
||||||
|
updateTask.Description = $"{description} - COMPLETE";
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Log.Warning(ex, "[{UserNum:0} of {UserTotal}] Failed to update User Info for: {Username:l}");
|
||||||
|
AnsiConsole.Markup($"[red]Failed to update User Info for '{username}'\n[/]");
|
||||||
|
|
||||||
|
updateTask.Description = $"{description} - FAILED: {ex.Message}";
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
updateTask.Increment(1);
|
||||||
|
updateTask.StopTask();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<Dictionary<string, long>> GetUsersFromSpecificListsAsync(UserListResult allUsersAndLists, string[] listNames)
|
||||||
|
{
|
||||||
|
Config currentConfig = _configService.CurrentConfig;
|
||||||
|
Dictionary<string, long> usersFromLists = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
foreach (string name in listNames)
|
||||||
|
{
|
||||||
|
if (!allUsersAndLists.Lists.TryGetValue(name, out long listId))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", name, currentConfig.IncludeRestrictedSubscriptions);
|
||||||
|
AnsiConsole.MarkupLine($"[green]Getting Users from list '{name}' (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})[/]");
|
||||||
|
|
||||||
|
Dictionary<string, long> listUsernames = await _apiService.GetListUsers($"/lists/{listId}/users") ?? [];
|
||||||
|
|
||||||
|
foreach ((string username, long userId) in listUsernames)
|
||||||
|
{
|
||||||
|
if (usersFromLists.ContainsKey(username))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
usersFromLists[username] = userId;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return usersFromLists;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Dictionary<string, long> GetUsersFromSpecificUsernames(UserListResult allUsersAndLists, HashSet<string> usernames)
|
||||||
|
{
|
||||||
|
Dictionary<string, long> filteredUsers = allUsersAndLists.Users
|
||||||
|
.Where(u => usernames.Contains(u.Key))
|
||||||
|
.ToDictionary(u => u.Key, u => u.Value);
|
||||||
|
|
||||||
|
return filteredUsers;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<UserListResult> GetAvailableUsersAsync()
|
||||||
|
{
|
||||||
|
Config currentConfig = _configService.CurrentConfig;
|
||||||
|
UserListResult result = new()
|
||||||
|
{
|
||||||
|
Users = new(StringComparer.OrdinalIgnoreCase)
|
||||||
|
};
|
||||||
|
|
||||||
|
await FetchUsersAsync();
|
||||||
|
await FetchListsAsync();
|
||||||
|
|
||||||
|
AnsiConsole.WriteLine();
|
||||||
|
|
||||||
|
await _dbService.CreateUsersDb(result.Users);
|
||||||
|
await _dbService.InitializeUserInfoTablesAsync();
|
||||||
|
|
||||||
|
await UpdateNonNudeListAsync();
|
||||||
|
|
||||||
|
return result;
|
||||||
|
|
||||||
|
async Task FetchUsersAsync()
|
||||||
|
{
|
||||||
|
Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
|
||||||
|
AnsiConsole.MarkupLine($"[green]Getting Active Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})[/]");
|
||||||
|
|
||||||
|
Dictionary<string, long>? activeSubs = await _apiService.GetActiveSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
|
||||||
|
AddToResult(activeSubs);
|
||||||
|
|
||||||
|
if (currentConfig.IncludeExpiredSubscriptions)
|
||||||
|
{
|
||||||
|
Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
|
||||||
|
AnsiConsole.MarkupLine($"[green]Getting Expired Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})[/]");
|
||||||
|
|
||||||
|
Dictionary<string, long>? expiredSubs = await _apiService.GetExpiredSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
|
||||||
|
AddToResult(expiredSubs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task FetchListsAsync()
|
||||||
|
{
|
||||||
|
Log.Information("Getting Lists");
|
||||||
|
result.Lists = await _apiService.GetLists("/lists") ?? [];
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task UpdateNonNudeListAsync()
|
||||||
|
{
|
||||||
|
const string LIST_NAME = "NonNude";
|
||||||
|
const long LIST_ID = 1220021758;
|
||||||
|
|
||||||
|
HashSet<string> listNames = new(StringComparer.OrdinalIgnoreCase);
|
||||||
|
|
||||||
|
if (result.Lists.ContainsKey(LIST_NAME))
|
||||||
|
listNames.Add(LIST_NAME);
|
||||||
|
|
||||||
|
string? nameById = result.Lists.FirstOrDefault(l => l.Value == LIST_ID).Key;
|
||||||
|
if (!string.IsNullOrWhiteSpace(nameById))
|
||||||
|
listNames.Add(nameById);
|
||||||
|
|
||||||
|
Dictionary<string, long> usersInNonNudeLists = await GetUsersFromSpecificListsAsync(result, [.. listNames]);
|
||||||
|
|
||||||
|
AnsiConsole.MarkupLine($"[grey]Updating Non-Nude collection with {usersInNonNudeLists.Count} Users[/]");
|
||||||
|
await _dbService.UpdateNonNudeCollectionAsync(usersInNonNudeLists);
|
||||||
|
|
||||||
|
AnsiConsole.WriteLine();
|
||||||
|
}
|
||||||
|
|
||||||
|
void AddToResult(Dictionary<string, long>? subscriptions)
|
||||||
|
{
|
||||||
|
foreach ((string username, long userId) in subscriptions ?? [])
|
||||||
|
{
|
||||||
|
if (result.Users.TryAdd(username, userId))
|
||||||
|
Log.Debug($"Name: {username} ID: {userId}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static ILogger LoggerWithConfigContext(Config config, CajetanConfig cajetanConfig)
|
||||||
|
=> Log.Logger
|
||||||
|
.ForContext(nameof(Config.DownloadPath), config.DownloadPath)
|
||||||
|
.ForContext(nameof(Config.DownloadPosts), config.DownloadPosts)
|
||||||
|
.ForContext(nameof(Config.DownloadPaidPosts), config.DownloadPaidPosts)
|
||||||
|
.ForContext(nameof(Config.DownloadMessages), config.DownloadMessages)
|
||||||
|
.ForContext(nameof(Config.DownloadPaidMessages), config.DownloadPaidMessages)
|
||||||
|
.ForContext(nameof(Config.DownloadStories), config.DownloadStories)
|
||||||
|
.ForContext(nameof(Config.DownloadStreams), config.DownloadStreams)
|
||||||
|
.ForContext(nameof(Config.DownloadHighlights), config.DownloadHighlights)
|
||||||
|
.ForContext(nameof(Config.DownloadArchived), config.DownloadArchived)
|
||||||
|
.ForContext(nameof(Config.DownloadAvatarHeaderPhoto), config.DownloadAvatarHeaderPhoto)
|
||||||
|
.ForContext(nameof(Config.DownloadImages), config.DownloadImages)
|
||||||
|
.ForContext(nameof(Config.DownloadVideos), config.DownloadVideos)
|
||||||
|
.ForContext(nameof(Config.DownloadAudios), config.DownloadAudios)
|
||||||
|
.ForContext(nameof(Config.IgnoreOwnMessages), config.IgnoreOwnMessages)
|
||||||
|
.ForContext(nameof(Config.DownloadPostsIncrementally), config.DownloadPostsIncrementally)
|
||||||
|
.ForContext(nameof(Config.BypassContentForCreatorsWhoNoLongerExist), config.BypassContentForCreatorsWhoNoLongerExist)
|
||||||
|
.ForContext(nameof(Config.SkipAds), config.SkipAds)
|
||||||
|
.ForContext(nameof(Config.IncludeExpiredSubscriptions), config.IncludeExpiredSubscriptions)
|
||||||
|
.ForContext(nameof(Config.IncludeRestrictedSubscriptions), config.IncludeRestrictedSubscriptions)
|
||||||
|
.ForContext(nameof(CajetanConfig.NonInteractiveSpecificLists), cajetanConfig.NonInteractiveSpecificLists)
|
||||||
|
.ForContext(nameof(CajetanConfig.NonInteractiveSpecificUsers), cajetanConfig.NonInteractiveSpecificUsers);
|
||||||
|
}
|
||||||
37
Dockerfile
37
Dockerfile
@ -1,10 +1,7 @@
|
|||||||
FROM alpine:3.23 AS build
|
FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
|
||||||
|
|
||||||
ARG VERSION
|
ARG VERSION
|
||||||
|
|
||||||
RUN apk --no-cache --repository community add \
|
|
||||||
dotnet10-sdk
|
|
||||||
|
|
||||||
# Copy source code
|
# Copy source code
|
||||||
COPY ["OF DL.sln", "/src/OF DL.sln"]
|
COPY ["OF DL.sln", "/src/OF DL.sln"]
|
||||||
COPY ["OF DL", "/src/OF DL"]
|
COPY ["OF DL", "/src/OF DL"]
|
||||||
@ -22,21 +19,24 @@ RUN /src/out/OF\ DL --non-interactive || true && \
|
|||||||
mv /src/updated_config.conf /src/config.conf
|
mv /src/updated_config.conf /src/config.conf
|
||||||
|
|
||||||
|
|
||||||
FROM alpine:3.23 AS final
|
FROM mcr.microsoft.com/dotnet/runtime:10.0 AS final
|
||||||
|
|
||||||
# Install dependencies
|
# Install dependencies
|
||||||
RUN apk --no-cache --repository community add \
|
RUN apt-get update \
|
||||||
bash \
|
&& apt-get install -y \
|
||||||
tini \
|
tini \
|
||||||
dotnet10-runtime \
|
ffmpeg \
|
||||||
ffmpeg7 \
|
|
||||||
udev \
|
|
||||||
ttf-freefont \
|
|
||||||
chromium \
|
|
||||||
supervisor \
|
supervisor \
|
||||||
xvfb \
|
xvfb \
|
||||||
x11vnc \
|
x11vnc \
|
||||||
novnc
|
novnc \
|
||||||
|
npm \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN npx playwright install-deps
|
||||||
|
|
||||||
|
RUN apt-get remove --purge -y npm \
|
||||||
|
&& apt-get autoremove -y
|
||||||
|
|
||||||
# Redirect webroot to vnc.html instead of displaying directory listing
|
# Redirect webroot to vnc.html instead of displaying directory listing
|
||||||
RUN echo "<!DOCTYPE html><html><head><meta http-equiv=\"Refresh\" content=\"0; url='vnc.html'\" /></head><body></body></html>" > /usr/share/novnc/index.html
|
RUN echo "<!DOCTYPE html><html><head><meta http-equiv=\"Refresh\" content=\"0; url='vnc.html'\" /></head><body></body></html>" > /usr/share/novnc/index.html
|
||||||
@ -55,13 +55,14 @@ COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
|
|||||||
COPY docker/entrypoint.sh /app/entrypoint.sh
|
COPY docker/entrypoint.sh /app/entrypoint.sh
|
||||||
RUN chmod +x /app/entrypoint.sh
|
RUN chmod +x /app/entrypoint.sh
|
||||||
|
|
||||||
ENV DISPLAY=:0.0 \
|
ENV DEBIAN_FRONTEND="noninteractive" \
|
||||||
DISPLAY_WIDTH=1024 \
|
DISPLAY=:0.0 \
|
||||||
|
DISPLAY_WIDTH=1366 \
|
||||||
DISPLAY_HEIGHT=768 \
|
DISPLAY_HEIGHT=768 \
|
||||||
OFDL_PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium \
|
OFDL_DOCKER=true \
|
||||||
OFDL_DOCKER=true
|
PLAYWRIGHT_BROWSERS_PATH=/config/chromium
|
||||||
|
|
||||||
EXPOSE 8080
|
EXPOSE 8080
|
||||||
WORKDIR /config
|
WORKDIR /config
|
||||||
ENTRYPOINT ["/sbin/tini", "--"]
|
ENTRYPOINT ["/usr/bin/tini", "--"]
|
||||||
CMD ["/app/entrypoint.sh"]
|
CMD ["/app/entrypoint.sh"]
|
||||||
|
|||||||
@ -9,4 +9,6 @@ public static class Constants
|
|||||||
public const int WidevineRetryDelay = 10;
|
public const int WidevineRetryDelay = 10;
|
||||||
|
|
||||||
public const int WidevineMaxRetries = 3;
|
public const int WidevineMaxRetries = 3;
|
||||||
|
|
||||||
|
public const int DrmDownloadMaxRetries = 3;
|
||||||
}
|
}
|
||||||
|
|||||||
22
OF DL.Core/Helpers/ExitHelper.cs
Normal file
22
OF DL.Core/Helpers/ExitHelper.cs
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
using OF_DL.Services;
|
||||||
|
using Serilog;
|
||||||
|
|
||||||
|
namespace OF_DL.Helpers;
|
||||||
|
|
||||||
|
public class ExitHelper(IDownloadEventHandler eventHandler)
|
||||||
|
{
|
||||||
|
private readonly IDownloadEventHandler _eventHandler = eventHandler;
|
||||||
|
|
||||||
|
public void ExitWithCode(int exitCode)
|
||||||
|
{
|
||||||
|
Console.WriteLine();
|
||||||
|
|
||||||
|
_eventHandler?.OnMessage($"Exiting run with Code '{exitCode}'..");
|
||||||
|
|
||||||
|
Log.CloseAndFlush();
|
||||||
|
Log.CloseAndFlush();
|
||||||
|
Task.Delay(3000).GetAwaiter().GetResult();
|
||||||
|
|
||||||
|
Environment.Exit(exitCode);
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -76,6 +76,7 @@ public class Config : IFileNameFormatConfig
|
|||||||
[ToggleableConfig] public bool NonInteractiveModePurchasedTab { get; set; }
|
[ToggleableConfig] public bool NonInteractiveModePurchasedTab { get; set; }
|
||||||
|
|
||||||
public string? FFmpegPath { get; set; } = "";
|
public string? FFmpegPath { get; set; } = "";
|
||||||
|
public string? FFprobePath { get; set; } = "";
|
||||||
|
|
||||||
[ToggleableConfig] public bool BypassContentForCreatorsWhoNoLongerExist { get; set; }
|
[ToggleableConfig] public bool BypassContentForCreatorsWhoNoLongerExist { get; set; }
|
||||||
|
|
||||||
@ -95,6 +96,8 @@ public class Config : IFileNameFormatConfig
|
|||||||
[JsonConverter(typeof(StringEnumConverter))]
|
[JsonConverter(typeof(StringEnumConverter))]
|
||||||
public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
|
public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
|
||||||
|
|
||||||
|
public double DrmVideoDurationMatchThreshold { get; set; } = 0.98;
|
||||||
|
|
||||||
// When enabled, post/message text is stored as-is without XML stripping.
|
// When enabled, post/message text is stored as-is without XML stripping.
|
||||||
[ToggleableConfig] public bool DisableTextSanitization { get; set; }
|
[ToggleableConfig] public bool DisableTextSanitization { get; set; }
|
||||||
|
|
||||||
@ -115,22 +118,22 @@ public class Config : IFileNameFormatConfig
|
|||||||
|
|
||||||
if (CreatorConfigs.TryGetValue(username, out CreatorConfig? creatorConfig))
|
if (CreatorConfigs.TryGetValue(username, out CreatorConfig? creatorConfig))
|
||||||
{
|
{
|
||||||
if (creatorConfig.PaidPostFileNameFormat != null)
|
if (!string.IsNullOrEmpty(creatorConfig.PaidPostFileNameFormat))
|
||||||
{
|
{
|
||||||
combinedFilenameFormatConfig.PaidPostFileNameFormat = creatorConfig.PaidPostFileNameFormat;
|
combinedFilenameFormatConfig.PaidPostFileNameFormat = creatorConfig.PaidPostFileNameFormat;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (creatorConfig.PostFileNameFormat != null)
|
if (!string.IsNullOrEmpty(creatorConfig.PostFileNameFormat))
|
||||||
{
|
{
|
||||||
combinedFilenameFormatConfig.PostFileNameFormat = creatorConfig.PostFileNameFormat;
|
combinedFilenameFormatConfig.PostFileNameFormat = creatorConfig.PostFileNameFormat;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (creatorConfig.PaidMessageFileNameFormat != null)
|
if (!string.IsNullOrEmpty(creatorConfig.PaidMessageFileNameFormat))
|
||||||
{
|
{
|
||||||
combinedFilenameFormatConfig.PaidMessageFileNameFormat = creatorConfig.PaidMessageFileNameFormat;
|
combinedFilenameFormatConfig.PaidMessageFileNameFormat = creatorConfig.PaidMessageFileNameFormat;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (creatorConfig.MessageFileNameFormat != null)
|
if (!string.IsNullOrEmpty(creatorConfig.MessageFileNameFormat))
|
||||||
{
|
{
|
||||||
combinedFilenameFormatConfig.MessageFileNameFormat = creatorConfig.MessageFileNameFormat;
|
combinedFilenameFormatConfig.MessageFileNameFormat = creatorConfig.MessageFileNameFormat;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -17,6 +17,8 @@ public class CreatorDownloadResult
|
|||||||
public int MessagesCount { get; set; }
|
public int MessagesCount { get; set; }
|
||||||
|
|
||||||
public int PaidMessagesCount { get; set; }
|
public int PaidMessagesCount { get; set; }
|
||||||
|
|
||||||
|
public CreatorDownloadResult? NewDownloads { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
public class UserListResult
|
public class UserListResult
|
||||||
|
|||||||
36
OF DL.Core/Models/Entities/Users/UserInfo.cs
Normal file
36
OF DL.Core/Models/Entities/Users/UserInfo.cs
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
namespace OF_DL.Models.Entities.Users;
|
||||||
|
|
||||||
|
public class UserInfo : User
|
||||||
|
{
|
||||||
|
public long? Id { get; set; }
|
||||||
|
|
||||||
|
public string? SubscribePrice { get; set; }
|
||||||
|
public string? CurrentSubscribePrice { get; set; }
|
||||||
|
|
||||||
|
public bool? IsPaywallRequired { get; set; }
|
||||||
|
public bool? IsActive { get; set; }
|
||||||
|
public bool? IsRestricted { get; set; }
|
||||||
|
|
||||||
|
public bool? SubscribedBy { get; set; }
|
||||||
|
public bool? SubscribedByExpire { get; set; }
|
||||||
|
public DateTimeOffset? SubscribedByExpireDate { get; set; }
|
||||||
|
public bool? SubscribedByAutoprolong { get; set; }
|
||||||
|
public bool? IsPendingAutoprolong { get; set; }
|
||||||
|
|
||||||
|
public bool? SubscribedIsExpiredNow { get; set; }
|
||||||
|
|
||||||
|
public bool? SubscribedOn { get; set; }
|
||||||
|
public bool? SubscribedOnExpiredNow { get; set; }
|
||||||
|
public string? SubscribedOnDuration { get; set; }
|
||||||
|
|
||||||
|
public string? About { get; set; }
|
||||||
|
|
||||||
|
public int? PostsCount { get; set; }
|
||||||
|
public int? ArchivedPostsCount { get; set; }
|
||||||
|
public int? PrivateArchivedPostsCount { get; set; }
|
||||||
|
|
||||||
|
public int? PhotosCount { get; set; }
|
||||||
|
public int? VideosCount { get; set; }
|
||||||
|
public int? AudiosCount { get; set; }
|
||||||
|
public int? MediasCount { get; set; }
|
||||||
|
}
|
||||||
@ -14,6 +14,14 @@ public class StartupResult
|
|||||||
|
|
||||||
public string? FfmpegVersion { get; set; }
|
public string? FfmpegVersion { get; set; }
|
||||||
|
|
||||||
|
public bool FfprobeFound { get; set; }
|
||||||
|
|
||||||
|
public bool FfprobePathAutoDetected { get; set; }
|
||||||
|
|
||||||
|
public string? FfprobePath { get; set; }
|
||||||
|
|
||||||
|
public string? FfprobeVersion { get; set; }
|
||||||
|
|
||||||
public bool ClientIdBlobMissing { get; set; }
|
public bool ClientIdBlobMissing { get; set; }
|
||||||
|
|
||||||
public bool DevicePrivateKeyMissing { get; set; }
|
public bool DevicePrivateKeyMissing { get; set; }
|
||||||
|
|||||||
@ -5,6 +5,7 @@
|
|||||||
<RootNamespace>OF_DL</RootNamespace>
|
<RootNamespace>OF_DL</RootNamespace>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
|
<DebugType>embedded</DebugType>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
@ -13,9 +14,9 @@
|
|||||||
<PackageReference Include="HtmlAgilityPack" Version="1.12.4"/>
|
<PackageReference Include="HtmlAgilityPack" Version="1.12.4"/>
|
||||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.3"/>
|
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.3"/>
|
||||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.3"/>
|
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.3"/>
|
||||||
|
<PackageReference Include="Microsoft.Playwright" Version="1.58.0"/>
|
||||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.4"/>
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.4"/>
|
||||||
<PackageReference Include="protobuf-net" Version="3.2.56"/>
|
<PackageReference Include="protobuf-net" Version="3.2.56"/>
|
||||||
<PackageReference Include="PuppeteerSharp" Version="20.2.6"/>
|
|
||||||
<PackageReference Include="Serilog" Version="4.3.1"/>
|
<PackageReference Include="Serilog" Version="4.3.1"/>
|
||||||
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1"/>
|
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1"/>
|
||||||
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0"/>
|
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0"/>
|
||||||
@ -23,4 +24,8 @@
|
|||||||
<PackageReference Include="xFFmpeg.NET" Version="7.2.0"/>
|
<PackageReference Include="xFFmpeg.NET" Version="7.2.0"/>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<InternalsVisibleTo Include="Cajetan.OF-DL" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
using System.Globalization;
|
using System.Globalization;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
using System.Xml;
|
||||||
using System.Xml.Linq;
|
using System.Xml.Linq;
|
||||||
using Newtonsoft.Json;
|
using Newtonsoft.Json;
|
||||||
using Newtonsoft.Json.Linq;
|
using Newtonsoft.Json.Linq;
|
||||||
@ -43,7 +44,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
|||||||
{
|
{
|
||||||
private const int MaxAttempts = 30;
|
private const int MaxAttempts = 30;
|
||||||
private const int DelayBetweenAttempts = 3000;
|
private const int DelayBetweenAttempts = 3000;
|
||||||
private static readonly JsonSerializerSettings s_mJsonSerializerSettings;
|
protected static readonly JsonSerializerSettings s_mJsonSerializerSettings;
|
||||||
private static DateTime? s_cachedDynamicRulesExpiration;
|
private static DateTime? s_cachedDynamicRulesExpiration;
|
||||||
private static DynamicRules? s_cachedDynamicRules;
|
private static DynamicRules? s_cachedDynamicRules;
|
||||||
|
|
||||||
@ -147,7 +148,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
|||||||
return headers;
|
return headers;
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool HasSignedRequestAuth()
|
protected bool HasSignedRequestAuth()
|
||||||
{
|
{
|
||||||
Auth? currentAuth = authService.CurrentAuth;
|
Auth? currentAuth = authService.CurrentAuth;
|
||||||
return currentAuth is { UserId: not null, Cookie: not null, UserAgent: not null, XBc: not null };
|
return currentAuth is { UserId: not null, Cookie: not null, UserAgent: not null, XBc: not null };
|
||||||
@ -2575,19 +2576,26 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
|||||||
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Retrieves the Widevine PSSH from an MPD manifest.
|
/// Retrieves DRM metadata (PSSH, Last-Modified, and duration) from an MPD manifest
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="mpdUrl">The MPD URL.</param>
|
/// <param name="mpdUrl">The MPD URL.</param>
|
||||||
/// <param name="policy">CloudFront policy token.</param>
|
/// <param name="policy">CloudFront policy token.</param>
|
||||||
/// <param name="signature">CloudFront signature token.</param>
|
/// <param name="signature">CloudFront signature token.</param>
|
||||||
/// <param name="kvp">CloudFront key pair ID.</param>
|
/// <param name="kvp">CloudFront key pair ID.</param>
|
||||||
/// <returns>The PSSH value or an empty string.</returns>
|
/// <returns>Tuple with PSSH, Last-Modified, and duration seconds.</returns>
|
||||||
public async Task<string> GetDrmMpdPssh(string mpdUrl, string policy, string signature, string kvp)
|
public async Task<(string pssh, DateTime lastModified, double? durationSeconds)> GetDrmMpdInfo(
|
||||||
|
string mpdUrl, string policy, string signature, string kvp)
|
||||||
{
|
{
|
||||||
|
Log.Debug("Calling GetDrmMpdInfo");
|
||||||
|
Log.Debug("mpdUrl: {MpdUrl}", mpdUrl);
|
||||||
|
Log.Debug("policy: {Policy}", policy);
|
||||||
|
Log.Debug("signature: {Signature}", signature);
|
||||||
|
Log.Debug("kvp: {Kvp}", kvp);
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
Auth? currentAuth = authService.CurrentAuth;
|
Auth? currentAuth = authService.CurrentAuth;
|
||||||
if (currentAuth == null || currentAuth.UserAgent == null || currentAuth.Cookie == null)
|
if (currentAuth?.UserAgent == null || currentAuth.Cookie == null)
|
||||||
{
|
{
|
||||||
throw new Exception("Auth service is missing required fields");
|
throw new Exception("Auth service is missing required fields");
|
||||||
}
|
}
|
||||||
@ -2598,70 +2606,44 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
|||||||
request.Headers.Add("Accept", "*/*");
|
request.Headers.Add("Accept", "*/*");
|
||||||
request.Headers.Add("Cookie",
|
request.Headers.Add("Cookie",
|
||||||
$"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {currentAuth.Cookie};");
|
$"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {currentAuth.Cookie};");
|
||||||
|
|
||||||
using HttpResponseMessage response = await client.SendAsync(request);
|
using HttpResponseMessage response = await client.SendAsync(request);
|
||||||
response.EnsureSuccessStatusCode();
|
response.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
|
DateTime lastModified = response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now;
|
||||||
|
if (response.Content.Headers.LastModified == null
|
||||||
|
&& response.Headers.TryGetValues("Last-Modified", out IEnumerable<string>? lastModifiedValues))
|
||||||
|
{
|
||||||
|
string? lastModifiedRaw = lastModifiedValues.FirstOrDefault();
|
||||||
|
if (!string.IsNullOrWhiteSpace(lastModifiedRaw)
|
||||||
|
&& DateTimeOffset.TryParse(lastModifiedRaw, out DateTimeOffset parsedLastModified))
|
||||||
|
{
|
||||||
|
lastModified = parsedLastModified.LocalDateTime;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
string body = await response.Content.ReadAsStringAsync();
|
string body = await response.Content.ReadAsStringAsync();
|
||||||
XNamespace cenc = "urn:mpeg:cenc:2013";
|
|
||||||
XDocument xmlDoc = XDocument.Parse(body);
|
XDocument xmlDoc = XDocument.Parse(body);
|
||||||
IEnumerable<XElement> psshElements = xmlDoc.Descendants(cenc + "pssh");
|
|
||||||
string pssh = psshElements.ElementAt(1).Value;
|
|
||||||
|
|
||||||
return pssh;
|
XNamespace cenc = "urn:mpeg:cenc:2013";
|
||||||
|
List<XElement> psshElements = xmlDoc.Descendants(cenc + "pssh").ToList();
|
||||||
|
string pssh = psshElements.Skip(1).FirstOrDefault()?.Value
|
||||||
|
?? psshElements.FirstOrDefault()?.Value
|
||||||
|
?? string.Empty;
|
||||||
|
|
||||||
|
string? durationText = xmlDoc.Root?.Attribute("mediaPresentationDuration")?.Value
|
||||||
|
?? xmlDoc.Root?.Elements().FirstOrDefault(e => e.Name.LocalName == "Period")
|
||||||
|
?.Attribute("duration")?.Value;
|
||||||
|
double? durationSeconds = ParseDurationSeconds(durationText);
|
||||||
|
|
||||||
|
return (pssh, lastModified, durationSeconds);
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
ExceptionLoggerHelper.LogException(ex);
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
}
|
}
|
||||||
|
|
||||||
return string.Empty;
|
return (string.Empty, DateTime.Now, null);
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Retrieves the Last-Modified timestamp for an MPD manifest.
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="mpdUrl">The MPD URL.</param>
|
|
||||||
/// <param name="policy">CloudFront policy token.</param>
|
|
||||||
/// <param name="signature">CloudFront signature token.</param>
|
|
||||||
/// <param name="kvp">CloudFront key pair ID.</param>
|
|
||||||
/// <returns>The last modified timestamp.</returns>
|
|
||||||
public async Task<DateTime> GetDrmMpdLastModified(string mpdUrl, string policy, string signature, string kvp)
|
|
||||||
{
|
|
||||||
Log.Debug("Calling GetDrmMpdLastModified");
|
|
||||||
Log.Debug($"mpdUrl: {mpdUrl}");
|
|
||||||
Log.Debug($"policy: {policy}");
|
|
||||||
Log.Debug($"signature: {signature}");
|
|
||||||
Log.Debug($"kvp: {kvp}");
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
Auth? currentAuth = authService.CurrentAuth;
|
|
||||||
if (currentAuth == null || currentAuth.UserAgent == null || currentAuth.Cookie == null)
|
|
||||||
{
|
|
||||||
throw new Exception("Auth service is missing required fields");
|
|
||||||
}
|
|
||||||
|
|
||||||
HttpClient client = new();
|
|
||||||
HttpRequestMessage request = new(HttpMethod.Get, mpdUrl);
|
|
||||||
request.Headers.Add("user-agent", currentAuth.UserAgent);
|
|
||||||
request.Headers.Add("Accept", "*/*");
|
|
||||||
request.Headers.Add("Cookie",
|
|
||||||
$"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {currentAuth.Cookie};");
|
|
||||||
using HttpResponseMessage response =
|
|
||||||
await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
|
|
||||||
response.EnsureSuccessStatusCode();
|
|
||||||
DateTime lastmodified = response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now;
|
|
||||||
|
|
||||||
Log.Debug($"Last modified: {lastmodified}");
|
|
||||||
|
|
||||||
return lastmodified;
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
ExceptionLoggerHelper.LogException(ex);
|
|
||||||
}
|
|
||||||
|
|
||||||
return DateTime.Now;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@ -2772,12 +2754,12 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint,
|
protected async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint,
|
||||||
HttpClient client)
|
HttpClient client, HttpMethod? method = null, object? reqBody = null)
|
||||||
{
|
{
|
||||||
Log.Debug("Calling BuildHeaderAndExecuteRequests");
|
Log.Debug("Calling BuildHeaderAndExecuteRequests");
|
||||||
|
|
||||||
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
|
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method, reqBody);
|
||||||
using HttpResponseMessage response = await client.SendAsync(request);
|
using HttpResponseMessage response = await client.SendAsync(request);
|
||||||
response.EnsureSuccessStatusCode();
|
response.EnsureSuccessStatusCode();
|
||||||
string body = await response.Content.ReadAsStringAsync();
|
string body = await response.Content.ReadAsStringAsync();
|
||||||
@ -2788,16 +2770,22 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
|
protected Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
|
||||||
string endpoint)
|
string endpoint, HttpMethod? method = null, object? reqBody = null)
|
||||||
{
|
{
|
||||||
Log.Debug("Calling BuildHttpRequestMessage");
|
Log.Debug("Calling BuildHttpRequestMessage");
|
||||||
|
|
||||||
string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
string queryParams = "";
|
||||||
|
|
||||||
|
if (getParams.Count != 0)
|
||||||
|
queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
||||||
|
|
||||||
Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
|
Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
|
||||||
|
|
||||||
HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.ApiUrl}{endpoint}{queryParams}");
|
HttpRequestMessage request = new(method ?? HttpMethod.Get, $"{Constants.ApiUrl}{endpoint}{queryParams}");
|
||||||
|
|
||||||
|
if ((method == HttpMethod.Post || method == HttpMethod.Put || method == HttpMethod.Patch) && reqBody != null)
|
||||||
|
request.Content = new StringContent(JsonConvert.SerializeObject(reqBody, s_mJsonSerializerSettings), Encoding.UTF8, "application/json");
|
||||||
|
|
||||||
Log.Debug($"Full request URL: {Constants.ApiUrl}{endpoint}{queryParams}");
|
Log.Debug($"Full request URL: {Constants.ApiUrl}{endpoint}{queryParams}");
|
||||||
|
|
||||||
@ -2809,6 +2797,24 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
|||||||
return Task.FromResult(request);
|
return Task.FromResult(request);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static double? ParseDurationSeconds(string? iso8601Duration)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(iso8601Duration))
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
TimeSpan duration = XmlConvert.ToTimeSpan(iso8601Duration);
|
||||||
|
return duration.TotalSeconds > 0 ? duration.TotalSeconds : null;
|
||||||
|
}
|
||||||
|
catch (FormatException)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private static double ConvertToUnixTimestampWithMicrosecondPrecision(DateTime date)
|
private static double ConvertToUnixTimestampWithMicrosecondPrecision(DateTime date)
|
||||||
{
|
{
|
||||||
DateTime origin = new(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
|
DateTime origin = new(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
|
||||||
@ -2821,7 +2827,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
|||||||
private static bool IsStringOnlyDigits(string input) => input.All(char.IsDigit);
|
private static bool IsStringOnlyDigits(string input) => input.All(char.IsDigit);
|
||||||
|
|
||||||
|
|
||||||
private HttpClient GetHttpClient()
|
protected HttpClient GetHttpClient()
|
||||||
{
|
{
|
||||||
HttpClient client = new();
|
HttpClient client = new();
|
||||||
if (configService.CurrentConfig.Timeout is > 0)
|
if (configService.CurrentConfig.Timeout is > 0)
|
||||||
@ -2832,7 +2838,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
|||||||
return client;
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static T? DeserializeJson<T>(string? body, JsonSerializerSettings? settings = null)
|
protected static T? DeserializeJson<T>(string? body, JsonSerializerSettings? settings = null)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(body))
|
if (string.IsNullOrWhiteSpace(body))
|
||||||
{
|
{
|
||||||
|
|||||||
@ -1,9 +1,8 @@
|
|||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
|
using Microsoft.Playwright;
|
||||||
using Newtonsoft.Json;
|
using Newtonsoft.Json;
|
||||||
using OF_DL.Models;
|
using OF_DL.Models;
|
||||||
using PuppeteerSharp;
|
|
||||||
using PuppeteerSharp.BrowserData;
|
|
||||||
using Serilog;
|
using Serilog;
|
||||||
using UserEntities = OF_DL.Models.Entities.Users;
|
using UserEntities = OF_DL.Models.Entities.Users;
|
||||||
|
|
||||||
@ -11,8 +10,25 @@ namespace OF_DL.Services;
|
|||||||
|
|
||||||
public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
||||||
{
|
{
|
||||||
private const int LoginTimeout = 600000; // 10 minutes
|
private const float LoginTimeout = 600000f; // 10 minutes
|
||||||
private const int FeedLoadTimeout = 60000; // 1 minute
|
private const float FeedLoadTimeout = 60000f; // 1 minute
|
||||||
|
private const int AdditionalWaitAfterPageLoad = 3000; // 3 seconds
|
||||||
|
|
||||||
|
private readonly string _userDataDir = Path.GetFullPath("chromium-data");
|
||||||
|
private const string InitScriptsDirName = "chromium-scripts";
|
||||||
|
|
||||||
|
private readonly BrowserTypeLaunchPersistentContextOptions _options = new()
|
||||||
|
{
|
||||||
|
Headless = false,
|
||||||
|
Channel = "chromium",
|
||||||
|
Args =
|
||||||
|
[
|
||||||
|
"--no-sandbox",
|
||||||
|
"--disable-setuid-sandbox",
|
||||||
|
"--disable-blink-features=AutomationControlled",
|
||||||
|
"--disable-infobars"
|
||||||
|
]
|
||||||
|
};
|
||||||
|
|
||||||
private readonly string[] _desiredCookies =
|
private readonly string[] _desiredCookies =
|
||||||
[
|
[
|
||||||
@ -20,14 +36,6 @@ public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
|||||||
"sess"
|
"sess"
|
||||||
];
|
];
|
||||||
|
|
||||||
private readonly LaunchOptions _options = new()
|
|
||||||
{
|
|
||||||
Headless = false,
|
|
||||||
Channel = ChromeReleaseChannel.Stable,
|
|
||||||
DefaultViewport = null,
|
|
||||||
Args = ["--no-sandbox", "--disable-setuid-sandbox"],
|
|
||||||
UserDataDir = Path.GetFullPath("chrome-data")
|
|
||||||
};
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets or sets the current authentication state.
|
/// Gets or sets the current authentication state.
|
||||||
@ -35,7 +43,7 @@ public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
|||||||
public Auth? CurrentAuth { get; set; }
|
public Auth? CurrentAuth { get; set; }
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Loads authentication data from disk.
|
/// Loads authentication data from the disk.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="filePath">The auth file path.</param>
|
/// <param name="filePath">The auth file path.</param>
|
||||||
/// <returns>True when auth data is loaded successfully.</returns>
|
/// <returns>True when auth data is loaded successfully.</returns>
|
||||||
@ -107,43 +115,37 @@ public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task SetupBrowser(bool runningInDocker)
|
private Task SetupBrowser(bool runningInDocker)
|
||||||
{
|
{
|
||||||
string? executablePath = Environment.GetEnvironmentVariable("OFDL_PUPPETEER_EXECUTABLE_PATH");
|
|
||||||
if (executablePath != null)
|
|
||||||
{
|
|
||||||
Log.Information(
|
|
||||||
"OFDL_PUPPETEER_EXECUTABLE_PATH environment variable found. Using browser executable path: {executablePath}",
|
|
||||||
executablePath);
|
|
||||||
_options.ExecutablePath = executablePath;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
BrowserFetcher browserFetcher = new();
|
|
||||||
List<InstalledBrowser> installedBrowsers = browserFetcher.GetInstalledBrowsers().ToList();
|
|
||||||
if (installedBrowsers.Count == 0)
|
|
||||||
{
|
|
||||||
Log.Information("Downloading browser.");
|
|
||||||
InstalledBrowser? downloadedBrowser = await browserFetcher.DownloadAsync();
|
|
||||||
Log.Information("Browser downloaded. Path: {executablePath}",
|
|
||||||
downloadedBrowser.GetExecutablePath());
|
|
||||||
_options.ExecutablePath = downloadedBrowser.GetExecutablePath();
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_options.ExecutablePath = installedBrowsers.First().GetExecutablePath();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (runningInDocker)
|
if (runningInDocker)
|
||||||
{
|
{
|
||||||
Log.Information("Running in Docker. Disabling sandbox and GPU.");
|
Log.Information("Running in Docker. Disabling sandbox and GPU.");
|
||||||
_options.Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-gpu"];
|
_options.Args =
|
||||||
|
[
|
||||||
|
"--no-sandbox", "--disable-setuid-sandbox", "--disable-gpu",
|
||||||
|
"--disable-blink-features=AutomationControlled", "--disable-infobars"
|
||||||
|
];
|
||||||
|
|
||||||
|
// If chromium is already downloaded, skip installation
|
||||||
|
string? playwrightBrowsersPath = Environment.GetEnvironmentVariable("PLAYWRIGHT_BROWSERS_PATH");
|
||||||
|
IEnumerable<string> folders = Directory.GetDirectories(playwrightBrowsersPath ?? "/config/chromium")
|
||||||
|
.Where(folder => folder.Contains("chromium-"));
|
||||||
|
|
||||||
|
if (folders.Any())
|
||||||
|
{
|
||||||
|
Log.Information("chromium already downloaded. Skipping install step.");
|
||||||
|
return Task.CompletedTask;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<string> GetBcToken(IPage page) =>
|
int exitCode = Program.Main(["install", "--with-deps", "chromium"]);
|
||||||
await page.EvaluateExpressionAsync<string>("window.localStorage.getItem('bcTokenSha') || ''");
|
return exitCode != 0
|
||||||
|
? throw new Exception($"Playwright chromium download failed. Exited with code {exitCode}")
|
||||||
|
: Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<string> GetBcToken(IPage page) =>
|
||||||
|
await page.EvaluateAsync<string>("window.localStorage.getItem('bcTokenSha') || ''");
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Normalizes the stored cookie string to only include required cookie values.
|
/// Normalizes the stored cookie string to only include required cookie values.
|
||||||
@ -194,10 +196,10 @@ public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public void Logout()
|
public void Logout()
|
||||||
{
|
{
|
||||||
if (Directory.Exists("chrome-data"))
|
if (Directory.Exists("chromium-data"))
|
||||||
{
|
{
|
||||||
Log.Information("Deleting chrome-data folder");
|
Log.Information("Deleting chromium-data folder");
|
||||||
Directory.Delete("chrome-data", true);
|
Directory.Delete("chromium-data", true);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (File.Exists("auth.json"))
|
if (File.Exists("auth.json"))
|
||||||
@ -211,18 +213,23 @@ public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
IBrowser? browser;
|
IBrowserContext? browser;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
browser = await Puppeteer.LaunchAsync(_options);
|
IPlaywright playwright = await Playwright.CreateAsync();
|
||||||
|
browser = await playwright.Chromium.LaunchPersistentContextAsync(_userDataDir, _options);
|
||||||
}
|
}
|
||||||
catch (ProcessException e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
if (e.Message.Contains("Failed to launch browser") && Directory.Exists(_options.UserDataDir))
|
if ((
|
||||||
|
e.Message.Contains("An error occurred trying to start process") ||
|
||||||
|
e.Message.Contains("The profile appears to be in use by another Chromium process")
|
||||||
|
) && Directory.Exists(_userDataDir))
|
||||||
{
|
{
|
||||||
Log.Error("Failed to launch browser. Deleting chrome-data directory and trying again.");
|
Log.Error("Failed to launch browser. Deleting chromium-data directory and trying again.");
|
||||||
Directory.Delete(_options.UserDataDir, true);
|
Directory.Delete(_userDataDir, true);
|
||||||
browser = await Puppeteer.LaunchAsync(_options);
|
IPlaywright playwright = await Playwright.CreateAsync();
|
||||||
|
browser = await playwright.Chromium.LaunchPersistentContextAsync(_userDataDir, _options);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -235,27 +242,39 @@ public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
|||||||
throw new Exception("Could not get browser");
|
throw new Exception("Could not get browser");
|
||||||
}
|
}
|
||||||
|
|
||||||
IPage[]? pages = await browser.PagesAsync();
|
IPage? page = browser.Pages[0];
|
||||||
IPage? page = pages.First();
|
|
||||||
|
|
||||||
if (page == null)
|
if (page == null)
|
||||||
{
|
{
|
||||||
throw new Exception("Could not get page");
|
throw new Exception("Could not get page");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
string exeDirectory = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ??
|
||||||
|
string.Empty;
|
||||||
|
string initScriptsDir = Path.Combine(exeDirectory, InitScriptsDirName);
|
||||||
|
if (Directory.Exists(initScriptsDir))
|
||||||
|
{
|
||||||
|
Log.Information("Loading init scripts from {initScriptsDir}", initScriptsDir);
|
||||||
|
foreach (string initScript in Directory.GetFiles(initScriptsDir, "*.js"))
|
||||||
|
{
|
||||||
|
Log.Debug("Loading init script {initScript}", initScript);
|
||||||
|
await page.AddInitScriptAsync(initScript);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Log.Debug("Navigating to OnlyFans.");
|
Log.Debug("Navigating to OnlyFans.");
|
||||||
await page.GoToAsync("https://onlyfans.com");
|
await page.GotoAsync("https://onlyfans.com");
|
||||||
|
|
||||||
Log.Debug("Waiting for user to login");
|
Log.Debug("Waiting for user to login");
|
||||||
await page.WaitForSelectorAsync(".b-feed", new WaitForSelectorOptions { Timeout = LoginTimeout });
|
await page.WaitForSelectorAsync(".b-feed", new PageWaitForSelectorOptions { Timeout = LoginTimeout });
|
||||||
Log.Debug("Feed element detected (user logged in)");
|
Log.Debug("Feed element detected (user logged in)");
|
||||||
|
|
||||||
await page.ReloadAsync();
|
await page.ReloadAsync(
|
||||||
|
new PageReloadOptions { Timeout = FeedLoadTimeout, WaitUntil = WaitUntilState.DOMContentLoaded });
|
||||||
|
|
||||||
|
// Wait for an additional time to ensure the DOM is fully loaded
|
||||||
|
await Task.Delay(AdditionalWaitAfterPageLoad);
|
||||||
|
|
||||||
await page.WaitForNavigationAsync(new NavigationOptions
|
|
||||||
{
|
|
||||||
WaitUntil = [WaitUntilNavigation.Networkidle2], Timeout = FeedLoadTimeout
|
|
||||||
});
|
|
||||||
Log.Debug("DOM loaded. Getting BC token and cookies ...");
|
Log.Debug("DOM loaded. Getting BC token and cookies ...");
|
||||||
|
|
||||||
string xBc;
|
string xBc;
|
||||||
@ -265,35 +284,40 @@ public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
|||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
{
|
{
|
||||||
Log.Error(e, "Error getting bcToken");
|
await browser.CloseAsync();
|
||||||
throw new Exception("Error getting bcToken");
|
throw new Exception($"Error getting bcToken. {e.Message}");
|
||||||
}
|
}
|
||||||
|
|
||||||
Dictionary<string, string> mappedCookies = (await page.GetCookiesAsync())
|
Dictionary<string, string> mappedCookies = (await browser.CookiesAsync())
|
||||||
.Where(cookie => cookie.Domain.Contains("onlyfans.com"))
|
.Where(cookie => cookie.Domain.Contains("onlyfans.com"))
|
||||||
.ToDictionary(cookie => cookie.Name, cookie => cookie.Value);
|
.ToDictionary(cookie => cookie.Name, cookie => cookie.Value);
|
||||||
|
|
||||||
mappedCookies.TryGetValue("auth_id", out string? userId);
|
mappedCookies.TryGetValue("auth_id", out string? userId);
|
||||||
if (userId == null)
|
if (userId == null)
|
||||||
{
|
{
|
||||||
|
await browser.CloseAsync();
|
||||||
throw new Exception("Could not find 'auth_id' cookie");
|
throw new Exception("Could not find 'auth_id' cookie");
|
||||||
}
|
}
|
||||||
|
|
||||||
mappedCookies.TryGetValue("sess", out string? sess);
|
mappedCookies.TryGetValue("sess", out string? sess);
|
||||||
if (sess == null)
|
if (sess == null)
|
||||||
{
|
{
|
||||||
|
await browser.CloseAsync();
|
||||||
throw new Exception("Could not find 'sess' cookie");
|
throw new Exception("Could not find 'sess' cookie");
|
||||||
}
|
}
|
||||||
|
|
||||||
string? userAgent = await browser.GetUserAgentAsync();
|
string userAgent = await page.EvaluateAsync<string>("navigator.userAgent");
|
||||||
if (userAgent == null)
|
if (string.IsNullOrWhiteSpace(userAgent))
|
||||||
{
|
{
|
||||||
|
await browser.CloseAsync();
|
||||||
throw new Exception("Could not get user agent");
|
throw new Exception("Could not get user agent");
|
||||||
}
|
}
|
||||||
|
|
||||||
string cookies = string.Join(" ", mappedCookies.Keys.Where(key => _desiredCookies.Contains(key))
|
string cookies = string.Join(" ", mappedCookies.Keys.Where(key => _desiredCookies.Contains(key))
|
||||||
.Select(key => $"${key}={mappedCookies[key]};"));
|
.Select(key => $"${key}={mappedCookies[key]};"));
|
||||||
|
|
||||||
|
await browser.CloseAsync();
|
||||||
|
|
||||||
return new Auth { Cookie = cookies, UserAgent = userAgent, UserId = userId, XBc = xBc };
|
return new Auth { Cookie = cookies, UserAgent = userAgent, UserId = userId, XBc = xBc };
|
||||||
}
|
}
|
||||||
catch (Exception e)
|
catch (Exception e)
|
||||||
|
|||||||
@ -1,3 +1,4 @@
|
|||||||
|
using System.Globalization;
|
||||||
using System.Reflection;
|
using System.Reflection;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using Akka.Configuration;
|
using Akka.Configuration;
|
||||||
@ -164,6 +165,7 @@ public class ConfigService(ILoggingService loggingService) : IConfigService
|
|||||||
|
|
||||||
// FFmpeg Settings
|
// FFmpeg Settings
|
||||||
FFmpegPath = hoconConfig.GetString("External.FFmpegPath"),
|
FFmpegPath = hoconConfig.GetString("External.FFmpegPath"),
|
||||||
|
FFprobePath = hoconConfig.GetString("External.FFprobePath", ""),
|
||||||
|
|
||||||
// Download Settings
|
// Download Settings
|
||||||
DownloadAvatarHeaderPhoto = hoconConfig.GetBoolean("Download.Media.DownloadAvatarHeaderPhoto"),
|
DownloadAvatarHeaderPhoto = hoconConfig.GetBoolean("Download.Media.DownloadAvatarHeaderPhoto"),
|
||||||
@ -194,11 +196,13 @@ public class ConfigService(ILoggingService loggingService) : IConfigService
|
|||||||
: null,
|
: null,
|
||||||
ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
|
ShowScrapeSize = hoconConfig.GetBoolean("Download.ShowScrapeSize"),
|
||||||
DisableTextSanitization =
|
DisableTextSanitization =
|
||||||
bool.TryParse(hoconConfig.GetString("Download.DisableTextSanitization", "false"), out bool dts)
|
bool.TryParse(hoconConfig.GetString("Download.DisableTextSanitization", "false"), out bool dts) &&
|
||||||
? dts
|
dts,
|
||||||
: false,
|
|
||||||
DownloadVideoResolution =
|
DownloadVideoResolution =
|
||||||
ParseVideoResolution(hoconConfig.GetString("Download.DownloadVideoResolution", "source")),
|
ParseVideoResolution(hoconConfig.GetString("Download.DownloadVideoResolution", "source")),
|
||||||
|
DrmVideoDurationMatchThreshold =
|
||||||
|
ParseDrmVideoDurationMatchThreshold(
|
||||||
|
hoconConfig.GetString("Download.DrmVideoDurationMatchThreshold", "0.98")),
|
||||||
|
|
||||||
// File Settings
|
// File Settings
|
||||||
PaidPostFileNameFormat = hoconConfig.GetString("File.PaidPostFileNameFormat"),
|
PaidPostFileNameFormat = hoconConfig.GetString("File.PaidPostFileNameFormat"),
|
||||||
@ -311,6 +315,7 @@ public class ConfigService(ILoggingService loggingService) : IConfigService
|
|||||||
hocon.AppendLine("# External Tools");
|
hocon.AppendLine("# External Tools");
|
||||||
hocon.AppendLine("External {");
|
hocon.AppendLine("External {");
|
||||||
hocon.AppendLine($" FFmpegPath = \"{config.FFmpegPath}\"");
|
hocon.AppendLine($" FFmpegPath = \"{config.FFmpegPath}\"");
|
||||||
|
hocon.AppendLine($" FFprobePath = \"{config.FFprobePath}\"");
|
||||||
hocon.AppendLine("}");
|
hocon.AppendLine("}");
|
||||||
|
|
||||||
hocon.AppendLine("# Download Settings");
|
hocon.AppendLine("# Download Settings");
|
||||||
@ -343,6 +348,8 @@ public class ConfigService(ILoggingService loggingService) : IConfigService
|
|||||||
hocon.AppendLine($" DisableTextSanitization = {config.DisableTextSanitization.ToString().ToLower()}");
|
hocon.AppendLine($" DisableTextSanitization = {config.DisableTextSanitization.ToString().ToLower()}");
|
||||||
hocon.AppendLine(
|
hocon.AppendLine(
|
||||||
$" DownloadVideoResolution = \"{(config.DownloadVideoResolution == VideoResolution.source ? "source" : config.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
|
$" DownloadVideoResolution = \"{(config.DownloadVideoResolution == VideoResolution.source ? "source" : config.DownloadVideoResolution.ToString().TrimStart('_'))}\"");
|
||||||
|
hocon.AppendLine(
|
||||||
|
$" DrmVideoDurationMatchThreshold = {config.DrmVideoDurationMatchThreshold.ToString(CultureInfo.InvariantCulture)}");
|
||||||
hocon.AppendLine("}");
|
hocon.AppendLine("}");
|
||||||
|
|
||||||
hocon.AppendLine("# File Settings");
|
hocon.AppendLine("# File Settings");
|
||||||
@ -492,4 +499,9 @@ public class ConfigService(ILoggingService loggingService) : IConfigService
|
|||||||
|
|
||||||
return Enum.Parse<VideoResolution>("_" + value, true);
|
return Enum.Parse<VideoResolution>("_" + value, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static double ParseDrmVideoDurationMatchThreshold(string value) =>
|
||||||
|
!double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double parsed)
|
||||||
|
? 0.98
|
||||||
|
: Math.Clamp(parsed, 0.01, 1.0);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -160,6 +160,7 @@ public class DownloadOrchestrationService(
|
|||||||
{
|
{
|
||||||
Config config = configService.CurrentConfig;
|
Config config = configService.CurrentConfig;
|
||||||
CreatorDownloadResult counts = new();
|
CreatorDownloadResult counts = new();
|
||||||
|
CreatorDownloadResult newCounts = new();
|
||||||
|
|
||||||
eventHandler.OnUserStarting(username);
|
eventHandler.OnUserStarting(username);
|
||||||
Log.Debug($"Scraping Data for {username}");
|
Log.Debug($"Scraping Data for {username}");
|
||||||
@ -185,14 +186,15 @@ public class DownloadOrchestrationService(
|
|||||||
posts => posts.PaidPosts.Values.ToList(),
|
posts => posts.PaidPosts.Values.ToList(),
|
||||||
async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users,
|
async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users,
|
||||||
clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
|
clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
|
||||||
eventHandler);
|
eventHandler,
|
||||||
|
n => newCounts.PaidPostCount = n);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.DownloadPosts)
|
if (config.DownloadPosts)
|
||||||
{
|
{
|
||||||
eventHandler.OnMessage(
|
eventHandler.OnMessage(
|
||||||
"Getting Posts (this may take a long time, depending on the number of Posts the creator has)");
|
"Getting Posts (this may take a long time, depending on the number of Posts the creator has)");
|
||||||
Log.Debug($"Calling DownloadFreePosts - {username}");
|
Log.Debug("Calling DownloadFreePosts - {Username}", username);
|
||||||
|
|
||||||
counts.PostCount = await DownloadContentTypeAsync("Posts",
|
counts.PostCount = await DownloadContentTypeAsync("Posts",
|
||||||
async statusReporter =>
|
async statusReporter =>
|
||||||
@ -202,7 +204,8 @@ public class DownloadOrchestrationService(
|
|||||||
posts => posts.Posts.Values.ToList(),
|
posts => posts.Posts.Values.ToList(),
|
||||||
async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users,
|
async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users,
|
||||||
clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
|
clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
|
||||||
eventHandler);
|
eventHandler,
|
||||||
|
n => newCounts.PostCount = n);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.DownloadArchived)
|
if (config.DownloadArchived)
|
||||||
@ -215,7 +218,8 @@ public class DownloadOrchestrationService(
|
|||||||
archived => archived.ArchivedPosts.Values.ToList(),
|
archived => archived.ArchivedPosts.Values.ToList(),
|
||||||
async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users,
|
async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users,
|
||||||
clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter),
|
clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter),
|
||||||
eventHandler);
|
eventHandler,
|
||||||
|
n => newCounts.ArchivedCount = n);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.DownloadStreams)
|
if (config.DownloadStreams)
|
||||||
@ -228,7 +232,8 @@ public class DownloadOrchestrationService(
|
|||||||
streams => streams.Streams.Values.ToList(),
|
streams => streams.Streams.Values.ToList(),
|
||||||
async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users,
|
async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users,
|
||||||
clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter),
|
clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter),
|
||||||
eventHandler);
|
eventHandler,
|
||||||
|
n => newCounts.StreamsCount = n);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.DownloadStories)
|
if (config.DownloadStories)
|
||||||
@ -252,6 +257,7 @@ public class DownloadOrchestrationService(
|
|||||||
|
|
||||||
eventHandler.OnDownloadComplete("Stories", result);
|
eventHandler.OnDownloadComplete("Stories", result);
|
||||||
counts.StoriesCount = result.TotalCount;
|
counts.StoriesCount = result.TotalCount;
|
||||||
|
newCounts.StoriesCount = result.NewDownloads;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -280,6 +286,7 @@ public class DownloadOrchestrationService(
|
|||||||
|
|
||||||
eventHandler.OnDownloadComplete("Highlights", result);
|
eventHandler.OnDownloadComplete("Highlights", result);
|
||||||
counts.HighlightsCount = result.TotalCount;
|
counts.HighlightsCount = result.TotalCount;
|
||||||
|
newCounts.HighlightsCount = result.NewDownloads;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -297,7 +304,8 @@ public class DownloadOrchestrationService(
|
|||||||
messages => messages.Messages.Values.ToList(),
|
messages => messages.Messages.Values.ToList(),
|
||||||
async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users,
|
async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users,
|
||||||
clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter),
|
clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter),
|
||||||
eventHandler);
|
eventHandler,
|
||||||
|
n => newCounts.MessagesCount = n);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (config.DownloadPaidMessages)
|
if (config.DownloadPaidMessages)
|
||||||
@ -310,10 +318,13 @@ public class DownloadOrchestrationService(
|
|||||||
paidMessages => paidMessages.PaidMessages.Values.ToList(),
|
paidMessages => paidMessages.PaidMessages.Values.ToList(),
|
||||||
async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users,
|
async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users,
|
||||||
clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter),
|
clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter),
|
||||||
eventHandler);
|
eventHandler,
|
||||||
|
n => newCounts.PaidMessagesCount = n);
|
||||||
}
|
}
|
||||||
|
|
||||||
eventHandler.OnUserComplete(username, counts);
|
eventHandler.OnUserComplete(username, counts);
|
||||||
|
|
||||||
|
counts.NewDownloads = newCounts;
|
||||||
return counts;
|
return counts;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -612,7 +623,8 @@ public class DownloadOrchestrationService(
|
|||||||
Func<T, int> getObjectCount,
|
Func<T, int> getObjectCount,
|
||||||
Func<T, List<string>?> getUrls,
|
Func<T, List<string>?> getUrls,
|
||||||
Func<T, IProgressReporter, Task<DownloadResult>> downloadData,
|
Func<T, IProgressReporter, Task<DownloadResult>> downloadData,
|
||||||
IDownloadEventHandler eventHandler)
|
IDownloadEventHandler eventHandler,
|
||||||
|
Action<int>? newPostAssignmentAction)
|
||||||
{
|
{
|
||||||
T data = await eventHandler.WithStatusAsync($"Getting {contentType}",
|
T data = await eventHandler.WithStatusAsync($"Getting {contentType}",
|
||||||
async statusReporter => await fetchData(statusReporter));
|
async statusReporter => await fetchData(statusReporter));
|
||||||
@ -627,7 +639,7 @@ public class DownloadOrchestrationService(
|
|||||||
|
|
||||||
int objectCount = getObjectCount(data);
|
int objectCount = getObjectCount(data);
|
||||||
eventHandler.OnContentFound(contentType, mediaCount, objectCount);
|
eventHandler.OnContentFound(contentType, mediaCount, objectCount);
|
||||||
Log.Debug($"Found {mediaCount} Media from {objectCount} {contentType}");
|
Log.Debug("Found {MediaCount} Media from {ObjectCount} {ContentType}", mediaCount, objectCount, contentType);
|
||||||
|
|
||||||
Config config = configService.CurrentConfig;
|
Config config = configService.CurrentConfig;
|
||||||
List<string>? urls = getUrls(data);
|
List<string>? urls = getUrls(data);
|
||||||
@ -641,7 +653,9 @@ public class DownloadOrchestrationService(
|
|||||||
|
|
||||||
eventHandler.OnDownloadComplete(contentType, result);
|
eventHandler.OnDownloadComplete(contentType, result);
|
||||||
Log.Debug(
|
Log.Debug(
|
||||||
$"{contentType} Already Downloaded: {result.ExistingDownloads} New {contentType} Downloaded: {result.NewDownloads}");
|
$"{contentType} Media Already Downloaded: {result.ExistingDownloads} New {contentType} Media Downloaded: {result.NewDownloads}");
|
||||||
|
|
||||||
|
newPostAssignmentAction?.Invoke(result.NewDownloads);
|
||||||
|
|
||||||
return result.TotalCount;
|
return result.TotalCount;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,3 +1,5 @@
|
|||||||
|
using System.Diagnostics;
|
||||||
|
using System.Globalization;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
using System.Text.RegularExpressions;
|
using System.Text.RegularExpressions;
|
||||||
using FFmpeg.NET;
|
using FFmpeg.NET;
|
||||||
@ -111,14 +113,26 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<bool> DownloadDrmMedia(string userAgent, string policy, string signature, string kvp,
|
private async Task<bool> DownloadDrmMedia(
|
||||||
string sess, string url, string decryptionKey, string folder, DateTime lastModified, long mediaId,
|
string userAgent,
|
||||||
string apiType, IProgressReporter progressReporter, string customFileName, string filename, string path)
|
string policy,
|
||||||
|
string signature,
|
||||||
|
string kvp,
|
||||||
|
string sess,
|
||||||
|
string url,
|
||||||
|
string decryptionKey,
|
||||||
|
string folder,
|
||||||
|
DateTime lastModified,
|
||||||
|
long mediaId,
|
||||||
|
string apiType,
|
||||||
|
IProgressReporter progressReporter,
|
||||||
|
string customFileName,
|
||||||
|
string filename,
|
||||||
|
string path,
|
||||||
|
double? expectedDurationSeconds)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
_completionSource = new TaskCompletionSource<bool>();
|
|
||||||
|
|
||||||
int pos1 = decryptionKey.IndexOf(':');
|
int pos1 = decryptionKey.IndexOf(':');
|
||||||
string decKey = "";
|
string decKey = "";
|
||||||
if (pos1 >= 0)
|
if (pos1 >= 0)
|
||||||
@ -126,25 +140,20 @@ public class DownloadService(
|
|||||||
decKey = decryptionKey[(pos1 + 1)..];
|
decKey = decryptionKey[(pos1 + 1)..];
|
||||||
}
|
}
|
||||||
|
|
||||||
int streamIndex = 0;
|
const int streamIndex = 0;
|
||||||
string tempFilename = $"{folder}{path}/{filename}_source.mp4";
|
string tempFilename = $"{folder}{path}/{filename}_source.mp4";
|
||||||
|
string finalFilePath = !string.IsNullOrEmpty(customFileName)
|
||||||
|
? $"{folder}{path}/{customFileName}.mp4"
|
||||||
|
: tempFilename;
|
||||||
|
|
||||||
// Configure ffmpeg log level and optional report file location
|
// Configure ffmpeg log level and optional report file location.
|
||||||
bool ffmpegDebugLogging = Log.IsEnabled(LogEventLevel.Debug);
|
string ffToolLogLevel = GetFfToolLogLevel();
|
||||||
|
bool enableFfReport = string.Equals(ffToolLogLevel, "debug", StringComparison.OrdinalIgnoreCase);
|
||||||
string logLevelArgs = ffmpegDebugLogging ||
|
string logLevelArgs = enableFfReport
|
||||||
configService.CurrentConfig.LoggingLevel is LoggingLevel.Verbose or LoggingLevel.Debug
|
|
||||||
? "-loglevel debug -report"
|
? "-loglevel debug -report"
|
||||||
: configService.CurrentConfig.LoggingLevel switch
|
: $"-loglevel {ffToolLogLevel}";
|
||||||
{
|
|
||||||
LoggingLevel.Information => "-loglevel info",
|
|
||||||
LoggingLevel.Warning => "-loglevel warning",
|
|
||||||
LoggingLevel.Error => "-loglevel error",
|
|
||||||
LoggingLevel.Fatal => "-loglevel fatal",
|
|
||||||
_ => ""
|
|
||||||
};
|
|
||||||
|
|
||||||
if (logLevelArgs.Contains("-report", StringComparison.OrdinalIgnoreCase))
|
if (enableFfReport)
|
||||||
{
|
{
|
||||||
// Use a relative path so FFREPORT parsing works on Windows (drive-letter ':' breaks option parsing).
|
// Use a relative path so FFREPORT parsing works on Windows (drive-letter ':' breaks option parsing).
|
||||||
string logDir = Path.Combine(Environment.CurrentDirectory, "logs");
|
string logDir = Path.Combine(Environment.CurrentDirectory, "logs");
|
||||||
@ -167,6 +176,28 @@ public class DownloadService(
|
|||||||
$"CloudFront-Key-Pair-Id={kvp}; " +
|
$"CloudFront-Key-Pair-Id={kvp}; " +
|
||||||
$"{sess}";
|
$"{sess}";
|
||||||
|
|
||||||
|
if (expectedDurationSeconds.HasValue)
|
||||||
|
{
|
||||||
|
Log.Debug("Expected DRM video duration for media {MediaId}: {ExpectedDurationSeconds:F2}s", mediaId,
|
||||||
|
expectedDurationSeconds.Value);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Log.Warning("MPD video duration missing for media {MediaId}; skipping DRM duration validation.",
|
||||||
|
mediaId);
|
||||||
|
}
|
||||||
|
|
||||||
|
double threshold = configService.CurrentConfig.DrmVideoDurationMatchThreshold;
|
||||||
|
for (int attempt = 1; attempt <= Constants.DrmDownloadMaxRetries; attempt++)
|
||||||
|
{
|
||||||
|
TryDeleteFile(tempFilename);
|
||||||
|
if (!string.Equals(finalFilePath, tempFilename, StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
TryDeleteFile(finalFilePath);
|
||||||
|
}
|
||||||
|
|
||||||
|
_completionSource = new TaskCompletionSource<bool>();
|
||||||
|
|
||||||
string parameters =
|
string parameters =
|
||||||
$"{logLevelArgs} " +
|
$"{logLevelArgs} " +
|
||||||
$"-cenc_decryption_key {decKey} " +
|
$"-cenc_decryption_key {decKey} " +
|
||||||
@ -181,56 +212,213 @@ public class DownloadService(
|
|||||||
"-c copy " +
|
"-c copy " +
|
||||||
$"\"{tempFilename}\"";
|
$"\"{tempFilename}\"";
|
||||||
|
|
||||||
Log.Debug($"Calling FFMPEG with Parameters: {parameters}");
|
Log.Debug("Calling FFmpeg with Parameters: {Parameters}", parameters);
|
||||||
|
|
||||||
Engine ffmpeg = new(configService.CurrentConfig.FFmpegPath);
|
Engine ffmpeg = new(configService.CurrentConfig.FFmpegPath);
|
||||||
ffmpeg.Error += OnError;
|
ffmpeg.Error += OnError;
|
||||||
ffmpeg.Complete += async (_, _) =>
|
ffmpeg.Complete += (_, _) => { _completionSource.TrySetResult(true); };
|
||||||
{
|
|
||||||
_completionSource.TrySetResult(true);
|
|
||||||
await OnFFMPEGDownloadComplete(tempFilename, lastModified, folder, path, customFileName, filename,
|
|
||||||
mediaId, apiType, progressReporter);
|
|
||||||
};
|
|
||||||
await ffmpeg.ExecuteAsync(parameters, CancellationToken.None);
|
await ffmpeg.ExecuteAsync(parameters, CancellationToken.None);
|
||||||
|
|
||||||
return await _completionSource.Task;
|
bool ffmpegSuccess = await _completionSource.Task;
|
||||||
|
if (!ffmpegSuccess || !File.Exists(tempFilename))
|
||||||
|
{
|
||||||
|
Log.Warning("DRM download attempt {Attempt}/{MaxAttempts} failed for media {MediaId}.", attempt,
|
||||||
|
Constants.DrmDownloadMaxRetries, mediaId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!expectedDurationSeconds.HasValue)
|
||||||
|
{
|
||||||
|
return await FinalizeDrmDownload(tempFilename, lastModified, folder, path, customFileName, filename,
|
||||||
|
mediaId, apiType, progressReporter);
|
||||||
|
}
|
||||||
|
|
||||||
|
double? actualDurationSeconds = await TryGetVideoDurationSeconds(tempFilename);
|
||||||
|
if (!actualDurationSeconds.HasValue)
|
||||||
|
{
|
||||||
|
Log.Warning(
|
||||||
|
"DRM download attempt {Attempt}/{MaxAttempts} could not determine output duration for media {MediaId}.",
|
||||||
|
attempt, Constants.DrmDownloadMaxRetries, mediaId);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
double durationRatio = actualDurationSeconds.Value / expectedDurationSeconds.Value;
|
||||||
|
Log.Debug("Expected duration: {ExpectedSeconds:F2}s Actual duration: {ActualSeconds:F2}s",
|
||||||
|
expectedDurationSeconds.Value, actualDurationSeconds.Value);
|
||||||
|
Log.Debug("Ratio: {Ratio:P2} Threshold: {Threshold:P2}", durationRatio, threshold);
|
||||||
|
if (durationRatio >= threshold)
|
||||||
|
{
|
||||||
|
return await FinalizeDrmDownload(tempFilename, lastModified, folder, path, customFileName, filename,
|
||||||
|
mediaId, apiType, progressReporter);
|
||||||
|
}
|
||||||
|
|
||||||
|
Log.Warning(
|
||||||
|
"DRM download attempt {Attempt}/{MaxAttempts} produced a short file for media {MediaId}. Expected={ExpectedSeconds:F2}s Actual={ActualSeconds:F2}s Ratio={Ratio:P2} Threshold={Threshold:P2}.",
|
||||||
|
attempt, Constants.DrmDownloadMaxRetries, mediaId, expectedDurationSeconds.Value,
|
||||||
|
actualDurationSeconds.Value, durationRatio, threshold);
|
||||||
|
}
|
||||||
|
|
||||||
|
TryDeleteFile(tempFilename);
|
||||||
|
Log.Warning("DRM download failed validation after {MaxAttempts} attempts for media {MediaId}.",
|
||||||
|
Constants.DrmDownloadMaxRetries, mediaId);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
ExceptionLoggerHelper.LogException(ex);
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
}
|
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private async Task OnFFMPEGDownloadComplete(string tempFilename, DateTime lastModified, string folder, string path,
|
private async Task<double?> TryGetVideoDurationSeconds(string filePath)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string ffprobePath = configService.CurrentConfig.FFprobePath
|
||||||
|
?? throw new InvalidOperationException("FFprobePath is not configured.");
|
||||||
|
string ffprobeLogLevel = GetFfToolLogLevel();
|
||||||
|
ProcessStartInfo startInfo = new()
|
||||||
|
{
|
||||||
|
FileName = ffprobePath,
|
||||||
|
Arguments =
|
||||||
|
$"-v {ffprobeLogLevel} -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 \"{filePath}\"",
|
||||||
|
RedirectStandardOutput = true,
|
||||||
|
RedirectStandardError = true,
|
||||||
|
UseShellExecute = false,
|
||||||
|
CreateNoWindow = true
|
||||||
|
};
|
||||||
|
|
||||||
|
using Process process = new();
|
||||||
|
process.StartInfo = startInfo;
|
||||||
|
process.Start();
|
||||||
|
|
||||||
|
Task<string> outputTask = process.StandardOutput.ReadToEndAsync();
|
||||||
|
Task<string> errorTask = process.StandardError.ReadToEndAsync();
|
||||||
|
Task waitForExitTask = process.WaitForExitAsync();
|
||||||
|
await Task.WhenAll(outputTask, errorTask, waitForExitTask);
|
||||||
|
|
||||||
|
string output = outputTask.Result;
|
||||||
|
string error = errorTask.Result;
|
||||||
|
|
||||||
|
if (process.ExitCode != 0)
|
||||||
|
{
|
||||||
|
Log.Warning("FFprobe failed for file {FilePath}. ExitCode={ExitCode} Error={Error}", filePath,
|
||||||
|
process.ExitCode, error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
string[] outputLines = output.Split(['\r', '\n'], StringSplitOptions.RemoveEmptyEntries);
|
||||||
|
foreach (string outputLine in outputLines)
|
||||||
|
{
|
||||||
|
if (double.TryParse(outputLine.Trim(), NumberStyles.Float, CultureInfo.InvariantCulture,
|
||||||
|
out double durationSeconds))
|
||||||
|
{
|
||||||
|
return durationSeconds > 0 ? durationSeconds : null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Log.Warning("Unable to parse FFprobe duration for file {FilePath}. RawOutput={RawOutput}", filePath,
|
||||||
|
output.Trim());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
Log.Warning(ex, "Failed to inspect downloaded video duration for {FilePath}", filePath);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private string GetFfToolLogLevel()
|
||||||
|
{
|
||||||
|
bool ffToolDebugLogging = Log.IsEnabled(LogEventLevel.Debug);
|
||||||
|
if (ffToolDebugLogging ||
|
||||||
|
configService.CurrentConfig.LoggingLevel is LoggingLevel.Verbose or LoggingLevel.Debug)
|
||||||
|
{
|
||||||
|
return "debug";
|
||||||
|
}
|
||||||
|
|
||||||
|
return configService.CurrentConfig.LoggingLevel switch
|
||||||
|
{
|
||||||
|
LoggingLevel.Information => "info",
|
||||||
|
LoggingLevel.Warning => "warning",
|
||||||
|
LoggingLevel.Error => "error",
|
||||||
|
LoggingLevel.Fatal => "fatal",
|
||||||
|
_ => "error"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private async Task<bool> FinalizeDrmDownload(string tempFilename, DateTime lastModified, string folder, string path,
|
||||||
string customFileName, string filename, long mediaId, string apiType, IProgressReporter progressReporter)
|
string customFileName, string filename, long mediaId, string apiType, IProgressReporter progressReporter)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (File.Exists(tempFilename))
|
if (!File.Exists(tempFilename))
|
||||||
{
|
{
|
||||||
File.SetLastWriteTime(tempFilename, lastModified);
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
File.SetLastWriteTime(tempFilename, lastModified);
|
||||||
|
|
||||||
|
string finalPath = tempFilename;
|
||||||
|
string finalName = filename + "_source.mp4";
|
||||||
if (!string.IsNullOrEmpty(customFileName))
|
if (!string.IsNullOrEmpty(customFileName))
|
||||||
{
|
{
|
||||||
File.Move(tempFilename, $"{folder + path + "/" + customFileName + ".mp4"}");
|
finalPath = $"{folder}{path}/{customFileName}.mp4";
|
||||||
|
finalName = customFileName + ".mp4";
|
||||||
|
|
||||||
|
if (!AreSamePath(tempFilename, finalPath))
|
||||||
|
{
|
||||||
|
TryDeleteFile(finalPath);
|
||||||
|
File.Move(tempFilename, finalPath);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Cleanup Files
|
long fileSizeInBytes = new FileInfo(finalPath).Length;
|
||||||
long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName)
|
|
||||||
? folder + path + "/" + customFileName + ".mp4"
|
|
||||||
: tempFilename).Length;
|
|
||||||
ReportProgress(progressReporter, fileSizeInBytes);
|
ReportProgress(progressReporter, fileSizeInBytes);
|
||||||
|
|
||||||
await dbService.UpdateMedia(folder, mediaId, apiType, folder + path,
|
await dbService.UpdateMedia(folder, mediaId, apiType, folder + path, finalName, fileSizeInBytes, true,
|
||||||
!string.IsNullOrEmpty(customFileName) ? customFileName + ".mp4" : filename + "_source.mp4",
|
lastModified);
|
||||||
fileSizeInBytes, true, lastModified);
|
return true;
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
ExceptionLoggerHelper.LogException(ex);
|
ExceptionLoggerHelper.LogException(ex);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void TryDeleteFile(string filePath)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (File.Exists(filePath))
|
||||||
|
{
|
||||||
|
File.Delete(filePath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Best-effort cleanup only.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool AreSamePath(string firstPath, string secondPath)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
string firstFullPath = Path.GetFullPath(firstPath);
|
||||||
|
string secondFullPath = Path.GetFullPath(secondPath);
|
||||||
|
StringComparison comparison = OperatingSystem.IsWindows()
|
||||||
|
? StringComparison.OrdinalIgnoreCase
|
||||||
|
: StringComparison.Ordinal;
|
||||||
|
return string.Equals(firstFullPath, secondFullPath, comparison);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
StringComparison comparison = OperatingSystem.IsWindows()
|
||||||
|
? StringComparison.OrdinalIgnoreCase
|
||||||
|
: StringComparison.Ordinal;
|
||||||
|
return string.Equals(firstPath, secondPath, comparison);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -869,12 +1057,13 @@ public class DownloadService(
|
|||||||
/// <param name="postMedia">Media info.</param>
|
/// <param name="postMedia">Media info.</param>
|
||||||
/// <param name="author">Author info.</param>
|
/// <param name="author">Author info.</param>
|
||||||
/// <param name="users">Known users map.</param>
|
/// <param name="users">Known users map.</param>
|
||||||
|
/// <param name="expectedDurationSeconds">The expected duration of the video in seconds.</param>
|
||||||
/// <returns>True when the media is newly downloaded.</returns>
|
/// <returns>True when the media is newly downloaded.</returns>
|
||||||
private async Task<bool> DownloadDrmVideo(string policy, string signature, string kvp, string url,
|
private async Task<bool> DownloadDrmVideo(string policy, string signature, string kvp, string url,
|
||||||
string decryptionKey, string folder, DateTime lastModified, long mediaId, string apiType,
|
string decryptionKey, string folder, DateTime lastModified, long mediaId, string apiType,
|
||||||
IProgressReporter progressReporter, string path,
|
IProgressReporter progressReporter, string path,
|
||||||
string? filenameFormat, object? postInfo, object? postMedia,
|
string? filenameFormat, object? postInfo, object? postMedia,
|
||||||
object? author, Dictionary<string, long> users)
|
object? author, Dictionary<string, long> users, double? expectedDurationSeconds)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@ -912,7 +1101,7 @@ public class DownloadService(
|
|||||||
{
|
{
|
||||||
return await DownloadDrmMedia(authService.CurrentAuth.UserAgent, policy, signature, kvp,
|
return await DownloadDrmMedia(authService.CurrentAuth.UserAgent, policy, signature, kvp,
|
||||||
authService.CurrentAuth.Cookie, url, decryptionKey, folder, lastModified, mediaId, apiType,
|
authService.CurrentAuth.Cookie, url, decryptionKey, folder, lastModified, mediaId, apiType,
|
||||||
progressReporter, customFileName, filename, path);
|
progressReporter, customFileName, filename, path, expectedDurationSeconds);
|
||||||
}
|
}
|
||||||
|
|
||||||
long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName)
|
long fileSizeInBytes = new FileInfo(!string.IsNullOrEmpty(customFileName)
|
||||||
@ -985,15 +1174,14 @@ public class DownloadService(
|
|||||||
/// <param name="drmType">The DRM type.</param>
|
/// <param name="drmType">The DRM type.</param>
|
||||||
/// <param name="clientIdBlobMissing">Whether the CDM client ID blob is missing.</param>
|
/// <param name="clientIdBlobMissing">Whether the CDM client ID blob is missing.</param>
|
||||||
/// <param name="devicePrivateKeyMissing">Whether the CDM private key is missing.</param>
|
/// <param name="devicePrivateKeyMissing">Whether the CDM private key is missing.</param>
|
||||||
/// <returns>The decryption key and last modified timestamp.</returns>
|
/// <returns>The decryption key, last modified timestamp, and MPD duration seconds.</returns>
|
||||||
public async Task<(string decryptionKey, DateTime lastModified)?> GetDecryptionInfo(
|
public async Task<(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)?> GetDecryptionInfo(
|
||||||
string mpdUrl, string policy, string signature, string kvp,
|
string mpdUrl, string policy, string signature, string kvp,
|
||||||
string mediaId, string contentId, string drmType,
|
string mediaId, string contentId, string drmType,
|
||||||
bool clientIdBlobMissing, bool devicePrivateKeyMissing)
|
bool clientIdBlobMissing, bool devicePrivateKeyMissing)
|
||||||
{
|
{
|
||||||
string pssh = await apiService.GetDrmMpdPssh(mpdUrl, policy, signature, kvp);
|
(string pssh, DateTime lastModified, double? durationSeconds) =
|
||||||
|
await apiService.GetDrmMpdInfo(mpdUrl, policy, signature, kvp);
|
||||||
DateTime lastModified = await apiService.GetDrmMpdLastModified(mpdUrl, policy, signature, kvp);
|
|
||||||
Dictionary<string, string> drmHeaders =
|
Dictionary<string, string> drmHeaders =
|
||||||
apiService.GetDynamicHeaders($"/api2/v2/users/media/{mediaId}/drm/{drmType}/{contentId}",
|
apiService.GetDynamicHeaders($"/api2/v2/users/media/{mediaId}/drm/{drmType}/{contentId}",
|
||||||
"?type=widevine");
|
"?type=widevine");
|
||||||
@ -1004,7 +1192,7 @@ public class DownloadService(
|
|||||||
? await apiService.GetDecryptionKeyOfdl(drmHeaders, licenseUrl, pssh)
|
? await apiService.GetDecryptionKeyOfdl(drmHeaders, licenseUrl, pssh)
|
||||||
: await apiService.GetDecryptionKeyCdm(drmHeaders, licenseUrl, pssh);
|
: await apiService.GetDecryptionKeyCdm(drmHeaders, licenseUrl, pssh);
|
||||||
|
|
||||||
return (decryptionKey, lastModified);
|
return (decryptionKey, lastModified, durationSeconds);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@ -1058,7 +1246,7 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug(
|
Log.Debug(
|
||||||
$"Highlights Already Downloaded: {oldHighlightsCount} New Highlights Downloaded: {newHighlightsCount}");
|
$"Highlights Media Already Downloaded: {oldHighlightsCount} New Highlights Media Downloaded: {newHighlightsCount}");
|
||||||
|
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
@ -1119,7 +1307,8 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug($"Stories Already Downloaded: {oldStoriesCount} New Stories Downloaded: {newStoriesCount}");
|
Log.Debug(
|
||||||
|
$"Stories Media Already Downloaded: {oldStoriesCount} New Stories Media Downloaded: {newStoriesCount}");
|
||||||
|
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
@ -1182,7 +1371,8 @@ public class DownloadService(
|
|||||||
if (archivedKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (archivedKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = archivedKvp.Value.Split(',');
|
string[] parsed = archivedKvp.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1],
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
|
await GetDecryptionInfo(parsed[0], parsed[1],
|
||||||
parsed[2], parsed[3],
|
parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -1193,7 +1383,7 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, archivedKvp.Key, "Posts",
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, archivedKvp.Key, "Posts",
|
||||||
progressReporter, "/Archived/Posts/Free/Videos", filenameFormat,
|
progressReporter, "/Archived/Posts/Free/Videos", filenameFormat,
|
||||||
postInfo, mediaInfo, postInfo?.Author, users);
|
postInfo, mediaInfo, postInfo?.Author, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -1212,7 +1402,7 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug(
|
Log.Debug(
|
||||||
$"Archived Posts Already Downloaded: {oldArchivedCount} New Archived Posts Downloaded: {newArchivedCount}");
|
$"Archived Posts Media Already Downloaded: {oldArchivedCount} New Archived Posts Media Downloaded: {newArchivedCount}");
|
||||||
|
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
@ -1276,7 +1466,8 @@ public class DownloadService(
|
|||||||
if (messageKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (messageKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = messageKvp.Value.Split(',');
|
string[] parsed = messageKvp.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1],
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
|
await GetDecryptionInfo(parsed[0], parsed[1],
|
||||||
parsed[2], parsed[3],
|
parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -1287,7 +1478,7 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, messageKvp.Key, "Messages",
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, messageKvp.Key, "Messages",
|
||||||
progressReporter, messagePath + "/Videos", filenameFormat,
|
progressReporter, messagePath + "/Videos", filenameFormat,
|
||||||
messageInfo, mediaInfo, messageInfo?.FromUser, users);
|
messageInfo, mediaInfo, messageInfo?.FromUser, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -1305,7 +1496,8 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug($"Messages Already Downloaded: {oldMessagesCount} New Messages Downloaded: {newMessagesCount}");
|
Log.Debug(
|
||||||
|
$"Messages Media Already Downloaded: {oldMessagesCount} New Messages Media Downloaded: {newMessagesCount}");
|
||||||
|
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
@ -1363,16 +1555,18 @@ public class DownloadService(
|
|||||||
PurchasedEntities.ListItem? messageInfo = paidMessageCollection.PaidMessageObjects.FirstOrDefault(p =>
|
PurchasedEntities.ListItem? messageInfo = paidMessageCollection.PaidMessageObjects.FirstOrDefault(p =>
|
||||||
p.Media?.Any(m => m.Id == kvpEntry.Key) == true);
|
p.Media?.Any(m => m.Id == kvpEntry.Key) == true);
|
||||||
string filenameFormat =
|
string filenameFormat =
|
||||||
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).MessageFileNameFormat ?? "";
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PaidMessageFileNameFormat ?? "";
|
||||||
string paidMsgPath = configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null &&
|
string paidMsgPath = configService.CurrentConfig.FolderPerPaidMessage && messageInfo != null &&
|
||||||
messageInfo.Id != 0 && messageInfo.CreatedAt is not null
|
messageInfo.Id != 0 && messageInfo.CreatedAt is not null
|
||||||
? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}"
|
? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}"
|
||||||
: "/Messages/Paid";
|
: "/Messages/Paid";
|
||||||
|
object? messageAuthor = messageInfo?.FromUser ?? (object?)messageInfo?.Author;
|
||||||
|
|
||||||
if (kvpEntry.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (kvpEntry.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = kvpEntry.Value.Split(',');
|
string[] parsed = kvpEntry.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1],
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
|
await GetDecryptionInfo(parsed[0], parsed[1],
|
||||||
parsed[2], parsed[3],
|
parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -1383,12 +1577,12 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, kvpEntry.Key, "Messages",
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, kvpEntry.Key, "Messages",
|
||||||
progressReporter, paidMsgPath + "/Videos", filenameFormat,
|
progressReporter, paidMsgPath + "/Videos", filenameFormat,
|
||||||
messageInfo, mediaInfo, messageInfo?.FromUser, users);
|
messageInfo, mediaInfo, messageAuthor, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
isNew = await DownloadMedia(kvpEntry.Value, path, kvpEntry.Key, "Messages", progressReporter,
|
isNew = await DownloadMedia(kvpEntry.Value, path, kvpEntry.Key, "Messages", progressReporter,
|
||||||
paidMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users);
|
paidMsgPath, filenameFormat, messageInfo, mediaInfo, messageAuthor, users);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isNew)
|
if (isNew)
|
||||||
@ -1401,7 +1595,7 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug($"Paid Messages Already Downloaded: {oldCount} New Paid Messages Downloaded: {newCount}");
|
Log.Debug($"Paid Messages Media Already Downloaded: {oldCount} New Paid Messages Media Downloaded: {newCount}");
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
TotalCount = paidMessageCollection.PaidMessages.Count,
|
TotalCount = paidMessageCollection.PaidMessages.Count,
|
||||||
@ -1464,7 +1658,8 @@ public class DownloadService(
|
|||||||
if (kvpEntry.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (kvpEntry.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = kvpEntry.Value.Split(',');
|
string[] parsed = kvpEntry.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1],
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
|
await GetDecryptionInfo(parsed[0], parsed[1],
|
||||||
parsed[2], parsed[3],
|
parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -1475,7 +1670,7 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, kvpEntry.Key, "Streams",
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, kvpEntry.Key, "Streams",
|
||||||
progressReporter, streamPath + "/Videos", filenameFormat,
|
progressReporter, streamPath + "/Videos", filenameFormat,
|
||||||
streamInfo, mediaInfo, streamInfo?.Author, users);
|
streamInfo, mediaInfo, streamInfo?.Author, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -1493,7 +1688,7 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug($"Streams Already Downloaded: {oldCount} New Streams Downloaded: {newCount}");
|
Log.Debug($"Streams Media Already Downloaded: {oldCount} New Streams Media Downloaded: {newCount}");
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
TotalCount = streams.Streams.Count,
|
TotalCount = streams.Streams.Count,
|
||||||
@ -1556,7 +1751,8 @@ public class DownloadService(
|
|||||||
if (postKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (postKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = postKvp.Value.Split(',');
|
string[] parsed = postKvp.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1],
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
|
await GetDecryptionInfo(parsed[0], parsed[1],
|
||||||
parsed[2], parsed[3],
|
parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -1567,7 +1763,7 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKvp.Key, "Posts",
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKvp.Key, "Posts",
|
||||||
progressReporter, postPath + "/Videos", filenameFormat,
|
progressReporter, postPath + "/Videos", filenameFormat,
|
||||||
postInfo, mediaInfo, postInfo?.Author, users);
|
postInfo, mediaInfo, postInfo?.Author, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -1585,7 +1781,7 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug($"Posts Already Downloaded: {oldCount} New Posts Downloaded: {newCount}");
|
Log.Debug($"Posts Media Already Downloaded: {oldCount} New Posts Media Downloaded: {newCount}");
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
TotalCount = posts.Posts.Count,
|
TotalCount = posts.Posts.Count,
|
||||||
@ -1640,16 +1836,18 @@ public class DownloadService(
|
|||||||
PurchasedEntities.ListItem? postInfo =
|
PurchasedEntities.ListItem? postInfo =
|
||||||
purchasedPosts.PaidPostObjects.FirstOrDefault(p => p.Media?.Any(m => m.Id == postKvp.Key) == true);
|
purchasedPosts.PaidPostObjects.FirstOrDefault(p => p.Media?.Any(m => m.Id == postKvp.Key) == true);
|
||||||
string filenameFormat =
|
string filenameFormat =
|
||||||
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "";
|
configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PaidPostFileNameFormat ?? "";
|
||||||
string paidPostPath = configService.CurrentConfig.FolderPerPaidPost && postInfo != null &&
|
string paidPostPath = configService.CurrentConfig.FolderPerPaidPost && postInfo != null &&
|
||||||
postInfo.Id != 0 && postInfo.PostedAt is not null
|
postInfo.Id != 0 && postInfo.PostedAt is not null
|
||||||
? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}"
|
? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}"
|
||||||
: "/Posts/Paid";
|
: "/Posts/Paid";
|
||||||
|
object? postAuthor = postInfo?.FromUser ?? (object?)postInfo?.Author;
|
||||||
|
|
||||||
if (postKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (postKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = postKvp.Value.Split(',');
|
string[] parsed = postKvp.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo = await GetDecryptionInfo(parsed[0], parsed[1],
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
|
await GetDecryptionInfo(parsed[0], parsed[1],
|
||||||
parsed[2], parsed[3],
|
parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -1660,12 +1858,12 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKvp.Key, "Posts",
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKvp.Key, "Posts",
|
||||||
progressReporter, paidPostPath + "/Videos", filenameFormat,
|
progressReporter, paidPostPath + "/Videos", filenameFormat,
|
||||||
postInfo, mediaInfo, postInfo?.FromUser, users);
|
postInfo, mediaInfo, postAuthor, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
isNew = await DownloadMedia(postKvp.Value, path, postKvp.Key, "Posts", progressReporter,
|
isNew = await DownloadMedia(postKvp.Value, path, postKvp.Key, "Posts", progressReporter,
|
||||||
paidPostPath, filenameFormat, postInfo, mediaInfo, postInfo?.FromUser, users);
|
paidPostPath, filenameFormat, postInfo, mediaInfo, postAuthor, users);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isNew)
|
if (isNew)
|
||||||
@ -1678,7 +1876,7 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug($"Paid Posts Already Downloaded: {oldCount} New Paid Posts Downloaded: {newCount}");
|
Log.Debug($"Paid Posts Media Already Downloaded: {oldCount} New Paid Posts Media Downloaded: {newCount}");
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
TotalCount = purchasedPosts.PaidPosts.Count,
|
TotalCount = purchasedPosts.PaidPosts.Count,
|
||||||
@ -1729,11 +1927,12 @@ public class DownloadService(
|
|||||||
postInfo.Id != 0 && postInfo.PostedAt is not null
|
postInfo.Id != 0 && postInfo.PostedAt is not null
|
||||||
? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}"
|
? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt.Value:yyyy-MM-dd HH-mm-ss}"
|
||||||
: "/Posts/Paid";
|
: "/Posts/Paid";
|
||||||
|
object? postAuthor = postInfo?.FromUser ?? (object?)postInfo?.Author;
|
||||||
|
|
||||||
if (purchasedPostKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (purchasedPostKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = purchasedPostKvp.Value.Split(',');
|
string[] parsed = purchasedPostKvp.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo =
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -1744,13 +1943,13 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, purchasedPostKvp.Key,
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, purchasedPostKvp.Key,
|
||||||
"Posts", progressReporter, paidPostPath + "/Videos", filenameFormat,
|
"Posts", progressReporter, paidPostPath + "/Videos", filenameFormat,
|
||||||
postInfo, mediaInfo, postInfo?.FromUser, users);
|
postInfo, mediaInfo, postAuthor, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
isNew = await DownloadMedia(purchasedPostKvp.Value, path,
|
isNew = await DownloadMedia(purchasedPostKvp.Value, path,
|
||||||
purchasedPostKvp.Key, "Posts", progressReporter,
|
purchasedPostKvp.Key, "Posts", progressReporter,
|
||||||
paidPostPath, filenameFormat, postInfo, mediaInfo, postInfo?.FromUser, users);
|
paidPostPath, filenameFormat, postInfo, mediaInfo, postAuthor, users);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isNew)
|
if (isNew)
|
||||||
@ -1763,7 +1962,7 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug($"Paid Posts Already Downloaded: {oldCount} New Paid Posts Downloaded: {newCount}");
|
Log.Debug($"Paid Posts Media Already Downloaded: {oldCount} New Paid Posts Media Downloaded: {newCount}");
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
TotalCount = purchasedPosts?.PaidPosts.Count ?? 0,
|
TotalCount = purchasedPosts?.PaidPosts.Count ?? 0,
|
||||||
@ -1813,11 +2012,12 @@ public class DownloadService(
|
|||||||
messageInfo.Id != 0 && messageInfo.CreatedAt is not null
|
messageInfo.Id != 0 && messageInfo.CreatedAt is not null
|
||||||
? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}"
|
? $"/Messages/Paid/{messageInfo.Id} {messageInfo.CreatedAt.Value:yyyy-MM-dd HH-mm-ss}"
|
||||||
: "/Messages/Paid";
|
: "/Messages/Paid";
|
||||||
|
object? messageAuthor = messageInfo?.FromUser ?? (object?)messageInfo?.Author;
|
||||||
|
|
||||||
if (paidMessageKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (paidMessageKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = paidMessageKvp.Value.Split(',');
|
string[] parsed = paidMessageKvp.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo =
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -1828,13 +2028,13 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKvp.Key,
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKvp.Key,
|
||||||
"Messages", progressReporter, paidMsgPath + "/Videos", filenameFormat,
|
"Messages", progressReporter, paidMsgPath + "/Videos", filenameFormat,
|
||||||
messageInfo, mediaInfo, messageInfo?.FromUser, users);
|
messageInfo, mediaInfo, messageAuthor, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
isNew = await DownloadMedia(paidMessageKvp.Value, path,
|
isNew = await DownloadMedia(paidMessageKvp.Value, path,
|
||||||
paidMessageKvp.Key, "Messages", progressReporter,
|
paidMessageKvp.Key, "Messages", progressReporter,
|
||||||
paidMsgPath, filenameFormat, messageInfo, mediaInfo, messageInfo?.FromUser, users);
|
paidMsgPath, filenameFormat, messageInfo, mediaInfo, messageAuthor, users);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isNew)
|
if (isNew)
|
||||||
@ -1847,7 +2047,7 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug($"Paid Messages Already Downloaded: {oldCount} New Paid Messages Downloaded: {newCount}");
|
Log.Debug($"Paid Messages Media Already Downloaded: {oldCount} New Paid Messages Media Downloaded: {newCount}");
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
TotalCount = paidMessageCollection.PaidMessages.Count,
|
TotalCount = paidMessageCollection.PaidMessages.Count,
|
||||||
@ -1882,6 +2082,7 @@ public class DownloadService(
|
|||||||
}
|
}
|
||||||
|
|
||||||
int oldCount = 0, newCount = 0;
|
int oldCount = 0, newCount = 0;
|
||||||
|
bool hasPaidPostMedia = false;
|
||||||
|
|
||||||
foreach (KeyValuePair<long, string> postKvp in post.SinglePosts)
|
foreach (KeyValuePair<long, string> postKvp in post.SinglePosts)
|
||||||
{
|
{
|
||||||
@ -1889,9 +2090,21 @@ public class DownloadService(
|
|||||||
PostEntities.SinglePost? postInfo = mediaInfo == null
|
PostEntities.SinglePost? postInfo = mediaInfo == null
|
||||||
? null
|
? null
|
||||||
: post.SinglePostObjects.FirstOrDefault(p => p.Media?.Contains(mediaInfo) == true);
|
: post.SinglePostObjects.FirstOrDefault(p => p.Media?.Contains(mediaInfo) == true);
|
||||||
string filenameFormat = configService.CurrentConfig.GetCreatorFileNameFormatConfig(username)
|
|
||||||
.PostFileNameFormat ?? "";
|
bool isPaidPost = IsPaidSinglePost(postInfo);
|
||||||
string postPath = configService.CurrentConfig.FolderPerPost && postInfo != null && postInfo.Id != 0
|
if (isPaidPost)
|
||||||
|
{
|
||||||
|
hasPaidPostMedia = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
string filenameFormat = hasPaidPostMedia
|
||||||
|
? configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PaidPostFileNameFormat ?? ""
|
||||||
|
: configService.CurrentConfig.GetCreatorFileNameFormatConfig(username).PostFileNameFormat ?? "";
|
||||||
|
string postPath = hasPaidPostMedia
|
||||||
|
? configService.CurrentConfig.FolderPerPaidPost && postInfo != null && postInfo.Id != 0
|
||||||
|
? $"/Posts/Paid/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}"
|
||||||
|
: "/Posts/Paid"
|
||||||
|
: configService.CurrentConfig.FolderPerPost && postInfo != null && postInfo.Id != 0
|
||||||
? $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}"
|
? $"/Posts/Free/{postInfo.Id} {postInfo.PostedAt:yyyy-MM-dd HH-mm-ss}"
|
||||||
: "/Posts/Free";
|
: "/Posts/Free";
|
||||||
|
|
||||||
@ -1899,7 +2112,7 @@ public class DownloadService(
|
|||||||
if (postKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (postKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = postKvp.Value.Split(',');
|
string[] parsed = postKvp.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo =
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "post", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -1910,7 +2123,7 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKvp.Key, "Posts",
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, postKvp.Key, "Posts",
|
||||||
progressReporter, postPath + "/Videos", filenameFormat,
|
progressReporter, postPath + "/Videos", filenameFormat,
|
||||||
postInfo, mediaInfo, postInfo?.Author, users);
|
postInfo, mediaInfo, postInfo?.Author, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -1942,11 +2155,34 @@ public class DownloadService(
|
|||||||
TotalCount = post.SinglePosts.Count,
|
TotalCount = post.SinglePosts.Count,
|
||||||
NewDownloads = newCount,
|
NewDownloads = newCount,
|
||||||
ExistingDownloads = oldCount,
|
ExistingDownloads = oldCount,
|
||||||
MediaType = "Posts",
|
MediaType = hasPaidPostMedia ? "Paid Posts" : "Posts",
|
||||||
Success = true
|
Success = true
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static bool IsPaidSinglePost(PostEntities.SinglePost? postInfo)
|
||||||
|
{
|
||||||
|
if (postInfo == null || !postInfo.IsOpened)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (string.IsNullOrWhiteSpace(postInfo.Price))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
string normalizedPrice = postInfo.Price.Trim();
|
||||||
|
if (decimal.TryParse(normalizedPrice, NumberStyles.Any, CultureInfo.InvariantCulture, out decimal amount))
|
||||||
|
{
|
||||||
|
return amount > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return !string.Equals(normalizedPrice, "0", StringComparison.OrdinalIgnoreCase) &&
|
||||||
|
!string.Equals(normalizedPrice, "0.0", StringComparison.OrdinalIgnoreCase) &&
|
||||||
|
!string.Equals(normalizedPrice, "0.00", StringComparison.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Downloads a single paid message collection (including previews).
|
/// Downloads a single paid message collection (including previews).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
@ -1989,7 +2225,7 @@ public class DownloadService(
|
|||||||
if (paidMessageKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (paidMessageKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = paidMessageKvp.Value.Split(',');
|
string[] parsed = paidMessageKvp.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo =
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -2000,7 +2236,7 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKvp.Key,
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKvp.Key,
|
||||||
"Messages", progressReporter, previewMsgPath + "/Videos", filenameFormat,
|
"Messages", progressReporter, previewMsgPath + "/Videos", filenameFormat,
|
||||||
messageInfo, mediaInfo, messageInfo?.FromUser, users);
|
messageInfo, mediaInfo, messageInfo?.FromUser, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -2043,7 +2279,7 @@ public class DownloadService(
|
|||||||
if (paidMessageKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
if (paidMessageKvp.Value.Contains("cdn3.onlyfans.com/dash/files"))
|
||||||
{
|
{
|
||||||
string[] parsed = paidMessageKvp.Value.Split(',');
|
string[] parsed = paidMessageKvp.Value.Split(',');
|
||||||
(string decryptionKey, DateTime lastModified)? drmInfo =
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? drmInfo =
|
||||||
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
await GetDecryptionInfo(parsed[0], parsed[1], parsed[2], parsed[3],
|
||||||
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
parsed[4], parsed[5], "message", clientIdBlobMissing, devicePrivateKeyMissing);
|
||||||
if (drmInfo == null)
|
if (drmInfo == null)
|
||||||
@ -2054,7 +2290,7 @@ public class DownloadService(
|
|||||||
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
isNew = await DownloadDrmVideo(parsed[1], parsed[2], parsed[3], parsed[0],
|
||||||
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKvp.Key,
|
drmInfo.Value.decryptionKey, path, drmInfo.Value.lastModified, paidMessageKvp.Key,
|
||||||
"Messages", progressReporter, singlePaidMsgPath + "/Videos", filenameFormat,
|
"Messages", progressReporter, singlePaidMsgPath + "/Videos", filenameFormat,
|
||||||
messageInfo, mediaInfo, messageInfo?.FromUser, users);
|
messageInfo, mediaInfo, messageInfo?.FromUser, users, drmInfo.Value.mpdDurationSeconds);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -2076,7 +2312,7 @@ public class DownloadService(
|
|||||||
|
|
||||||
int totalCount = singlePaidMessageCollection.PreviewSingleMessages.Count +
|
int totalCount = singlePaidMessageCollection.PreviewSingleMessages.Count +
|
||||||
singlePaidMessageCollection.SingleMessages.Count;
|
singlePaidMessageCollection.SingleMessages.Count;
|
||||||
Log.Debug($"Paid Messages Already Downloaded: {totalOld} New Paid Messages Downloaded: {totalNew}");
|
Log.Debug($"Paid Messages Media Already Downloaded: {totalOld} New Paid Messages Media Downloaded: {totalNew}");
|
||||||
return new DownloadResult
|
return new DownloadResult
|
||||||
{
|
{
|
||||||
TotalCount = totalCount,
|
TotalCount = totalCount,
|
||||||
|
|||||||
@ -200,8 +200,18 @@ public class FileNameService(IAuthService authService) : IFileNameService
|
|||||||
object? value = source;
|
object? value = source;
|
||||||
foreach (string propertyName in propertyPath.Split('.'))
|
foreach (string propertyName in propertyPath.Split('.'))
|
||||||
{
|
{
|
||||||
PropertyInfo property = value?.GetType().GetProperty(propertyName) ??
|
if (value == null)
|
||||||
throw new ArgumentException($"Property '{propertyName}' not found.");
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
PropertyInfo? property = value.GetType().GetProperty(propertyName,
|
||||||
|
BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase);
|
||||||
|
if (property == null)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
value = property.GetValue(value);
|
value = property.GetValue(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -17,14 +17,10 @@ public interface IApiService
|
|||||||
Task<string> GetDecryptionKeyCdm(Dictionary<string, string> drmHeaders, string licenceUrl, string pssh);
|
Task<string> GetDecryptionKeyCdm(Dictionary<string, string> drmHeaders, string licenceUrl, string pssh);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Retrieves the last modified timestamp for a DRM MPD manifest.
|
/// Retrieves DRM MPD metadata from a single request.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Task<DateTime> GetDrmMpdLastModified(string mpdUrl, string policy, string signature, string kvp);
|
Task<(string pssh, DateTime lastModified, double? durationSeconds)> GetDrmMpdInfo(
|
||||||
|
string mpdUrl, string policy, string signature, string kvp);
|
||||||
/// <summary>
|
|
||||||
/// Retrieves the Widevine PSSH from an MPD manifest.
|
|
||||||
/// </summary>
|
|
||||||
Task<string> GetDrmMpdPssh(string mpdUrl, string policy, string signature, string kvp);
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Retrieves the user's lists.
|
/// Retrieves the user's lists.
|
||||||
|
|||||||
@ -36,7 +36,7 @@ public interface IAuthService
|
|||||||
Task<UserEntities.User?> ValidateAuthAsync();
|
Task<UserEntities.User?> ValidateAuthAsync();
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Logs out by deleting chrome-data and auth.json.
|
/// Logs out by deleting chromium-data and auth.json.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
void Logout();
|
void Logout();
|
||||||
}
|
}
|
||||||
|
|||||||
@ -23,7 +23,7 @@ public interface IDownloadService
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Retrieves decryption information for a DRM media item.
|
/// Retrieves decryption information for a DRM media item.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
Task<(string decryptionKey, DateTime lastModified)?> GetDecryptionInfo(
|
Task<(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)?> GetDecryptionInfo(
|
||||||
string mpdUrl, string policy, string signature, string kvp,
|
string mpdUrl, string policy, string signature, string kvp,
|
||||||
string mediaId, string contentId, string drmType,
|
string mediaId, string contentId, string drmType,
|
||||||
bool clientIdBlobMissing, bool devicePrivateKeyMissing);
|
bool clientIdBlobMissing, bool devicePrivateKeyMissing);
|
||||||
|
|||||||
@ -34,8 +34,10 @@ public class StartupService(IConfigService configService, IAuthService authServi
|
|||||||
|
|
||||||
// FFmpeg detection
|
// FFmpeg detection
|
||||||
DetectFfmpeg(result);
|
DetectFfmpeg(result);
|
||||||
|
// FFprobe detection
|
||||||
|
DetectFfprobe(result);
|
||||||
|
|
||||||
if (result.FfmpegFound && result.FfmpegPath != null)
|
if (result is { FfmpegFound: true, FfmpegPath: not null })
|
||||||
{
|
{
|
||||||
// Escape backslashes for Windows
|
// Escape backslashes for Windows
|
||||||
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
|
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
|
||||||
@ -47,7 +49,22 @@ public class StartupService(IConfigService configService, IAuthService authServi
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Get FFmpeg version
|
// Get FFmpeg version
|
||||||
result.FfmpegVersion = await GetFfmpegVersionAsync(result.FfmpegPath);
|
result.FfmpegVersion = await GetToolVersionAsync(result.FfmpegPath, "ffmpeg");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result is { FfprobeFound: true, FfprobePath: not null })
|
||||||
|
{
|
||||||
|
// Escape backslashes for Windows
|
||||||
|
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows) &&
|
||||||
|
result.FfprobePath.Contains(@":\") &&
|
||||||
|
!result.FfprobePath.Contains(@":\\"))
|
||||||
|
{
|
||||||
|
result.FfprobePath = result.FfprobePath.Replace(@"\", @"\\");
|
||||||
|
configService.CurrentConfig.FFprobePath = result.FfprobePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get FFprobe version
|
||||||
|
result.FfprobeVersion = await GetToolVersionAsync(result.FfprobePath, "ffprobe");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Widevine device checks
|
// Widevine device checks
|
||||||
@ -146,8 +163,8 @@ public class StartupService(IConfigService configService, IAuthService authServi
|
|||||||
{
|
{
|
||||||
result.FfmpegFound = true;
|
result.FfmpegFound = true;
|
||||||
result.FfmpegPath = configService.CurrentConfig.FFmpegPath;
|
result.FfmpegPath = configService.CurrentConfig.FFmpegPath;
|
||||||
Log.Debug($"FFMPEG found: {result.FfmpegPath}");
|
Log.Debug($"FFmpeg found: {result.FfmpegPath}");
|
||||||
Log.Debug("FFMPEG path set in config.conf");
|
Log.Debug("FFmpeg path set in config.conf");
|
||||||
}
|
}
|
||||||
else if (!string.IsNullOrEmpty(authService.CurrentAuth?.FfmpegPath) &&
|
else if (!string.IsNullOrEmpty(authService.CurrentAuth?.FfmpegPath) &&
|
||||||
ValidateFilePath(authService.CurrentAuth.FfmpegPath))
|
ValidateFilePath(authService.CurrentAuth.FfmpegPath))
|
||||||
@ -155,8 +172,8 @@ public class StartupService(IConfigService configService, IAuthService authServi
|
|||||||
result.FfmpegFound = true;
|
result.FfmpegFound = true;
|
||||||
result.FfmpegPath = authService.CurrentAuth.FfmpegPath;
|
result.FfmpegPath = authService.CurrentAuth.FfmpegPath;
|
||||||
configService.CurrentConfig.FFmpegPath = result.FfmpegPath;
|
configService.CurrentConfig.FFmpegPath = result.FfmpegPath;
|
||||||
Log.Debug($"FFMPEG found: {result.FfmpegPath}");
|
Log.Debug($"FFmpeg found: {result.FfmpegPath}");
|
||||||
Log.Debug("FFMPEG path set in auth.json");
|
Log.Debug("FFmpeg path set in auth.json");
|
||||||
}
|
}
|
||||||
else if (string.IsNullOrEmpty(configService.CurrentConfig.FFmpegPath))
|
else if (string.IsNullOrEmpty(configService.CurrentConfig.FFmpegPath))
|
||||||
{
|
{
|
||||||
@ -167,8 +184,8 @@ public class StartupService(IConfigService configService, IAuthService authServi
|
|||||||
result.FfmpegPathAutoDetected = true;
|
result.FfmpegPathAutoDetected = true;
|
||||||
result.FfmpegPath = ffmpegPath;
|
result.FfmpegPath = ffmpegPath;
|
||||||
configService.CurrentConfig.FFmpegPath = ffmpegPath;
|
configService.CurrentConfig.FFmpegPath = ffmpegPath;
|
||||||
Log.Debug($"FFMPEG found: {ffmpegPath}");
|
Log.Debug($"FFmpeg found: {ffmpegPath}");
|
||||||
Log.Debug("FFMPEG path found via PATH or current directory");
|
Log.Debug("FFmpeg path found via PATH or current directory");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -178,13 +195,65 @@ public class StartupService(IConfigService configService, IAuthService authServi
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task<string?> GetFfmpegVersionAsync(string ffmpegPath)
|
private void DetectFfprobe(StartupResult result)
|
||||||
|
{
|
||||||
|
if (!string.IsNullOrEmpty(configService.CurrentConfig.FFprobePath) &&
|
||||||
|
ValidateFilePath(configService.CurrentConfig.FFprobePath))
|
||||||
|
{
|
||||||
|
result.FfprobeFound = true;
|
||||||
|
result.FfprobePath = configService.CurrentConfig.FFprobePath;
|
||||||
|
Log.Debug($"FFprobe found: {result.FfprobePath}");
|
||||||
|
Log.Debug("FFprobe path set in config.conf");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!result.FfprobeFound && !string.IsNullOrEmpty(result.FfmpegPath))
|
||||||
|
{
|
||||||
|
string? ffmpegDirectory = Path.GetDirectoryName(result.FfmpegPath);
|
||||||
|
if (!string.IsNullOrEmpty(ffmpegDirectory))
|
||||||
|
{
|
||||||
|
string ffprobeFileName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
|
||||||
|
? "ffprobe.exe"
|
||||||
|
: "ffprobe";
|
||||||
|
string inferredFfprobePath = Path.Combine(ffmpegDirectory, ffprobeFileName);
|
||||||
|
if (ValidateFilePath(inferredFfprobePath))
|
||||||
|
{
|
||||||
|
result.FfprobeFound = true;
|
||||||
|
result.FfprobePathAutoDetected = true;
|
||||||
|
result.FfprobePath = inferredFfprobePath;
|
||||||
|
configService.CurrentConfig.FFprobePath = inferredFfprobePath;
|
||||||
|
Log.Debug($"FFprobe found: {inferredFfprobePath}");
|
||||||
|
Log.Debug("FFprobe path inferred from FFmpeg path");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!result.FfprobeFound && string.IsNullOrEmpty(configService.CurrentConfig.FFprobePath))
|
||||||
|
{
|
||||||
|
string? ffprobePath = GetFullPath("ffprobe") ?? GetFullPath("ffprobe.exe");
|
||||||
|
if (ffprobePath != null)
|
||||||
|
{
|
||||||
|
result.FfprobeFound = true;
|
||||||
|
result.FfprobePathAutoDetected = true;
|
||||||
|
result.FfprobePath = ffprobePath;
|
||||||
|
configService.CurrentConfig.FFprobePath = ffprobePath;
|
||||||
|
Log.Debug($"FFprobe found: {ffprobePath}");
|
||||||
|
Log.Debug("FFprobe path found via PATH or current directory");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!result.FfprobeFound)
|
||||||
|
{
|
||||||
|
Log.Error($"Cannot locate FFprobe with path: {configService.CurrentConfig.FFprobePath}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task<string?> GetToolVersionAsync(string toolPath, string toolName)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
ProcessStartInfo processStartInfo = new()
|
ProcessStartInfo processStartInfo = new()
|
||||||
{
|
{
|
||||||
FileName = ffmpegPath,
|
FileName = toolPath,
|
||||||
Arguments = "-version",
|
Arguments = "-version",
|
||||||
RedirectStandardOutput = true,
|
RedirectStandardOutput = true,
|
||||||
RedirectStandardError = true,
|
RedirectStandardError = true,
|
||||||
@ -198,12 +267,13 @@ public class StartupService(IConfigService configService, IAuthService authServi
|
|||||||
string output = await process.StandardOutput.ReadToEndAsync();
|
string output = await process.StandardOutput.ReadToEndAsync();
|
||||||
await process.WaitForExitAsync();
|
await process.WaitForExitAsync();
|
||||||
|
|
||||||
Log.Information("FFmpeg version output:\n{Output}", output);
|
Log.Information("{ToolName} version output:\n{Output}", toolName, output);
|
||||||
|
|
||||||
string firstLine = output.Split('\n')[0].Trim();
|
string firstLine = output.Split('\n')[0].Trim();
|
||||||
if (firstLine.StartsWith("ffmpeg version"))
|
string expectedPrefix = $"{toolName} version ";
|
||||||
|
if (firstLine.StartsWith(expectedPrefix, StringComparison.OrdinalIgnoreCase))
|
||||||
{
|
{
|
||||||
int versionStart = "ffmpeg version ".Length;
|
int versionStart = expectedPrefix.Length;
|
||||||
int copyrightIndex = firstLine.IndexOf(" Copyright", StringComparison.Ordinal);
|
int copyrightIndex = firstLine.IndexOf(" Copyright", StringComparison.Ordinal);
|
||||||
return copyrightIndex > versionStart
|
return copyrightIndex > versionStart
|
||||||
? firstLine.Substring(versionStart, copyrightIndex - versionStart)
|
? firstLine.Substring(versionStart, copyrightIndex - versionStart)
|
||||||
@ -213,7 +283,7 @@ public class StartupService(IConfigService configService, IAuthService authServi
|
|||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
Log.Warning(ex, "Failed to get FFmpeg version");
|
Log.Warning(ex, "Failed to get {ToolName} version", toolName);
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
|||||||
122
OF DL.Tests/Models/Config/ConfigTests.cs
Normal file
122
OF DL.Tests/Models/Config/ConfigTests.cs
Normal file
@ -0,0 +1,122 @@
|
|||||||
|
using OF_DL.Models.Config;
|
||||||
|
|
||||||
|
namespace OF_DL.Tests.Models.Config;
|
||||||
|
|
||||||
|
public class ConfigTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void GetCreatorFileNameFormatConfig_UsesCreatorFormatWhenDefined()
|
||||||
|
{
|
||||||
|
OF_DL.Models.Config.Config config = new()
|
||||||
|
{
|
||||||
|
PaidPostFileNameFormat = "global-paid-post",
|
||||||
|
PostFileNameFormat = "global-post",
|
||||||
|
PaidMessageFileNameFormat = "global-paid-message",
|
||||||
|
MessageFileNameFormat = "global-message",
|
||||||
|
CreatorConfigs = new Dictionary<string, CreatorConfig>
|
||||||
|
{
|
||||||
|
["creator"] = new()
|
||||||
|
{
|
||||||
|
PaidPostFileNameFormat = "creator-paid-post",
|
||||||
|
PostFileNameFormat = "creator-post",
|
||||||
|
PaidMessageFileNameFormat = "creator-paid-message",
|
||||||
|
MessageFileNameFormat = "creator-message"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
IFileNameFormatConfig result = config.GetCreatorFileNameFormatConfig("creator");
|
||||||
|
|
||||||
|
Assert.Equal("creator-paid-post", result.PaidPostFileNameFormat);
|
||||||
|
Assert.Equal("creator-post", result.PostFileNameFormat);
|
||||||
|
Assert.Equal("creator-paid-message", result.PaidMessageFileNameFormat);
|
||||||
|
Assert.Equal("creator-message", result.MessageFileNameFormat);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void GetCreatorFileNameFormatConfig_FallsBackToGlobalWhenCreatorFormatIsNullOrEmpty()
|
||||||
|
{
|
||||||
|
OF_DL.Models.Config.Config config = new()
|
||||||
|
{
|
||||||
|
PaidPostFileNameFormat = "global-paid-post",
|
||||||
|
PostFileNameFormat = "global-post",
|
||||||
|
PaidMessageFileNameFormat = "global-paid-message",
|
||||||
|
MessageFileNameFormat = "global-message",
|
||||||
|
CreatorConfigs = new Dictionary<string, CreatorConfig>
|
||||||
|
{
|
||||||
|
["creator"] = new()
|
||||||
|
{
|
||||||
|
PaidPostFileNameFormat = null,
|
||||||
|
PostFileNameFormat = "",
|
||||||
|
PaidMessageFileNameFormat = null,
|
||||||
|
MessageFileNameFormat = ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
IFileNameFormatConfig result = config.GetCreatorFileNameFormatConfig("creator");
|
||||||
|
|
||||||
|
Assert.Equal("global-paid-post", result.PaidPostFileNameFormat);
|
||||||
|
Assert.Equal("global-post", result.PostFileNameFormat);
|
||||||
|
Assert.Equal("global-paid-message", result.PaidMessageFileNameFormat);
|
||||||
|
Assert.Equal("global-message", result.MessageFileNameFormat);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void GetCreatorFileNameFormatConfig_UsesGlobalWhenCreatorConfigDoesNotExist()
|
||||||
|
{
|
||||||
|
OF_DL.Models.Config.Config config = new()
|
||||||
|
{
|
||||||
|
PaidPostFileNameFormat = "global-paid-post",
|
||||||
|
PostFileNameFormat = "global-post",
|
||||||
|
PaidMessageFileNameFormat = "global-paid-message",
|
||||||
|
MessageFileNameFormat = "global-message",
|
||||||
|
CreatorConfigs = new Dictionary<string, CreatorConfig>
|
||||||
|
{
|
||||||
|
["other-creator"] = new()
|
||||||
|
{
|
||||||
|
PaidPostFileNameFormat = "other-paid-post",
|
||||||
|
PostFileNameFormat = "other-post",
|
||||||
|
PaidMessageFileNameFormat = "other-paid-message",
|
||||||
|
MessageFileNameFormat = "other-message"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
IFileNameFormatConfig result = config.GetCreatorFileNameFormatConfig("creator");
|
||||||
|
|
||||||
|
Assert.Equal("global-paid-post", result.PaidPostFileNameFormat);
|
||||||
|
Assert.Equal("global-post", result.PostFileNameFormat);
|
||||||
|
Assert.Equal("global-paid-message", result.PaidMessageFileNameFormat);
|
||||||
|
Assert.Equal("global-message", result.MessageFileNameFormat);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void GetCreatorFileNameFormatConfig_ReturnsEmptyFormatsWhenCreatorAndGlobalAreUndefined()
|
||||||
|
{
|
||||||
|
OF_DL.Models.Config.Config config = new()
|
||||||
|
{
|
||||||
|
PaidPostFileNameFormat = "",
|
||||||
|
PostFileNameFormat = "",
|
||||||
|
PaidMessageFileNameFormat = "",
|
||||||
|
MessageFileNameFormat = "",
|
||||||
|
CreatorConfigs = new Dictionary<string, CreatorConfig>
|
||||||
|
{
|
||||||
|
["creator"] = new()
|
||||||
|
{
|
||||||
|
PaidPostFileNameFormat = "",
|
||||||
|
PostFileNameFormat = null,
|
||||||
|
PaidMessageFileNameFormat = "",
|
||||||
|
MessageFileNameFormat = null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
IFileNameFormatConfig result = config.GetCreatorFileNameFormatConfig("creator");
|
||||||
|
|
||||||
|
Assert.True(string.IsNullOrEmpty(result.PaidPostFileNameFormat));
|
||||||
|
Assert.True(string.IsNullOrEmpty(result.PostFileNameFormat));
|
||||||
|
Assert.True(string.IsNullOrEmpty(result.PaidMessageFileNameFormat));
|
||||||
|
Assert.True(string.IsNullOrEmpty(result.MessageFileNameFormat));
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -281,7 +281,7 @@ public class ApiServiceTests
|
|||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task GetDrmMpdPssh_ReturnsSecondPssh()
|
public async Task GetDrmMpdInfo_ReturnsSecondPssh()
|
||||||
{
|
{
|
||||||
string mpd = """
|
string mpd = """
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
@ -304,14 +304,49 @@ public class ApiServiceTests
|
|||||||
};
|
};
|
||||||
ApiService service = CreateService(authService);
|
ApiService service = CreateService(authService);
|
||||||
|
|
||||||
string pssh = await service.GetDrmMpdPssh(server.Url.ToString(), "policy", "signature", "kvp");
|
(string pssh, _, _) = await service.GetDrmMpdInfo(server.Url.ToString(), "policy", "signature", "kvp");
|
||||||
await server.Completion;
|
await server.Completion;
|
||||||
|
|
||||||
Assert.Equal("SECOND", pssh);
|
Assert.Equal("SECOND", pssh);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task GetDrmMpdLastModified_ReturnsLastModifiedHeader()
|
public async Task GetDrmMpdInfo_ReturnsPsshLastModifiedAndDuration()
|
||||||
|
{
|
||||||
|
string mpd = """
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<MPD xmlns:cenc="urn:mpeg:cenc:2013" mediaPresentationDuration="PT1M2.5S">
|
||||||
|
<Period>
|
||||||
|
<ContentProtection>
|
||||||
|
<cenc:pssh>FIRST</cenc:pssh>
|
||||||
|
<cenc:pssh>SECOND</cenc:pssh>
|
||||||
|
</ContentProtection>
|
||||||
|
</Period>
|
||||||
|
</MPD>
|
||||||
|
""";
|
||||||
|
DateTime lastModifiedUtc = new(2024, 2, 3, 4, 5, 6, DateTimeKind.Utc);
|
||||||
|
using SimpleHttpServer server = new(mpd, lastModifiedUtc);
|
||||||
|
FakeAuthService authService = new()
|
||||||
|
{
|
||||||
|
CurrentAuth = new Auth
|
||||||
|
{
|
||||||
|
UserId = "123", UserAgent = "unit-test-agent", XBc = "xbc-token", Cookie = "auth_id=1; sess=2;"
|
||||||
|
}
|
||||||
|
};
|
||||||
|
ApiService service = CreateService(authService);
|
||||||
|
|
||||||
|
(string pssh, DateTime lastModified, double? durationSeconds) =
|
||||||
|
await service.GetDrmMpdInfo(server.Url.ToString(), "policy", "signature", "kvp");
|
||||||
|
await server.Completion;
|
||||||
|
|
||||||
|
Assert.Equal("SECOND", pssh);
|
||||||
|
Assert.True(durationSeconds.HasValue);
|
||||||
|
Assert.Equal(62.5, durationSeconds.Value, 3);
|
||||||
|
Assert.True((lastModified - lastModifiedUtc.ToLocalTime()).Duration() < TimeSpan.FromSeconds(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task GetDrmMpdInfo_ReturnsLastModifiedHeader()
|
||||||
{
|
{
|
||||||
DateTime lastModifiedUtc = new(2024, 2, 3, 4, 5, 6, DateTimeKind.Utc);
|
DateTime lastModifiedUtc = new(2024, 2, 3, 4, 5, 6, DateTimeKind.Utc);
|
||||||
using SimpleHttpServer server = new("<MPD />", lastModifiedUtc);
|
using SimpleHttpServer server = new("<MPD />", lastModifiedUtc);
|
||||||
@ -324,12 +359,12 @@ public class ApiServiceTests
|
|||||||
};
|
};
|
||||||
ApiService service = CreateService(authService);
|
ApiService service = CreateService(authService);
|
||||||
|
|
||||||
DateTime result =
|
(_, DateTime lastModified, _) =
|
||||||
await service.GetDrmMpdLastModified(server.Url.ToString(), "policy", "signature", "kvp");
|
await service.GetDrmMpdInfo(server.Url.ToString(), "policy", "signature", "kvp");
|
||||||
await server.Completion;
|
await server.Completion;
|
||||||
|
|
||||||
DateTime expectedLocal = lastModifiedUtc.ToLocalTime();
|
DateTime expectedLocal = lastModifiedUtc.ToLocalTime();
|
||||||
Assert.True((result - expectedLocal).Duration() < TimeSpan.FromSeconds(1));
|
Assert.True((lastModified - expectedLocal).Duration() < TimeSpan.FromSeconds(1));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|||||||
@ -29,10 +29,7 @@ public class AuthServiceTests
|
|||||||
AuthService service = CreateService();
|
AuthService service = CreateService();
|
||||||
service.CurrentAuth = new Auth
|
service.CurrentAuth = new Auth
|
||||||
{
|
{
|
||||||
UserId = "123",
|
UserId = "123", UserAgent = "agent", XBc = "xbc", Cookie = "auth_id=123; sess=abc;"
|
||||||
UserAgent = "agent",
|
|
||||||
XBc = "xbc",
|
|
||||||
Cookie = "auth_id=123; sess=abc;"
|
|
||||||
};
|
};
|
||||||
|
|
||||||
await service.SaveToFileAsync();
|
await service.SaveToFileAsync();
|
||||||
@ -53,10 +50,7 @@ public class AuthServiceTests
|
|||||||
using TempFolder temp = new();
|
using TempFolder temp = new();
|
||||||
using CurrentDirectoryScope _ = new(temp.Path);
|
using CurrentDirectoryScope _ = new(temp.Path);
|
||||||
AuthService service = CreateService();
|
AuthService service = CreateService();
|
||||||
service.CurrentAuth = new Auth
|
service.CurrentAuth = new Auth { Cookie = "auth_id=123; other=1; sess=abc" };
|
||||||
{
|
|
||||||
Cookie = "auth_id=123; other=1; sess=abc"
|
|
||||||
};
|
|
||||||
|
|
||||||
service.ValidateCookieString();
|
service.ValidateCookieString();
|
||||||
|
|
||||||
@ -74,13 +68,13 @@ public class AuthServiceTests
|
|||||||
using TempFolder temp = new();
|
using TempFolder temp = new();
|
||||||
using CurrentDirectoryScope _ = new(temp.Path);
|
using CurrentDirectoryScope _ = new(temp.Path);
|
||||||
AuthService service = CreateService();
|
AuthService service = CreateService();
|
||||||
Directory.CreateDirectory("chrome-data");
|
Directory.CreateDirectory("chromium-data");
|
||||||
File.WriteAllText("chrome-data/test.txt", "x");
|
File.WriteAllText("chromium-data/test.txt", "x");
|
||||||
File.WriteAllText("auth.json", "{}");
|
File.WriteAllText("auth.json", "{}");
|
||||||
|
|
||||||
service.Logout();
|
service.Logout();
|
||||||
|
|
||||||
Assert.False(Directory.Exists("chrome-data"));
|
Assert.False(Directory.Exists("chromium-data"));
|
||||||
Assert.False(File.Exists("auth.json"));
|
Assert.False(File.Exists("auth.json"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -21,6 +21,8 @@ public class ConfigServiceTests
|
|||||||
Assert.True(File.Exists("config.conf"));
|
Assert.True(File.Exists("config.conf"));
|
||||||
Assert.True(loggingService.UpdateCount > 0);
|
Assert.True(loggingService.UpdateCount > 0);
|
||||||
Assert.Equal(service.CurrentConfig.LoggingLevel, loggingService.LastLevel);
|
Assert.Equal(service.CurrentConfig.LoggingLevel, loggingService.LastLevel);
|
||||||
|
Assert.Equal("", service.CurrentConfig.FFprobePath);
|
||||||
|
Assert.Equal(0.98, service.CurrentConfig.DrmVideoDurationMatchThreshold, 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@ -58,6 +60,26 @@ public class ConfigServiceTests
|
|||||||
Assert.False(result);
|
Assert.False(result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task LoadConfigurationAsync_ParsesDrmVideoDurationMatchThreshold()
|
||||||
|
{
|
||||||
|
using TempFolder temp = new();
|
||||||
|
using CurrentDirectoryScope _ = new(temp.Path);
|
||||||
|
FakeLoggingService loggingService = new();
|
||||||
|
ConfigService service = new(loggingService);
|
||||||
|
await service.SaveConfigurationAsync();
|
||||||
|
|
||||||
|
string hocon = await File.ReadAllTextAsync("config.conf");
|
||||||
|
hocon = hocon.Replace("DrmVideoDurationMatchThreshold = 0.98",
|
||||||
|
"DrmVideoDurationMatchThreshold = 0.95");
|
||||||
|
await File.WriteAllTextAsync("config.conf", hocon);
|
||||||
|
|
||||||
|
bool result = await service.LoadConfigurationAsync([]);
|
||||||
|
|
||||||
|
Assert.True(result);
|
||||||
|
Assert.Equal(0.95, service.CurrentConfig.DrmVideoDurationMatchThreshold, 3);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void ApplyToggleableSelections_UpdatesConfigAndReturnsChange()
|
public void ApplyToggleableSelections_UpdatesConfigAndReturnsChange()
|
||||||
{
|
{
|
||||||
|
|||||||
@ -1,5 +1,10 @@
|
|||||||
|
using System.Reflection;
|
||||||
using OF_DL.Models.Config;
|
using OF_DL.Models.Config;
|
||||||
using OF_DL.Models.Downloads;
|
using OF_DL.Models.Downloads;
|
||||||
|
using OF_DL.Models;
|
||||||
|
using PostEntities = OF_DL.Models.Entities.Posts;
|
||||||
|
using PurchasedEntities = OF_DL.Models.Entities.Purchased;
|
||||||
|
using MessageEntities = OF_DL.Models.Entities.Messages;
|
||||||
using OF_DL.Services;
|
using OF_DL.Services;
|
||||||
|
|
||||||
namespace OF_DL.Tests.Services;
|
namespace OF_DL.Tests.Services;
|
||||||
@ -77,7 +82,7 @@ public class DownloadServiceTests
|
|||||||
DownloadService service =
|
DownloadService service =
|
||||||
CreateService(new FakeConfigService(new Config()), new MediaTrackingDbService(), apiService);
|
CreateService(new FakeConfigService(new Config()), new MediaTrackingDbService(), apiService);
|
||||||
|
|
||||||
(string decryptionKey, DateTime lastModified)? result = await service.GetDecryptionInfo(
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? result = await service.GetDecryptionInfo(
|
||||||
"https://example.com/file.mpd", "policy", "signature", "kvp", "1", "2", "post",
|
"https://example.com/file.mpd", "policy", "signature", "kvp", "1", "2", "post",
|
||||||
true, false);
|
true, false);
|
||||||
|
|
||||||
@ -95,7 +100,7 @@ public class DownloadServiceTests
|
|||||||
DownloadService service =
|
DownloadService service =
|
||||||
CreateService(new FakeConfigService(new Config()), new MediaTrackingDbService(), apiService);
|
CreateService(new FakeConfigService(new Config()), new MediaTrackingDbService(), apiService);
|
||||||
|
|
||||||
(string decryptionKey, DateTime lastModified)? result = await service.GetDecryptionInfo(
|
(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)? result = await service.GetDecryptionInfo(
|
||||||
"https://example.com/file.mpd", "policy", "signature", "kvp", "1", "2", "post",
|
"https://example.com/file.mpd", "policy", "signature", "kvp", "1", "2", "post",
|
||||||
false, false);
|
false, false);
|
||||||
|
|
||||||
@ -106,6 +111,38 @@ public class DownloadServiceTests
|
|||||||
Assert.False(apiService.OfdlCalled);
|
Assert.False(apiService.OfdlCalled);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FinalizeDrmDownload_DoesNotDeleteFile_WhenCustomPathMatchesTempPath()
|
||||||
|
{
|
||||||
|
using TempFolder temp = new();
|
||||||
|
string folder = NormalizeFolder(Path.Combine(temp.Path, "creator"));
|
||||||
|
string path = "/Posts/Free";
|
||||||
|
string filename = "video";
|
||||||
|
string customFileName = "video_source";
|
||||||
|
string tempFilename = $"{folder}{path}/{filename}_source.mp4";
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(tempFilename) ?? throw new InvalidOperationException());
|
||||||
|
await File.WriteAllTextAsync(tempFilename, "abc");
|
||||||
|
|
||||||
|
MediaTrackingDbService dbService = new();
|
||||||
|
DownloadService service = CreateService(new FakeConfigService(new Config { ShowScrapeSize = false }), dbService);
|
||||||
|
ProgressRecorder progress = new();
|
||||||
|
|
||||||
|
MethodInfo? finalizeMethod = typeof(DownloadService).GetMethod("FinalizeDrmDownload",
|
||||||
|
BindingFlags.Instance | BindingFlags.NonPublic);
|
||||||
|
Assert.NotNull(finalizeMethod);
|
||||||
|
|
||||||
|
object? resultObject = finalizeMethod.Invoke(service,
|
||||||
|
[
|
||||||
|
tempFilename, DateTime.UtcNow, folder, path, customFileName, filename, 1L, "Posts", progress
|
||||||
|
]);
|
||||||
|
|
||||||
|
bool result = await Assert.IsType<Task<bool>>(resultObject!);
|
||||||
|
Assert.True(result);
|
||||||
|
Assert.True(File.Exists(tempFilename));
|
||||||
|
Assert.NotNull(dbService.LastUpdateMedia);
|
||||||
|
Assert.Equal("video_source.mp4", dbService.LastUpdateMedia.Value.filename);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task DownloadHighlights_ReturnsZeroWhenNoMedia()
|
public async Task DownloadHighlights_ReturnsZeroWhenNoMedia()
|
||||||
{
|
{
|
||||||
@ -150,11 +187,218 @@ public class DownloadServiceTests
|
|||||||
Assert.Equal(2, progress.Total);
|
Assert.Equal(2, progress.Total);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static DownloadService CreateService(FakeConfigService configService, MediaTrackingDbService dbService,
|
[Fact]
|
||||||
StaticApiService? apiService = null) =>
|
public async Task DownloadFreePosts_UsesDefaultFilenameWhenNoGlobalOrCreatorFormatIsDefined()
|
||||||
new(new FakeAuthService(), configService, dbService, new FakeFileNameService(),
|
{
|
||||||
apiService ?? new StaticApiService());
|
using TempFolder temp = new();
|
||||||
|
string folder = NormalizeFolder(Path.Combine(temp.Path, "creator"));
|
||||||
|
const string serverFilename = "server-name";
|
||||||
|
string existingFilePath = $"{folder}/Posts/Free/Images/{serverFilename}.jpg";
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(existingFilePath) ?? throw new InvalidOperationException());
|
||||||
|
await File.WriteAllTextAsync(existingFilePath, "abc");
|
||||||
|
|
||||||
private static string NormalizeFolder(string folder) => folder.Replace("\\", "/");
|
Config config = new()
|
||||||
|
{
|
||||||
|
ShowScrapeSize = false,
|
||||||
|
PostFileNameFormat = "",
|
||||||
|
CreatorConfigs = new Dictionary<string, CreatorConfig>
|
||||||
|
{
|
||||||
|
["creator"] = new() { PostFileNameFormat = "" }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
MediaTrackingDbService dbService = new() { CheckDownloadedResult = false };
|
||||||
|
DownloadService service = CreateService(new FakeConfigService(config), dbService);
|
||||||
|
ProgressRecorder progress = new();
|
||||||
|
PostEntities.PostCollection posts = new()
|
||||||
|
{
|
||||||
|
Posts = new Dictionary<long, string> { { 1, $"https://example.com/{serverFilename}.jpg" } }
|
||||||
|
};
|
||||||
|
|
||||||
|
DownloadResult result = await service.DownloadFreePosts("creator", 1, folder, new Dictionary<string, long>(),
|
||||||
|
false, false, posts, progress);
|
||||||
|
|
||||||
|
Assert.Equal(1, result.TotalCount);
|
||||||
|
Assert.Equal(0, result.NewDownloads);
|
||||||
|
Assert.Equal(1, result.ExistingDownloads);
|
||||||
|
Assert.NotNull(dbService.LastUpdateMedia);
|
||||||
|
Assert.Equal($"{serverFilename}.jpg", dbService.LastUpdateMedia.Value.filename);
|
||||||
|
Assert.Equal(1, progress.Total);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task DownloadFreePosts_UsesGlobalCustomFormatWhenCreatorCustomFormatNotDefined()
|
||||||
|
{
|
||||||
|
using TempFolder temp = new();
|
||||||
|
string folder = NormalizeFolder(Path.Combine(temp.Path, "creator"));
|
||||||
|
const string serverFilename = "server-name";
|
||||||
|
string existingFilePath = $"{folder}/Posts/Free/Images/{serverFilename}.jpg";
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(existingFilePath) ?? throw new InvalidOperationException());
|
||||||
|
await File.WriteAllTextAsync(existingFilePath, "abc");
|
||||||
|
|
||||||
|
Config config = new()
|
||||||
|
{
|
||||||
|
ShowScrapeSize = false,
|
||||||
|
PostFileNameFormat = "global-custom-name",
|
||||||
|
CreatorConfigs = new Dictionary<string, CreatorConfig>
|
||||||
|
{
|
||||||
|
["creator"] = new() { PostFileNameFormat = "" }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
MediaTrackingDbService dbService = new() { CheckDownloadedResult = false };
|
||||||
|
DownloadService service = CreateService(new FakeConfigService(config), dbService,
|
||||||
|
fileNameService: new DeterministicFileNameService());
|
||||||
|
ProgressRecorder progress = new();
|
||||||
|
PostEntities.PostCollection posts = CreatePostCollection(1, serverFilename);
|
||||||
|
|
||||||
|
DownloadResult result = await service.DownloadFreePosts("creator", 1, folder, new Dictionary<string, long>(),
|
||||||
|
false, false, posts, progress);
|
||||||
|
|
||||||
|
string renamedPath = $"{folder}/Posts/Free/Images/global-custom-name.jpg";
|
||||||
|
Assert.Equal(1, result.TotalCount);
|
||||||
|
Assert.Equal(0, result.NewDownloads);
|
||||||
|
Assert.Equal(1, result.ExistingDownloads);
|
||||||
|
Assert.False(File.Exists(existingFilePath));
|
||||||
|
Assert.True(File.Exists(renamedPath));
|
||||||
|
Assert.NotNull(dbService.LastUpdateMedia);
|
||||||
|
Assert.Equal("global-custom-name.jpg", dbService.LastUpdateMedia.Value.filename);
|
||||||
|
Assert.Equal(1, progress.Total);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task DownloadFreePosts_UsesCreatorCustomFormatWhenGlobalAndCreatorFormatsAreDefined()
|
||||||
|
{
|
||||||
|
using TempFolder temp = new();
|
||||||
|
string folder = NormalizeFolder(Path.Combine(temp.Path, "creator"));
|
||||||
|
const string serverFilename = "server-name";
|
||||||
|
string existingFilePath = $"{folder}/Posts/Free/Images/{serverFilename}.jpg";
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(existingFilePath) ?? throw new InvalidOperationException());
|
||||||
|
await File.WriteAllTextAsync(existingFilePath, "abc");
|
||||||
|
|
||||||
|
Config config = new()
|
||||||
|
{
|
||||||
|
ShowScrapeSize = false,
|
||||||
|
PostFileNameFormat = "global-custom-name",
|
||||||
|
CreatorConfigs = new Dictionary<string, CreatorConfig>
|
||||||
|
{
|
||||||
|
["creator"] = new() { PostFileNameFormat = "creator-custom-name" }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
MediaTrackingDbService dbService = new() { CheckDownloadedResult = false };
|
||||||
|
DownloadService service = CreateService(new FakeConfigService(config), dbService,
|
||||||
|
fileNameService: new DeterministicFileNameService());
|
||||||
|
ProgressRecorder progress = new();
|
||||||
|
PostEntities.PostCollection posts = CreatePostCollection(1, serverFilename);
|
||||||
|
|
||||||
|
DownloadResult result = await service.DownloadFreePosts("creator", 1, folder, new Dictionary<string, long>(),
|
||||||
|
false, false, posts, progress);
|
||||||
|
|
||||||
|
string renamedPath = $"{folder}/Posts/Free/Images/creator-custom-name.jpg";
|
||||||
|
Assert.Equal(1, result.TotalCount);
|
||||||
|
Assert.Equal(0, result.NewDownloads);
|
||||||
|
Assert.Equal(1, result.ExistingDownloads);
|
||||||
|
Assert.False(File.Exists(existingFilePath));
|
||||||
|
Assert.True(File.Exists(renamedPath));
|
||||||
|
Assert.NotNull(dbService.LastUpdateMedia);
|
||||||
|
Assert.Equal("creator-custom-name.jpg", dbService.LastUpdateMedia.Value.filename);
|
||||||
|
Assert.Equal(1, progress.Total);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task DownloadPaidPosts_AppliesPaidCustomFormatForDrm_WhenAuthorExistsButFromUserMissing()
|
||||||
|
{
|
||||||
|
using TempFolder temp = new();
|
||||||
|
string folder = NormalizeFolder(Path.Combine(temp.Path, "creator"));
|
||||||
|
const string customName = "paid-custom-name";
|
||||||
|
const string drmBaseFilename = "video-file";
|
||||||
|
string basePath = $"{folder}/Posts/Paid/Videos";
|
||||||
|
Directory.CreateDirectory(basePath);
|
||||||
|
await File.WriteAllTextAsync($"{basePath}/{customName}.mp4", "custom");
|
||||||
|
await File.WriteAllTextAsync($"{basePath}/{drmBaseFilename}_source.mp4", "server");
|
||||||
|
|
||||||
|
Config config = new() { ShowScrapeSize = false, PaidPostFileNameFormat = customName, PostFileNameFormat = "" };
|
||||||
|
|
||||||
|
MediaTrackingDbService dbService = new() { CheckDownloadedResult = false };
|
||||||
|
StaticApiService apiService = new();
|
||||||
|
FakeAuthService authService = new()
|
||||||
|
{
|
||||||
|
CurrentAuth = new Auth { Cookie = "sess=test;", UserAgent = "unit-test-agent" }
|
||||||
|
};
|
||||||
|
|
||||||
|
DownloadService service = CreateService(new FakeConfigService(config), dbService,
|
||||||
|
apiService, new DeterministicFileNameService(), authService);
|
||||||
|
ProgressRecorder progress = new();
|
||||||
|
PurchasedEntities.PaidPostCollection posts = CreatePaidPostCollectionForDrm(1,
|
||||||
|
$"https://cdn3.onlyfans.com/dash/files/{drmBaseFilename}.mpd,policy,signature,kvp,1,2");
|
||||||
|
|
||||||
|
DownloadResult result = await service.DownloadPaidPosts("creator", 1, folder, new Dictionary<string, long>(),
|
||||||
|
false, false, posts, progress);
|
||||||
|
|
||||||
|
Assert.Equal(1, result.TotalCount);
|
||||||
|
Assert.Equal(0, result.NewDownloads);
|
||||||
|
Assert.Equal(1, result.ExistingDownloads);
|
||||||
|
Assert.NotNull(dbService.LastUpdateMedia);
|
||||||
|
Assert.Equal($"{customName}.mp4", dbService.LastUpdateMedia.Value.filename);
|
||||||
|
Assert.Equal(1, progress.Total);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static DownloadService CreateService(FakeConfigService configService, MediaTrackingDbService dbService,
|
||||||
|
StaticApiService? apiService = null, IFileNameService? fileNameService = null,
|
||||||
|
IAuthService? authService = null) =>
|
||||||
|
new(authService ?? new FakeAuthService(), configService, dbService,
|
||||||
|
fileNameService ?? new FakeFileNameService(),
|
||||||
|
apiService ?? new StaticApiService());
|
||||||
|
|
||||||
|
private static PostEntities.PostCollection CreatePostCollection(long mediaId, string serverFilename)
|
||||||
|
{
|
||||||
|
PostEntities.Medium media = new() { Id = mediaId };
|
||||||
|
PostEntities.ListItem post = new()
|
||||||
|
{
|
||||||
|
Id = 10,
|
||||||
|
PostedAt = new DateTime(2024, 1, 1),
|
||||||
|
Author = new OF_DL.Models.Entities.Common.Author { Id = 99 },
|
||||||
|
Media = [media]
|
||||||
|
};
|
||||||
|
|
||||||
|
return new PostEntities.PostCollection
|
||||||
|
{
|
||||||
|
Posts = new Dictionary<long, string> { { mediaId, $"https://example.com/{serverFilename}.jpg" } },
|
||||||
|
PostMedia = [media],
|
||||||
|
PostObjects = [post]
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static PurchasedEntities.PaidPostCollection CreatePaidPostCollectionForDrm(long mediaId, string drmUrl)
|
||||||
|
{
|
||||||
|
MessageEntities.Medium media = new() { Id = mediaId };
|
||||||
|
PurchasedEntities.ListItem post = new()
|
||||||
|
{
|
||||||
|
Id = 20,
|
||||||
|
PostedAt = new DateTime(2024, 1, 1),
|
||||||
|
Author = new OF_DL.Models.Entities.Common.Author { Id = 99 },
|
||||||
|
FromUser = null,
|
||||||
|
Media = [media]
|
||||||
|
};
|
||||||
|
|
||||||
|
return new PurchasedEntities.PaidPostCollection
|
||||||
|
{
|
||||||
|
PaidPosts = new Dictionary<long, string> { { mediaId, drmUrl } },
|
||||||
|
PaidPostMedia = [media],
|
||||||
|
PaidPostObjects = [post]
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string NormalizeFolder(string folder) => folder.Replace("\\", "/");
|
||||||
|
|
||||||
|
private sealed class DeterministicFileNameService : IFileNameService
|
||||||
|
{
|
||||||
|
public Task<string> BuildFilename(string fileFormat, Dictionary<string, string> values) =>
|
||||||
|
Task.FromResult(fileFormat);
|
||||||
|
|
||||||
|
public Task<Dictionary<string, string>> GetFilename(object info, object media, object author,
|
||||||
|
List<string> selectedProperties, string username, Dictionary<string, long>? users = null) =>
|
||||||
|
Task.FromResult(new Dictionary<string, string>());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@ -20,7 +20,7 @@ internal sealed class TestMedia
|
|||||||
|
|
||||||
internal sealed class TestMediaFiles
|
internal sealed class TestMediaFiles
|
||||||
{
|
{
|
||||||
public TestMediaFull Full { get; set; } = new();
|
public TestMediaFull? Full { get; set; } = new();
|
||||||
public object? Drm { get; set; }
|
public object? Drm { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -76,4 +76,24 @@ public class FileNameServiceTests
|
|||||||
|
|
||||||
Assert.Equal("creator_99", result);
|
Assert.Equal("creator_99", result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task GetFilename_UsesDrmFilenameWhenFullUrlMissing()
|
||||||
|
{
|
||||||
|
TestMedia media = new()
|
||||||
|
{
|
||||||
|
Id = 99,
|
||||||
|
Files = new TestMediaFiles
|
||||||
|
{
|
||||||
|
Full = null,
|
||||||
|
Drm = new { Manifest = new { Dash = "https://cdn.test/drm-name.mpd" } }
|
||||||
|
}
|
||||||
|
};
|
||||||
|
FileNameService service = new(new FakeAuthService());
|
||||||
|
|
||||||
|
Dictionary<string, string> values =
|
||||||
|
await service.GetFilename(new TestInfo(), media, new TestAuthor(), ["filename"], "creator");
|
||||||
|
|
||||||
|
Assert.Equal("drm-name_source", values["filename"]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -116,11 +116,10 @@ internal sealed class StaticApiService : IApiService
|
|||||||
|
|
||||||
public bool CdmCalled { get; private set; }
|
public bool CdmCalled { get; private set; }
|
||||||
|
|
||||||
public Task<string> GetDrmMpdPssh(string mpdUrl, string policy, string signature, string kvp) =>
|
public Task<(string pssh, DateTime lastModified, double? durationSeconds)> GetDrmMpdInfo(
|
||||||
Task.FromResult("pssh");
|
string mpdUrl, string policy, string signature, string kvp) =>
|
||||||
|
Task.FromResult<(string pssh, DateTime lastModified, double? durationSeconds)>(
|
||||||
public Task<DateTime> GetDrmMpdLastModified(string mpdUrl, string policy, string signature, string kvp) =>
|
("pssh", LastModifiedToReturn, null));
|
||||||
Task.FromResult(LastModifiedToReturn);
|
|
||||||
|
|
||||||
public Task<string> GetDecryptionKeyOfdl(Dictionary<string, string> drmHeaders, string licenceUrl, string pssh)
|
public Task<string> GetDecryptionKeyOfdl(Dictionary<string, string> drmHeaders, string licenceUrl, string pssh)
|
||||||
{
|
{
|
||||||
@ -271,10 +270,8 @@ internal sealed class ConfigurableApiService : IApiService
|
|||||||
public Task<string> GetDecryptionKeyCdm(Dictionary<string, string> drmHeaders, string licenceUrl, string pssh) =>
|
public Task<string> GetDecryptionKeyCdm(Dictionary<string, string> drmHeaders, string licenceUrl, string pssh) =>
|
||||||
throw new NotImplementedException();
|
throw new NotImplementedException();
|
||||||
|
|
||||||
public Task<DateTime> GetDrmMpdLastModified(string mpdUrl, string policy, string signature, string kvp) =>
|
public Task<(string pssh, DateTime lastModified, double? durationSeconds)> GetDrmMpdInfo(
|
||||||
throw new NotImplementedException();
|
string mpdUrl, string policy, string signature, string kvp) =>
|
||||||
|
|
||||||
public Task<string> GetDrmMpdPssh(string mpdUrl, string policy, string signature, string kvp) =>
|
|
||||||
throw new NotImplementedException();
|
throw new NotImplementedException();
|
||||||
|
|
||||||
public Task<string> GetDecryptionKeyOfdl(Dictionary<string, string> drmHeaders, string licenceUrl, string pssh) =>
|
public Task<string> GetDecryptionKeyOfdl(Dictionary<string, string> drmHeaders, string licenceUrl, string pssh) =>
|
||||||
@ -296,7 +293,7 @@ internal sealed class OrchestrationDownloadServiceStub : IDownloadService
|
|||||||
string serverFileName, string resolvedFileName, string extension, IProgressReporter progressReporter) =>
|
string serverFileName, string resolvedFileName, string extension, IProgressReporter progressReporter) =>
|
||||||
throw new NotImplementedException();
|
throw new NotImplementedException();
|
||||||
|
|
||||||
public Task<(string decryptionKey, DateTime lastModified)?> GetDecryptionInfo(string mpdUrl, string policy,
|
public Task<(string decryptionKey, DateTime lastModified, double? mpdDurationSeconds)?> GetDecryptionInfo(string mpdUrl, string policy,
|
||||||
string signature, string kvp, string mediaId, string contentId, string drmType, bool clientIdBlobMissing,
|
string signature, string kvp, string mediaId, string contentId, string drmType, bool clientIdBlobMissing,
|
||||||
bool devicePrivateKeyMissing) =>
|
bool devicePrivateKeyMissing) =>
|
||||||
throw new NotImplementedException();
|
throw new NotImplementedException();
|
||||||
|
|||||||
@ -71,7 +71,7 @@ public class SpectreDownloadEventHandler : IDownloadEventHandler
|
|||||||
|
|
||||||
public void OnDownloadComplete(string contentType, DownloadResult result) =>
|
public void OnDownloadComplete(string contentType, DownloadResult result) =>
|
||||||
AnsiConsole.Markup(
|
AnsiConsole.Markup(
|
||||||
$"[red]{Markup.Escape(contentType)} Already Downloaded: {result.ExistingDownloads} New {Markup.Escape(contentType)} Downloaded: {result.NewDownloads}[/]\n");
|
$"[red]{Markup.Escape(contentType)} Media Already Downloaded: {result.ExistingDownloads} New {Markup.Escape(contentType)} Media Downloaded: {result.NewDownloads}[/]\n");
|
||||||
|
|
||||||
public void OnUserStarting(string username) =>
|
public void OnUserStarting(string username) =>
|
||||||
AnsiConsole.Markup($"[red]\nScraping Data for {Markup.Escape(username)}\n[/]");
|
AnsiConsole.Markup($"[red]\nScraping Data for {Markup.Escape(username)}\n[/]");
|
||||||
|
|||||||
@ -1,12 +1,12 @@
|
|||||||
<Project Sdk="Microsoft.NET.Sdk">
|
<Project Sdk="Microsoft.NET.Sdk">
|
||||||
|
|
||||||
<PropertyGroup>
|
<PropertyGroup>
|
||||||
<OutputType>Exe</OutputType>
|
|
||||||
<TargetFramework>net10.0</TargetFramework>
|
<TargetFramework>net10.0</TargetFramework>
|
||||||
<RootNamespace>OF_DL</RootNamespace>
|
<RootNamespace>OF_DL</RootNamespace>
|
||||||
<ImplicitUsings>enable</ImplicitUsings>
|
<ImplicitUsings>enable</ImplicitUsings>
|
||||||
<Nullable>enable</Nullable>
|
<Nullable>enable</Nullable>
|
||||||
<ApplicationIcon>Icon\download.ico</ApplicationIcon>
|
<ApplicationIcon>Icon\download.ico</ApplicationIcon>
|
||||||
|
<DebugType>embedded</DebugType>
|
||||||
</PropertyGroup>
|
</PropertyGroup>
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
@ -23,9 +23,9 @@
|
|||||||
<PackageReference Include="HtmlAgilityPack" Version="1.12.4"/>
|
<PackageReference Include="HtmlAgilityPack" Version="1.12.4"/>
|
||||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.3"/>
|
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.3"/>
|
||||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.3"/>
|
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.3"/>
|
||||||
|
<PackageReference Include="Microsoft.Playwright" Version="1.58.0"/>
|
||||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.4"/>
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.4"/>
|
||||||
<PackageReference Include="protobuf-net" Version="3.2.56"/>
|
<PackageReference Include="protobuf-net" Version="3.2.56"/>
|
||||||
<PackageReference Include="PuppeteerSharp" Version="20.2.6"/>
|
|
||||||
<PackageReference Include="Serilog" Version="4.3.1"/>
|
<PackageReference Include="Serilog" Version="4.3.1"/>
|
||||||
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1"/>
|
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1"/>
|
||||||
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0"/>
|
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0"/>
|
||||||
@ -42,13 +42,23 @@
|
|||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<None Update="auth.json">
|
<None Update="auth.json">
|
||||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||||
|
<ExcludeFromSingleFile>true</ExcludeFromSingleFile>
|
||||||
</None>
|
</None>
|
||||||
<None Update="config.conf">
|
<None Update="config.conf">
|
||||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||||
|
<ExcludeFromSingleFile>true</ExcludeFromSingleFile>
|
||||||
</None>
|
</None>
|
||||||
<None Update="rules.json">
|
<None Update="rules.json">
|
||||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||||
|
<ExcludeFromSingleFile>true</ExcludeFromSingleFile>
|
||||||
|
</None>
|
||||||
|
<None Update="chromium-scripts/stealth.min.js">
|
||||||
|
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||||
</None>
|
</None>
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
|
<ItemGroup>
|
||||||
|
<InternalsVisibleTo Include="Cajetan.OF-DL" />
|
||||||
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@ -9,6 +9,7 @@ using OF_DL.Models.Entities.Users;
|
|||||||
using OF_DL.Services;
|
using OF_DL.Services;
|
||||||
using Serilog;
|
using Serilog;
|
||||||
using Spectre.Console;
|
using Spectre.Console;
|
||||||
|
using OF_DL.Helpers;
|
||||||
|
|
||||||
namespace OF_DL;
|
namespace OF_DL;
|
||||||
|
|
||||||
@ -37,8 +38,6 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
{
|
{
|
||||||
AnsiConsole.MarkupLine(
|
AnsiConsole.MarkupLine(
|
||||||
"[yellow]In the new window that has opened, please log in to your OF account. Do not close the window or tab. Do not navigate away from the page.[/]\n");
|
"[yellow]In the new window that has opened, please log in to your OF account. Do not close the window or tab. Do not navigate away from the page.[/]\n");
|
||||||
AnsiConsole.MarkupLine(
|
|
||||||
"[yellow]Note: Some users have reported that \"Sign in with Google\" has not been working with the new authentication method.[/]");
|
|
||||||
AnsiConsole.MarkupLine(
|
AnsiConsole.MarkupLine(
|
||||||
"[yellow]If you use this method or encounter other issues while logging in, use one of the legacy authentication methods documented here:[/]");
|
"[yellow]If you use this method or encounter other issues while logging in, use one of the legacy authentication methods documented here:[/]");
|
||||||
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
||||||
@ -59,7 +58,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
|
||||||
Log.Error("auth invalid after attempt to get auth from browser");
|
Log.Error("auth invalid after attempt to get auth from browser");
|
||||||
|
|
||||||
Environment.Exit(2);
|
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(2);
|
||||||
}
|
}
|
||||||
|
|
||||||
await authService.SaveToFileAsync();
|
await authService.SaveToFileAsync();
|
||||||
@ -85,11 +84,12 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
ServiceCollection services = new();
|
ServiceCollection services = new();
|
||||||
services.AddSingleton<ILoggingService, LoggingService>();
|
services.AddSingleton<ILoggingService, LoggingService>();
|
||||||
services.AddSingleton<IConfigService, ConfigService>();
|
services.AddSingleton<IConfigService, ConfigService>();
|
||||||
|
services.AddSingleton(new ExitHelper(new SpectreDownloadEventHandler()));
|
||||||
ServiceProvider tempServiceProvider = services.BuildServiceProvider();
|
ServiceProvider tempServiceProvider = services.BuildServiceProvider();
|
||||||
|
|
||||||
ILoggingService loggingService = tempServiceProvider.GetRequiredService<ILoggingService>();
|
ILoggingService loggingService = tempServiceProvider.GetRequiredService<ILoggingService>();
|
||||||
IConfigService configService = tempServiceProvider.GetRequiredService<IConfigService>();
|
IConfigService configService = tempServiceProvider.GetRequiredService<IConfigService>();
|
||||||
|
ExitHelper exitHelper = tempServiceProvider.GetRequiredService<ExitHelper>();
|
||||||
|
|
||||||
if (!await configService.LoadConfigurationAsync(args))
|
if (!await configService.LoadConfigurationAsync(args))
|
||||||
{
|
{
|
||||||
@ -100,7 +100,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
Console.ReadKey();
|
Console.ReadKey();
|
||||||
}
|
}
|
||||||
|
|
||||||
Environment.Exit(3);
|
exitHelper.ExitWithCode(3);
|
||||||
}
|
}
|
||||||
|
|
||||||
AnsiConsole.Markup("[green]config.conf located successfully!\n[/]");
|
AnsiConsole.Markup("[green]config.conf located successfully!\n[/]");
|
||||||
@ -109,6 +109,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
services = [];
|
services = [];
|
||||||
services.AddSingleton(loggingService);
|
services.AddSingleton(loggingService);
|
||||||
services.AddSingleton(configService);
|
services.AddSingleton(configService);
|
||||||
|
services.AddSingleton(exitHelper);
|
||||||
services.AddSingleton<IAuthService, AuthService>();
|
services.AddSingleton<IAuthService, AuthService>();
|
||||||
services.AddSingleton<IApiService, ApiService>();
|
services.AddSingleton<IApiService, ApiService>();
|
||||||
services.AddSingleton<IDbService, DbService>();
|
services.AddSingleton<IDbService, DbService>();
|
||||||
@ -152,7 +153,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
Console.ReadKey();
|
Console.ReadKey();
|
||||||
}
|
}
|
||||||
|
|
||||||
Environment.Exit(1);
|
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!startupResult.FfmpegFound)
|
if (!startupResult.FfmpegFound)
|
||||||
@ -169,6 +170,23 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
"[red]Cannot locate FFmpeg; please modify config.conf with the correct path.[/]");
|
"[red]Cannot locate FFmpeg; please modify config.conf with the correct path.[/]");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(4);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!startupResult.FfprobeFound)
|
||||||
|
{
|
||||||
|
if (!configService.CurrentConfig.NonInteractiveMode)
|
||||||
|
{
|
||||||
|
AnsiConsole.Markup(
|
||||||
|
"[red]Cannot locate FFprobe; please modify config.conf with the correct path. Press any key to exit.[/]");
|
||||||
|
Console.ReadKey();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
AnsiConsole.Markup(
|
||||||
|
"[red]Cannot locate FFprobe; please modify config.conf with the correct path.[/]");
|
||||||
|
}
|
||||||
|
|
||||||
Environment.Exit(4);
|
Environment.Exit(4);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -195,13 +213,13 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
Log.Error("Auth failed");
|
Log.Error("Auth failed");
|
||||||
authService.CurrentAuth = null;
|
authService.CurrentAuth = null;
|
||||||
|
|
||||||
if (!configService.CurrentConfig.DisableBrowserAuth)
|
//if (!configService.CurrentConfig.DisableBrowserAuth)
|
||||||
{
|
//{
|
||||||
if (File.Exists("auth.json"))
|
// if (File.Exists("auth.json"))
|
||||||
{
|
// {
|
||||||
File.Delete("auth.json");
|
// File.Delete("auth.json");
|
||||||
}
|
// }
|
||||||
}
|
//}
|
||||||
|
|
||||||
if (!configService.CurrentConfig.NonInteractiveMode &&
|
if (!configService.CurrentConfig.NonInteractiveMode &&
|
||||||
!configService.CurrentConfig.DisableBrowserAuth)
|
!configService.CurrentConfig.DisableBrowserAuth)
|
||||||
@ -220,7 +238,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
Console.ReadKey();
|
Console.ReadKey();
|
||||||
}
|
}
|
||||||
|
|
||||||
Environment.Exit(2);
|
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -249,7 +267,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
Console.ReadKey();
|
Console.ReadKey();
|
||||||
}
|
}
|
||||||
|
|
||||||
Environment.Exit(5);
|
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(5);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -733,11 +751,11 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
else if (File.Exists("auth.json"))
|
else if (File.Exists("auth.json"))
|
||||||
{
|
{
|
||||||
Log.Information("Auth file found but could not be deserialized");
|
Log.Information("Auth file found but could not be deserialized");
|
||||||
if (!configService.CurrentConfig.DisableBrowserAuth)
|
//if (!configService.CurrentConfig.DisableBrowserAuth)
|
||||||
{
|
//{
|
||||||
Log.Debug("Deleting auth.json");
|
// Log.Debug("Deleting auth.json");
|
||||||
File.Delete("auth.json");
|
// File.Delete("auth.json");
|
||||||
}
|
//}
|
||||||
|
|
||||||
if (configService.CurrentConfig.NonInteractiveMode)
|
if (configService.CurrentConfig.NonInteractiveMode)
|
||||||
{
|
{
|
||||||
@ -746,7 +764,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
AnsiConsole.MarkupLine(
|
AnsiConsole.MarkupLine(
|
||||||
"[red]You may also want to try using the browser extension which is documented here:[/]\n");
|
"[red]You may also want to try using the browser extension which is documented here:[/]\n");
|
||||||
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
AnsiConsole.MarkupLine("[link]https://docs.ofdl.tools/config/auth/#legacy-methods[/]");
|
||||||
Environment.Exit(2);
|
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(2);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!configService.CurrentConfig.DisableBrowserAuth)
|
if (!configService.CurrentConfig.DisableBrowserAuth)
|
||||||
@ -775,7 +793,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void ShowAuthMissingError(bool nonInteractiveMode)
|
private void ShowAuthMissingError(bool nonInteractiveMode)
|
||||||
{
|
{
|
||||||
AnsiConsole.MarkupLine(
|
AnsiConsole.MarkupLine(
|
||||||
"\n[red]auth.json is missing. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n");
|
"\n[red]auth.json is missing. The file can be generated automatically when OF-DL is run in the standard, interactive mode.[/]\n");
|
||||||
@ -789,7 +807,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
Console.ReadKey();
|
Console.ReadKey();
|
||||||
}
|
}
|
||||||
|
|
||||||
Environment.Exit(2);
|
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(2);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void DisplayVersionResult(VersionCheckResult result)
|
private static void DisplayVersionResult(VersionCheckResult result)
|
||||||
@ -840,25 +858,28 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
|
|
||||||
// FFmpeg
|
// FFmpeg
|
||||||
if (result.FfmpegFound)
|
if (result.FfmpegFound)
|
||||||
{
|
|
||||||
if (result.FfmpegPathAutoDetected && result.FfmpegPath != null)
|
|
||||||
{
|
{
|
||||||
AnsiConsole.Markup(
|
AnsiConsole.Markup(
|
||||||
$"[green]FFmpeg located successfully. Path auto-detected: {Markup.Escape(result.FfmpegPath)}\n[/]");
|
result is { FfmpegPathAutoDetected: true, FfmpegPath: not null }
|
||||||
}
|
? $"[green]FFmpeg located successfully. Path auto-detected: {Markup.Escape(result.FfmpegPath)}\n[/]"
|
||||||
else
|
: "[green]FFmpeg located successfully\n[/]");
|
||||||
{
|
|
||||||
AnsiConsole.Markup("[green]FFmpeg located successfully\n[/]");
|
AnsiConsole.Markup(result.FfmpegVersion != null
|
||||||
|
? $"[green]ffmpeg version detected as {Markup.Escape(result.FfmpegVersion)}[/]\n"
|
||||||
|
: "[yellow]ffmpeg version could not be parsed[/]\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (result.FfmpegVersion != null)
|
// FFprobe
|
||||||
|
if (result.FfprobeFound)
|
||||||
{
|
{
|
||||||
AnsiConsole.Markup($"[green]ffmpeg version detected as {Markup.Escape(result.FfmpegVersion)}[/]\n");
|
AnsiConsole.Markup(
|
||||||
}
|
result is { FfprobePathAutoDetected: true, FfprobePath: not null }
|
||||||
else
|
? $"[green]FFprobe located successfully. Path auto-detected: {Markup.Escape(result.FfprobePath)}\n[/]"
|
||||||
{
|
: "[green]FFprobe located successfully\n[/]");
|
||||||
AnsiConsole.Markup("[yellow]ffmpeg version could not be parsed[/]\n");
|
|
||||||
}
|
AnsiConsole.Markup(result.FfprobeVersion != null
|
||||||
|
? $"[green]FFprobe version detected as {Markup.Escape(result.FfprobeVersion)}[/]\n"
|
||||||
|
: "[yellow]FFprobe version could not be parsed[/]\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
// Widevine
|
// Widevine
|
||||||
@ -879,7 +900,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void DisplayRulesJsonResult(StartupResult result, IConfigService configService)
|
private void DisplayRulesJsonResult(StartupResult result, IConfigService configService)
|
||||||
{
|
{
|
||||||
if (result.RulesJsonExists)
|
if (result.RulesJsonExists)
|
||||||
{
|
{
|
||||||
@ -899,7 +920,7 @@ public class Program(IServiceProvider serviceProvider)
|
|||||||
Console.ReadKey();
|
Console.ReadKey();
|
||||||
}
|
}
|
||||||
|
|
||||||
Environment.Exit(2);
|
serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(2);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
14
OF DL/chromium-scripts/CREATING_STEALTH_SCRIPT.md
Normal file
14
OF DL/chromium-scripts/CREATING_STEALTH_SCRIPT.md
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
# Stealth Script Creation
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
- NodeJS (with npx CLI tool)
|
||||||
|
|
||||||
|
## Instructions
|
||||||
|
|
||||||
|
- Open a terminal in this directory (OF DL/chromium-scripts)
|
||||||
|
- Run `npx -y extract-stealth-evasions`
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
See the readme.md file and source code for the stealth script [here](https://github.com/berstend/puppeteer-extra/tree/master/packages/extract-stealth-evasions).
|
||||||
7
OF DL/chromium-scripts/stealth.min.js
vendored
Normal file
7
OF DL/chromium-scripts/stealth.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
39
Publish_OF-DL.bat
Normal file
39
Publish_OF-DL.bat
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
@ECHO OFF
|
||||||
|
|
||||||
|
ECHO.
|
||||||
|
|
||||||
|
ECHO ==============================
|
||||||
|
ECHO == Cleaning Output ===========
|
||||||
|
ECHO ==============================
|
||||||
|
dotnet clean ".\Cajetan.OF-DL\Cajetan.OF-DL.csproj" -v minimal
|
||||||
|
DEL /Q /F ".\Cajetan.OF-DL_Publish"
|
||||||
|
|
||||||
|
ECHO.
|
||||||
|
|
||||||
|
ECHO ==============================
|
||||||
|
ECHO == Publishing Cajetan OF-DL ==
|
||||||
|
ECHO ==============================
|
||||||
|
dotnet publish ".\Cajetan.OF-DL\Cajetan.OF-DL.csproj" -o ".\Cajetan.OF-DL_Publish" -c Debug
|
||||||
|
|
||||||
|
ECHO.
|
||||||
|
|
||||||
|
ECHO ==============================
|
||||||
|
ECHO == Renaming to 'OF DL.exe' ===
|
||||||
|
ECHO ==============================
|
||||||
|
REN ".\Cajetan.OF-DL_Publish\Cajetan.OF-DL.exe" "OF DL.exe"
|
||||||
|
|
||||||
|
ECHO.
|
||||||
|
ECHO ==============================
|
||||||
|
ECHO == Copy to network drive? ====
|
||||||
|
ECHO ==============================
|
||||||
|
CHOICE /C yn /m "Copy published files to network drive? "
|
||||||
|
|
||||||
|
IF %ERRORLEVEL%==1 (GOTO Copy) ELSE (GOTO Exit)
|
||||||
|
|
||||||
|
:Copy
|
||||||
|
xcopy .\Cajetan.OF-DL_Publish\* p:\_Utils\OF_DL /I /Y /Q /EXCLUDE:.\excludes.txt
|
||||||
|
|
||||||
|
:Exit
|
||||||
|
ECHO.
|
||||||
|
ECHO.
|
||||||
|
PAUSE
|
||||||
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
mkdir -p /config/cdm/devices/chrome_1610
|
mkdir -p /config/cdm/devices/chrome_1610
|
||||||
mkdir -p /config/logs/
|
mkdir -p /config/logs/
|
||||||
|
mkdir -p /config/chromium
|
||||||
|
|
||||||
if [ ! -f /config/config.conf ] && [ ! -f /config/config.json ]; then
|
if [ ! -f /config/config.conf ] && [ ! -f /config/config.json ]; then
|
||||||
cp /default-config/config.conf /config/config.conf
|
cp /default-config/config.conf /config/config.conf
|
||||||
|
|||||||
@ -1,6 +1,8 @@
|
|||||||
# All Configuration Options
|
# All Configuration Options
|
||||||
|
|
||||||
This page contains detailed information for each configuration option supported by OF-DL. For information about the structure of the `config.conf` file or a simple list of these configuration options, go to the [configuration page](/config/configuration).
|
This page contains detailed information for each configuration option supported by OF-DL. For information about the
|
||||||
|
structure of the `config.conf` file or a simple list of these configuration options, go to
|
||||||
|
the [configuration page](/config/configuration).
|
||||||
|
|
||||||
## BypassContentForCreatorsWhoNoLongerExist
|
## BypassContentForCreatorsWhoNoLongerExist
|
||||||
|
|
||||||
@ -10,9 +12,12 @@ Default: `false`
|
|||||||
|
|
||||||
Allowed values: `true`, `false`
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
Description: When a creator no longer exists (their account has been deleted), most of their content will be inaccessible.
|
Description: When a creator no longer exists (their account has been deleted), most of their content will be
|
||||||
Purchased content, however, will still be accessible by downloading media usisng the "Download Purchased Tab" menu option
|
inaccessible.
|
||||||
or with the [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) config option when downloading media in non-interactive mode.
|
Purchased content, however, will still be accessible by downloading media usisng the "Download Purchased Tab" menu
|
||||||
|
option
|
||||||
|
or with the [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) config option when downloading media in
|
||||||
|
non-interactive mode.
|
||||||
|
|
||||||
## CreatorConfigs
|
## CreatorConfigs
|
||||||
|
|
||||||
@ -23,12 +28,14 @@ Default: `{}`
|
|||||||
Allowed values: An array of Creator Config objects
|
Allowed values: An array of Creator Config objects
|
||||||
|
|
||||||
Description: This configuration options allows you to set file name formats for specific creators.
|
Description: This configuration options allows you to set file name formats for specific creators.
|
||||||
This is useful if you want to have different file name formats for different creators. The values set here will override the global values set in the config file
|
This is useful if you want to have different file name formats for different creators. The values set here will override
|
||||||
|
the global values set in the config file
|
||||||
(see [PaidPostFileNameFormat](#paidpostfilenameformat), [PostFileNameFormat](#postfilenameformat),
|
(see [PaidPostFileNameFormat](#paidpostfilenameformat), [PostFileNameFormat](#postfilenameformat),
|
||||||
[PaidMessageFileNAmeFormat](#paidmessagefilenameformat), and [MessageFileNameFormat](#messagefilenameformat)).
|
[PaidMessageFileNAmeFormat](#paidmessagefilenameformat), and [MessageFileNameFormat](#messagefilenameformat)).
|
||||||
For more information on the file name formats, see the [custom filename formats](/config/custom-filename-formats) page.
|
For more information on the file name formats, see the [custom filename formats](/config/custom-filename-formats) page.
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
|
|
||||||
```
|
```
|
||||||
"CreatorConfigs": {
|
"CreatorConfigs": {
|
||||||
"creator_one": {
|
"creator_one": {
|
||||||
@ -55,7 +62,8 @@ Default: `null`
|
|||||||
Allowed values: Any date in `yyyy-mm-dd` format or `null`
|
Allowed values: Any date in `yyyy-mm-dd` format or `null`
|
||||||
|
|
||||||
Description: [DownloadOnlySpecificDates](#downloadonlyspecificdates) needs to be set to `true` for this to work.
|
Description: [DownloadOnlySpecificDates](#downloadonlyspecificdates) needs to be set to `true` for this to work.
|
||||||
This date will be used when you are trying to download between/after a certain date. See [DownloadOnlySpecificDates](#downloadonlyspecificdates) and
|
This date will be used when you are trying to download between/after a certain date.
|
||||||
|
See [DownloadOnlySpecificDates](#downloadonlyspecificdates) and
|
||||||
[DownloadDateSelection](#downloaddateselection) for more information.
|
[DownloadDateSelection](#downloaddateselection) for more information.
|
||||||
|
|
||||||
## DisableBrowserAuth
|
## DisableBrowserAuth
|
||||||
@ -71,6 +79,16 @@ an `auth.json` file will need to be provided using a [legacy authentication meth
|
|||||||
If set to `true`, the `auth.json` file will not be deleted if authentication fails. If set to `false` (the default
|
If set to `true`, the `auth.json` file will not be deleted if authentication fails. If set to `false` (the default
|
||||||
behavior), OF-DL will delete the `auth.json` file if authentication fails.
|
behavior), OF-DL will delete the `auth.json` file if authentication fails.
|
||||||
|
|
||||||
|
## DisableTextSanitization
|
||||||
|
|
||||||
|
Type: `boolean`
|
||||||
|
|
||||||
|
Default: `false`
|
||||||
|
|
||||||
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
|
Description: When enabled, post/message text is stored as-is without XML stripping.
|
||||||
|
|
||||||
## DownloadArchived
|
## DownloadArchived
|
||||||
|
|
||||||
Type: `boolean`
|
Type: `boolean`
|
||||||
@ -109,7 +127,8 @@ Default: `"before"`
|
|||||||
|
|
||||||
Allowed values: `"before"`, `"after"`
|
Allowed values: `"before"`, `"after"`
|
||||||
|
|
||||||
Description: [DownloadOnlySpecificDates](#downloadonlyspecificdates) needs to be set to `true` for this to work. This will get all posts from before
|
Description: [DownloadOnlySpecificDates](#downloadonlyspecificdates) needs to be set to `true` for this to work. This
|
||||||
|
will get all posts from before
|
||||||
the date if set to `"before"`, and all posts from the date you specify up until the current date if set to `"after"`.
|
the date if set to `"before"`, and all posts from the date you specify up until the current date if set to `"after"`.
|
||||||
The date you specify will be in the [CustomDate](#customdate) config option.
|
The date you specify will be in the [CustomDate](#customdate) config option.
|
||||||
|
|
||||||
@ -121,7 +140,8 @@ Default: `false`
|
|||||||
|
|
||||||
Allowed values: `true`, `false`
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
Description: By default (or when set to `false`), the program will not download duplicated media. If set to `true`, duplicated media will be downloaded.
|
Description: By default (or when set to `false`), the program will not download duplicated media. If set to `true`,
|
||||||
|
duplicated media will be downloaded.
|
||||||
|
|
||||||
## DownloadHighlights
|
## DownloadHighlights
|
||||||
|
|
||||||
@ -151,7 +171,8 @@ Default: `4`
|
|||||||
|
|
||||||
Allowed values: Any positive integer
|
Allowed values: Any positive integer
|
||||||
|
|
||||||
Description: The download rate in MB per second. This will only be used if [LimitDownloadRate](#limitdownloadrate) is set to `true`.
|
Description: The download rate in MB per second. This will only be used if [LimitDownloadRate](#limitdownloadrate) is
|
||||||
|
set to `true`.
|
||||||
|
|
||||||
## DownloadMessages
|
## DownloadMessages
|
||||||
|
|
||||||
@ -171,7 +192,8 @@ Default: `false`
|
|||||||
|
|
||||||
Allowed values: `true`, `false`
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
Description: If set to `true`, posts will be downloaded based on the [DownloadDateSelection](#downloaddateselection) and [CustomDate](#customdate) config options.
|
Description: If set to `true`, posts will be downloaded based on the [DownloadDateSelection](#downloaddateselection)
|
||||||
|
and [CustomDate](#customdate) config options.
|
||||||
If set to `false`, all posts will be downloaded.
|
If set to `false`, all posts will be downloaded.
|
||||||
|
|
||||||
## DownloadPaidMessages
|
## DownloadPaidMessages
|
||||||
@ -228,8 +250,10 @@ Default: `false`
|
|||||||
|
|
||||||
Allowed values: `true`, `false`
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
Description: If set to `true`, only new posts will be downloaded from the date of the last post that was downloaded based off what's in the `user_data.db` file.
|
Description: If set to `true`, only new posts will be downloaded from the date of the last post that was downloaded
|
||||||
If set to `false`, the default behaviour will apply, and all posts will be gathered and compared against the database to see if they need to be downloaded or not.
|
based off what's in the `user_data.db` file.
|
||||||
|
If set to `false`, the default behaviour will apply, and all posts will be gathered and compared against the database to
|
||||||
|
see if they need to be downloaded or not.
|
||||||
|
|
||||||
## DownloadStories
|
## DownloadStories
|
||||||
|
|
||||||
@ -251,6 +275,17 @@ Allowed values: `true`, `false`
|
|||||||
|
|
||||||
Description: Posts in the "Streams" tab will be downloaded if set to `true`
|
Description: Posts in the "Streams" tab will be downloaded if set to `true`
|
||||||
|
|
||||||
|
## DownloadVideoResolution
|
||||||
|
|
||||||
|
Type: `string`
|
||||||
|
|
||||||
|
Default: `"source"`
|
||||||
|
|
||||||
|
Allowed values: `"source"`, `"240"`, `"720"`
|
||||||
|
|
||||||
|
Description: This allows you to download videos in alternative resolutions, by default videos are downloaded in source
|
||||||
|
resolution but some people prefer smoother videos at a lower resolution.
|
||||||
|
|
||||||
## DownloadVideos
|
## DownloadVideos
|
||||||
|
|
||||||
Type: `boolean`
|
Type: `boolean`
|
||||||
@ -261,6 +296,19 @@ Allowed values: `true`, `false`
|
|||||||
|
|
||||||
Description: Videos will be downloaded if set to `true`
|
Description: Videos will be downloaded if set to `true`
|
||||||
|
|
||||||
|
## DrmVideoDurationMatchThreshold
|
||||||
|
|
||||||
|
Type: `double`
|
||||||
|
|
||||||
|
Default: `0.98`
|
||||||
|
|
||||||
|
Allowed values: `0.01` to `1.0`
|
||||||
|
|
||||||
|
Description: Minimum required ratio between downloaded DRM video length and expected length.
|
||||||
|
Expected length is read from the MPD first, with media duration metadata used as a fallback.
|
||||||
|
For example, `0.98` requires the downloaded file to be at least 98% of the expected duration.
|
||||||
|
If the download is below this threshold, the program retries the download up to 3 times.
|
||||||
|
|
||||||
## FFmpegPath
|
## FFmpegPath
|
||||||
|
|
||||||
Type: `string`
|
Type: `string`
|
||||||
@ -270,14 +318,30 @@ Default: `""`
|
|||||||
Allowed values: Any valid path or `""`
|
Allowed values: Any valid path or `""`
|
||||||
|
|
||||||
Description: This is the path to the FFmpeg executable (`ffmpeg.exe` on Windows and `ffmpeg` on Linux/macOS).
|
Description: This is the path to the FFmpeg executable (`ffmpeg.exe` on Windows and `ffmpeg` on Linux/macOS).
|
||||||
If the path is not set then the program will try to find it in both the same directory as the OF-DL executable as well
|
If the path is not set, the program will try to find it in both the same directory as the OF-DL executable and the PATH
|
||||||
as the PATH environment variable.
|
environment variable.
|
||||||
|
|
||||||
!!! note
|
!!! note
|
||||||
|
|
||||||
If you are using a Windows path, you will need to escape the backslashes, e.g. `"C:\\ffmpeg\\bin\\ffmpeg.exe"`
|
If you are using a Windows path, you will need to escape the backslashes, e.g. `"C:\\ffmpeg\\bin\\ffmpeg.exe"`
|
||||||
For example, this is not valid: `"C:\some\path\ffmpeg.exe"`, but `"C:/some/path/ffmpeg.exe"` and `"C:\\some\\path\\ffmpeg.exe"` are both valid.
|
For example, this is not valid: `"C:\some\path\ffmpeg.exe"`, but `"C:/some/path/ffmpeg.exe"` and `"C:\\some\\path\\ffmpeg.exe"` are both valid.
|
||||||
|
|
||||||
|
## FFprobePath
|
||||||
|
|
||||||
|
Type: `string`
|
||||||
|
|
||||||
|
Default: `""`
|
||||||
|
|
||||||
|
Allowed values: Any valid path or `""`
|
||||||
|
|
||||||
|
Description: This is the path to the FFprobe executable (`ffprobe.exe` on Windows and `ffprobe` on Linux/macOS).
|
||||||
|
If the path is not set, the program will try to find it in both the same directory as the OF-DL executable and the PATH
|
||||||
|
environment variable.
|
||||||
|
|
||||||
|
!!! note
|
||||||
|
|
||||||
|
If you are using a Windows path, you will need to escape the backslashes, e.g. `"C:\\ffmpeg\\bin\\ffprobe.exe"`
|
||||||
|
|
||||||
## FolderPerMessage
|
## FolderPerMessage
|
||||||
|
|
||||||
Type: `boolean`
|
Type: `boolean`
|
||||||
@ -297,7 +361,8 @@ Default: `false`
|
|||||||
|
|
||||||
Allowed values: `true`, `false`
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
Description: A folder will be created for each paid message (containing all the media for that message) if set to `true`.
|
Description: A folder will be created for each paid message (containing all the media for that message) if set to
|
||||||
|
`true`.
|
||||||
When set to `false`, paid message media will be downloaded into the `Messages/Paid` folder.
|
When set to `false`, paid message media will be downloaded into the `Messages/Paid` folder.
|
||||||
|
|
||||||
## FolderPerPaidPost
|
## FolderPerPaidPost
|
||||||
@ -330,7 +395,9 @@ Default: `false`
|
|||||||
|
|
||||||
Allowed values: `true`, `false`
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
Description: By default (or when set to `false`), messages that were sent by yourself will be added to the metadata DB and any media which has been sent by yourself will be downloaded. If set to `true`, the program will not add messages sent by yourself to the metadata DB and will not download any media which has been sent by yourself.
|
Description: By default (or when set to `false`), messages that were sent by yourself will be added to the metadata DB
|
||||||
|
and any media which has been sent by yourself will be downloaded. If set to `true`, the program will not add messages
|
||||||
|
sent by yourself to the metadata DB and will not download any media which has been sent by yourself.
|
||||||
|
|
||||||
## IgnoredUsersListName
|
## IgnoredUsersListName
|
||||||
|
|
||||||
@ -361,7 +428,8 @@ Default: `false`
|
|||||||
|
|
||||||
Allowed values: `true`, `false`
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
Description: If set to `true`, media from restricted creators will be downloaded. If set to `false`, restricted creators will be ignored.
|
Description: If set to `true`, media from restricted creators will be downloaded. If set to `false`, restricted creators
|
||||||
|
will be ignored.
|
||||||
|
|
||||||
## LimitDownloadRate
|
## LimitDownloadRate
|
||||||
|
|
||||||
@ -371,7 +439,8 @@ Default: `false`
|
|||||||
|
|
||||||
Allowed values: `true`, `false`
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
Description: If set to `true`, the download rate will be limited to the value set in [DownloadLimitInMbPerSec](#downloadlimitinmbpersec).
|
Description: If set to `true`, the download rate will be limited to the value set
|
||||||
|
in [DownloadLimitInMbPerSec](#downloadlimitinmbpersec).
|
||||||
|
|
||||||
## LoggingLevel
|
## LoggingLevel
|
||||||
|
|
||||||
@ -392,7 +461,8 @@ Default: `""`
|
|||||||
|
|
||||||
Allowed values: Any valid string
|
Allowed values: Any valid string
|
||||||
|
|
||||||
Description: Please refer to [custom filename formats](/config/custom-filename-formats#messagefilenameformat) page to see what fields you can use.
|
Description: Please refer to [custom filename formats](/config/custom-filename-formats#messagefilenameformat) page to
|
||||||
|
see what fields you can use.
|
||||||
|
|
||||||
## NonInteractiveMode
|
## NonInteractiveMode
|
||||||
|
|
||||||
@ -402,8 +472,10 @@ Default: `false`
|
|||||||
|
|
||||||
Allowed values: `true`, `false`
|
Allowed values: `true`, `false`
|
||||||
|
|
||||||
Description: If set to `true`, the program will run without any input from the user. It will scrape all users automatically
|
Description: If set to `true`, the program will run without any input from the user. It will scrape all users
|
||||||
(unless [NonInteractiveModeListName](#noninteractivemodelistname) or [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) are configured).
|
automatically
|
||||||
|
(unless [NonInteractiveModeListName](#noninteractivemodelistname)
|
||||||
|
or [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) are configured).
|
||||||
If set to `false`, the default behaviour will apply, and you will be able to choose an option from the menu.
|
If set to `false`, the default behaviour will apply, and you will be able to choose an option from the menu.
|
||||||
|
|
||||||
!!! warning
|
!!! warning
|
||||||
@ -414,7 +486,6 @@ If set to `false`, the default behaviour will apply, and you will be able to cho
|
|||||||
1. Generate an auth.json file by running OF-DL with NonInteractiveMode disabled and authenticating OF-DL using the standard method **OR**
|
1. Generate an auth.json file by running OF-DL with NonInteractiveMode disabled and authenticating OF-DL using the standard method **OR**
|
||||||
2. Generate an auth.json file by using a [legacy authentication method](/config/auth#legacy-methods)
|
2. Generate an auth.json file by using a [legacy authentication method](/config/auth#legacy-methods)
|
||||||
|
|
||||||
|
|
||||||
## NonInteractiveModeListName
|
## NonInteractiveModeListName
|
||||||
|
|
||||||
Type: `string`
|
Type: `string`
|
||||||
@ -423,7 +494,8 @@ Default: `""`
|
|||||||
|
|
||||||
Allowed values: The name of a list of users you have created on OnlyFans or `""`
|
Allowed values: The name of a list of users you have created on OnlyFans or `""`
|
||||||
|
|
||||||
Description: When set to the name of a list, non-interactive mode will download media from the list of users instead of all
|
Description: When set to the name of a list, non-interactive mode will download media from the list of users instead of
|
||||||
|
all
|
||||||
users (when [NonInteractiveMode](#noninteractivemode) is set to `true`). If set to `""`, all users will be scraped
|
users (when [NonInteractiveMode](#noninteractivemode) is set to `true`). If set to `""`, all users will be scraped
|
||||||
(unless [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) is configured).
|
(unless [NonInteractiveModePurchasedTab](#noninteractivemodepurchasedtab) is configured).
|
||||||
|
|
||||||
@ -447,7 +519,8 @@ Default: `""`
|
|||||||
|
|
||||||
Allowed values: Any valid string
|
Allowed values: Any valid string
|
||||||
|
|
||||||
Description: Please refer to [custom filename formats](/config/custom-filename-formats#paidmessagefilenameformat) page to see what fields you can use.
|
Description: Please refer to [custom filename formats](/config/custom-filename-formats#paidmessagefilenameformat) page
|
||||||
|
to see what fields you can use.
|
||||||
|
|
||||||
## PaidPostFileNameFormat
|
## PaidPostFileNameFormat
|
||||||
|
|
||||||
@ -457,7 +530,8 @@ Default: `""`
|
|||||||
|
|
||||||
Allowed values: Any valid string
|
Allowed values: Any valid string
|
||||||
|
|
||||||
Description: Please refer to [custom filename formats](/config/custom-filename-formats#paidpostfilenameformat) page to see what fields you can use.
|
Description: Please refer to [custom filename formats](/config/custom-filename-formats#paidpostfilenameformat) page to
|
||||||
|
see what fields you can use.
|
||||||
|
|
||||||
## PostFileNameFormat
|
## PostFileNameFormat
|
||||||
|
|
||||||
@ -467,7 +541,8 @@ Default: `""`
|
|||||||
|
|
||||||
Allowed values: Any valid string
|
Allowed values: Any valid string
|
||||||
|
|
||||||
Description: Please refer to the [custom filename formats](/config/custom-filename-formats#postfilenameformat) page to see what fields you can use.
|
Description: Please refer to the [custom filename formats](/config/custom-filename-formats#postfilenameformat) page to
|
||||||
|
see what fields you can use.
|
||||||
|
|
||||||
## RenameExistingFilesWhenCustomFormatIsSelected
|
## RenameExistingFilesWhenCustomFormatIsSelected
|
||||||
|
|
||||||
@ -516,23 +591,3 @@ Allowed values: Any positive integer or `-1`
|
|||||||
|
|
||||||
Description: You won't need to set this, but if you see errors about the configured timeout of 100 seconds elapsing then
|
Description: You won't need to set this, but if you see errors about the configured timeout of 100 seconds elapsing then
|
||||||
you could set this to be more than 100. It is recommended that you leave this as the default value.
|
you could set this to be more than 100. It is recommended that you leave this as the default value.
|
||||||
|
|
||||||
## DisableTextSanitization
|
|
||||||
|
|
||||||
Type: `boolean`
|
|
||||||
|
|
||||||
Default: `false`
|
|
||||||
|
|
||||||
Allowed values: `true`, `false`
|
|
||||||
|
|
||||||
Description: When enabled, post/message text is stored as-is without XML stripping.
|
|
||||||
|
|
||||||
## DownloadVideoResolution
|
|
||||||
|
|
||||||
Type: `string`
|
|
||||||
|
|
||||||
Default: `"source"`
|
|
||||||
|
|
||||||
Allowed values: `"source"`, `"240"`, `"720"`
|
|
||||||
|
|
||||||
Description: This allows you to download videos in alternative resolutions, by default videos are downloaded in source resolution but some people prefer smoother videos at a lower resolution.
|
|
||||||
@ -8,6 +8,7 @@ information about what it does, its default value, and the allowed values.
|
|||||||
|
|
||||||
- External
|
- External
|
||||||
- [FFmpegPath](/config/all-configuration-options#ffmpegpath)
|
- [FFmpegPath](/config/all-configuration-options#ffmpegpath)
|
||||||
|
- [FFprobePath](/config/all-configuration-options#ffprobepath)
|
||||||
|
|
||||||
- Download
|
- Download
|
||||||
- [IgnoreOwnMessages](/config/all-configuration-options#ignoreownmessages)
|
- [IgnoreOwnMessages](/config/all-configuration-options#ignoreownmessages)
|
||||||
@ -22,6 +23,7 @@ information about what it does, its default value, and the allowed values.
|
|||||||
- [ShowScrapeSize](/config/all-configuration-options#showscrapesize)
|
- [ShowScrapeSize](/config/all-configuration-options#showscrapesize)
|
||||||
- [DisableTextSanitization](/config/all-configuration-options#disabletextsanitization)
|
- [DisableTextSanitization](/config/all-configuration-options#disabletextsanitization)
|
||||||
- [DownloadVideoResolution](/config/all-configuration-options#downloadvideoresolution)
|
- [DownloadVideoResolution](/config/all-configuration-options#downloadvideoresolution)
|
||||||
|
- [DrmVideoDurationMatchThreshold](/config/all-configuration-options#drmvideodurationmatchthreshold)
|
||||||
- Media
|
- Media
|
||||||
- [DownloadAvatarHeaderPhoto](/config/all-configuration-options#downloadavatarheaderphoto)
|
- [DownloadAvatarHeaderPhoto](/config/all-configuration-options#downloadavatarheaderphoto)
|
||||||
- [DownloadPaidPosts](/config/all-configuration-options#downloadpaidposts)
|
- [DownloadPaidPosts](/config/all-configuration-options#downloadpaidposts)
|
||||||
|
|||||||
@ -1,4 +1,4 @@
|
|||||||
# Linux
|
# Linux
|
||||||
|
|
||||||
A Linux release of OF-DL is not available at this time, however you can run OF-DL on Linux using Docker.
|
A Linux release of OF-DL is not available at this time, however you can run OF-DL on Linux using Docker.
|
||||||
Please refer to the [Docker](/installation/docker) page for instructions on how to run OF-DL in a Docker container.
|
Please refer to the [Docker](/installation/docker) page for instructions on how to run OF-DL in a Docker container.
|
||||||
@ -7,18 +7,17 @@ If you would like to run OF-DL natively on Linux, you can build it from source b
|
|||||||
|
|
||||||
## Building from source
|
## Building from source
|
||||||
|
|
||||||
- Install the libicu library
|
- Install FFmpeg (and FFprobe)
|
||||||
|
|
||||||
```bash
|
Follow the installtion instructions from FFmpeg ([https://ffmpeg.org/download.html](https://ffmpeg.org/download.html)) for your distro (Ubuntu, Debian, Fedora, etc.) to install FFmpeg and FFprobe
|
||||||
sudo apt-get install libicu-dev
|
|
||||||
```
|
|
||||||
|
|
||||||
- Install .NET version 8
|
!!! warning
|
||||||
|
|
||||||
```bash
|
Be sure to install FFmpeg version >= 6 and < 8. Other versions of FFmpeg may not decrypt DRM protected videos correctly.
|
||||||
wget https://dot.net/v1/dotnet-install.sh
|
|
||||||
sudo bash dotnet-install.sh --architecture x64 --install-dir /usr/share/dotnet/ --runtime dotnet --version 8.0.7
|
- Install .NET 10
|
||||||
```
|
|
||||||
|
Follow the installation instructions from Microsoft ([https://learn.microsoft.com/en-us/dotnet/core/install/linux](https://learn.microsoft.com/en-us/dotnet/core/install/linux)) for your distro (Ubuntu, Debian, Fedora, etc.) to install .NET 10.
|
||||||
|
|
||||||
- Clone the repo
|
- Clone the repo
|
||||||
|
|
||||||
@ -27,15 +26,16 @@ git clone https://git.ofdl.tools/sim0n00ps/OF-DL.git
|
|||||||
cd 'OF-DL'
|
cd 'OF-DL'
|
||||||
```
|
```
|
||||||
|
|
||||||
- Build the project. Replace `%VERSION%` with the current version number of OF-DL (e.g. `1.7.68`).
|
- Build the project. Replace `%VERSION%` with the current version number of OF-DL (e.g. `1.9.20`).
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
dotnet publish -p:Version=%VERSION% -c Release
|
dotnet publish "OF DL/OF DL.csproj" -p:Version=%VERSION% -p:PackageVersion=%VERSION% -c Release
|
||||||
cd 'OF DL/bin/Release/net8.0'
|
cd 'OF DL/bin/Release/net10.0'
|
||||||
```
|
```
|
||||||
|
|
||||||
- Download the windows release as described on [here](/installation/windows#installation).
|
- Download the windows release as described on [here](/installation/windows#installation).
|
||||||
- Add the `config.json` and `rules.json` files as well as the `cdm` folder to the `OF DL/bin/Release/net8.0` folder.
|
|
||||||
|
- Add the `config.conf` and `rules.json` files as well as the `cdm` folder to the `OF DL/bin/Release/net10.0` folder.
|
||||||
|
|
||||||
- Run the application
|
- Run the application
|
||||||
|
|
||||||
|
|||||||
@ -5,9 +5,9 @@
|
|||||||
### FFmpeg
|
### FFmpeg
|
||||||
|
|
||||||
You will need to download FFmpeg. You can download it from [here](https://www.gyan.dev/ffmpeg/builds/).
|
You will need to download FFmpeg. You can download it from [here](https://www.gyan.dev/ffmpeg/builds/).
|
||||||
Make sure you download `ffmpeg-release-essentials.zip`. Unzip it anywhere on your computer. You only need `ffmpeg.exe`, and you can ignore the rest.
|
Make sure you download `ffmpeg-release-essentials.zip`. Unzip it anywhere on your computer. You need both `ffmpeg.exe` and `ffprobe.exe`.
|
||||||
Move `ffmpeg.exe` to the same folder as `OF DL.exe` (downloaded in the installation steps below). If you choose to move `ffmpeg.exe` to a different folder,
|
Move `ffmpeg.exe` and `ffprobe.exe` to the same folder as `OF DL.exe` (downloaded in the installation steps below). If you choose to move them to a different folder,
|
||||||
you will need to specify the path to `ffmpeg.exe` in the config file (see the `FFmpegPath` [config option](/config/configuration#ffmpegpath)).
|
you will need to specify the paths in the config file (see the `FFmpegPath` and `FFprobePath` [config options](/config/configuration)).
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
@ -19,4 +19,5 @@ you will need to specify the path to `ffmpeg.exe` in the config file (see the `F
|
|||||||
- rules.json
|
- rules.json
|
||||||
- e_sqlite3.dll
|
- e_sqlite3.dll
|
||||||
- ffmpeg.exe
|
- ffmpeg.exe
|
||||||
|
- ffprobe.exe
|
||||||
4. Once you have done this, run OF DL.exe
|
4. Once you have done this, run OF DL.exe
|
||||||
|
|||||||
@ -4,7 +4,7 @@ Once you are happy you have filled everything in [auth.json](/config/auth) corre
|
|||||||
|
|
||||||

|

|
||||||
|
|
||||||
It should locate `config.json`, `rules.json` and FFmpeg successfully. If anything doesn't get located
|
It should locate `config.conf`, `rules.json`, FFmpeg, and FFprobe successfully. If anything doesn't get located
|
||||||
successfully, then make sure the files exist or the path is correct.
|
successfully, then make sure the files exist or the path is correct.
|
||||||
|
|
||||||
OF-DL will open a new window, if needed, to allow you to log into your OnlyFans account. The window will automatically close once
|
OF-DL will open a new window, if needed, to allow you to log into your OnlyFans account. The window will automatically close once
|
||||||
|
|||||||
2
excludes.txt
Normal file
2
excludes.txt
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
excludes.txt
|
||||||
|
rules.json
|
||||||
Loading…
x
Reference in New Issue
Block a user