sim0n00ps/OF-DL
32
35
Fork 9
Code Issues 20 Pull Requests 2 Actions Packages Projects Releases 6 Wiki Activity

WIP: Replace PuppeteerSharp with Playwright #44

Draft
whimsical-c4lic0 wants to merge 1 commits from whimsical-c4lic0/OF-DL:replace-puppeteer-with-playwright into master
pull from: whimsical-c4lic0/OF-DL:replace-puppeteer-with-playwright
merge into: sim0n00ps:master
Conversation 0 Commits 1 Files Changed 5 +79 -66
5 changed files with 79 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitea/workflows/publish-release.yml
View File

@ -57,7 +57,7 @@ jobs:
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
echo "➤ Creating release zip" echo "➤ Creating release zip"
zip ../OFDLV${{ steps.version.outputs.version }}.zip OF\ DL.exe e_sqlite3.dll rules.json config.conf cdm ffmpeg.exe LICENSE.ffmpeg zip ../OFDLV${{ steps.version.outputs.version }}.zip OF\ DL.exe e_sqlite3.dll rules.json config.conf cdm chromium-scripts ffmpeg.exe LICENSE.ffmpeg
cd .. cd ..
- name: Create release and upload artifact - name: Create release and upload artifact

27
Dockerfile
View File

@ -1,10 +1,7 @@
FROM alpine:3.20 AS build FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
ARG VERSION ARG VERSION
RUN apk --no-cache --repository community add \
dotnet8-sdk
# Copy source code # Copy source code
COPY ["OF DL.sln", "/src/OF DL.sln"] COPY ["OF DL.sln", "/src/OF DL.sln"]
COPY ["OF DL", "/src/OF DL"] COPY ["OF DL", "/src/OF DL"]
@ -21,21 +18,24 @@ RUN /src/out/OF\ DL --non-interactive || true && \
mv /src/updated_config.conf /src/config.conf mv /src/updated_config.conf /src/config.conf
FROM alpine:3.20 AS final FROM mcr.microsoft.com/dotnet/runtime:8.0 AS final
# Install dependencies # Install dependencies
RUN apk --no-cache --repository community add \ RUN apt-get update \
bash \ && apt-get install -y \
tini \ tini \
dotnet8-runtime \
ffmpeg \ ffmpeg \
udev \
ttf-freefont \
chromium \
supervisor \ supervisor \
xvfb \ xvfb \
x11vnc \ x11vnc \
novnc novnc \
npm \
&& rm -rf /var/lib/apt/lists/*
RUN npx playwright install --with-deps chromium
RUN apt-get remove --purge -y npm \
&& apt-get autoremove -y
# Redirect webroot to vnc.html instead of displaying directory listing # Redirect webroot to vnc.html instead of displaying directory listing
RUN echo "<!DOCTYPE html><html><head><meta http-equiv=\"Refresh\" content=\"0; url='vnc.html'\" /></head><body></body></html>" > /usr/share/novnc/index.html RUN echo "<!DOCTYPE html><html><head><meta http-equiv=\"Refresh\" content=\"0; url='vnc.html'\" /></head><body></body></html>" > /usr/share/novnc/index.html
@ -57,10 +57,9 @@ RUN chmod +x /app/entrypoint.sh
ENV DISPLAY=:0.0 \ ENV DISPLAY=:0.0 \
DISPLAY_WIDTH=1024 \ DISPLAY_WIDTH=1024 \
DISPLAY_HEIGHT=768 \ DISPLAY_HEIGHT=768 \
OFDL_PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium \
OFDL_DOCKER=true OFDL_DOCKER=true
EXPOSE 8080 EXPOSE 8080
WORKDIR /config WORKDIR /config
ENTRYPOINT ["/sbin/tini", "--"] ENTRYPOINT ["/usr/bin/tini", "--"]
CMD ["/app/entrypoint.sh"] CMD ["/app/entrypoint.sh"]

102
OF DL/Helpers/AuthHelper.cs
View File

@ -1,19 +1,20 @@
using OF_DL.Entities; using OF_DL.Entities;
using PuppeteerSharp;
using PuppeteerSharp.BrowserData;
using Serilog; using Serilog;
using Microsoft.Playwright;
namespace OF_DL.Helpers; namespace OF_DL.Helpers;
public class AuthHelper public class AuthHelper
{ {
private readonly LaunchOptions _options = new() private readonly string _userDataDir = Path.GetFullPath("chromium-data");
private const string _initScriptsDirName = "chromium-scripts";
private readonly BrowserTypeLaunchPersistentContextOptions _options = new()
{ {
Headless = false, Headless = false,
Channel = ChromeReleaseChannel.Stable, Channel = "chromium",
DefaultViewport = null, Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-blink-features=AutomationControlled", "--disable-infobars"],
Args = ["--no-sandbox", "--disable-setuid-sandbox"],
UserDataDir = Path.GetFullPath("chrome-data")
}; };
private readonly string[] _desiredCookies = private readonly string[] _desiredCookies =
@ -22,63 +23,49 @@ public class AuthHelper
"sess" "sess"
]; ];
private const int LoginTimeout = 600000; // 10 minutes private const float LoginTimeout = 600000f; // 10 minutes
private const int FeedLoadTimeout = 60000; // 1 minute private const float FeedLoadTimeout = 60000f; // 1 minute
public async Task SetupBrowser(bool runningInDocker) public async Task SetupBrowser(bool runningInDocker)
{ {
string? executablePath = Environment.GetEnvironmentVariable("OFDL_PUPPETEER_EXECUTABLE_PATH");
if (executablePath != null)
{
Log.Information("OFDL_PUPPETEER_EXECUTABLE_PATH environment variable found. Using browser executable path: {executablePath}", executablePath);
_options.ExecutablePath = executablePath;
}
else
{
var browserFetcher = new BrowserFetcher();
var installedBrowsers = browserFetcher.GetInstalledBrowsers().ToList();
if (installedBrowsers.Count == 0)
{
Log.Information("Downloading browser.");
var downloadedBrowser = await browserFetcher.DownloadAsync();
Log.Information("Browser downloaded. Path: {executablePath}",
downloadedBrowser.GetExecutablePath());
_options.ExecutablePath = downloadedBrowser.GetExecutablePath();
}
else
{
_options.ExecutablePath = installedBrowsers.First().GetExecutablePath();
}
}
if (runningInDocker) if (runningInDocker)
{ {
Log.Information("Running in Docker. Disabling sandbox and GPU."); Log.Information("Running in Docker. Disabling sandbox and GPU.");
_options.Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-gpu"]; _options.Args = ["--no-sandbox", "--disable-setuid-sandbox", "--disable-gpu", "--disable-blink-features=AutomationControlled", "--disable-infobars"];
}
else
{
var exitCode = Microsoft.Playwright.Program.Main(new[] {"install", "--with-deps", "chromium"});
if (exitCode != 0)
{
throw new Exception($"Playwright chromium download failed. Exited with code {exitCode}");
}
} }
} }
private async Task<string> GetBcToken(IPage page) private async Task<string> GetBcToken(IPage page)
{ {
return await page.EvaluateExpressionAsync<string>("window.localStorage.getItem('bcTokenSha') || ''"); return await page.EvaluateAsync<string>("window.localStorage.getItem('bcTokenSha') || ''");
} }
public async Task<Auth?> GetAuthFromBrowser(bool isDocker = false) public async Task<Auth?> GetAuthFromBrowser(bool isDocker = false)
{ {
try try
{ {
IBrowser? browser; IBrowserContext? browser = null;
try try
{ {
browser = await Puppeteer.LaunchAsync(_options); var playwright = await Playwright.CreateAsync();
browser = await playwright.Chromium.LaunchPersistentContextAsync(_userDataDir, _options);
} }
catch (ProcessException e) catch (Exception e)
{ {
if (e.Message.Contains("Failed to launch browser") && Directory.Exists(_options.UserDataDir)) if (e.Message.Contains("An error occurred trying to start process") && Directory.Exists(_userDataDir))
{ {
Log.Error("Failed to launch browser. Deleting chrome-data directory and trying again."); Log.Error("Failed to launch browser. Deleting chrome-data directory and trying again.");
Directory.Delete(_options.UserDataDir, true); Directory.Delete(_userDataDir, true);
browser = await Puppeteer.LaunchAsync(_options); IPlaywright playwright = await Playwright.CreateAsync();
browser = await playwright.Chromium.LaunchPersistentContextAsync(_userDataDir, _options);
} }
else else
{ {
@ -91,25 +78,36 @@ public class AuthHelper
throw new Exception("Could not get browser"); throw new Exception("Could not get browser");
} }
IPage[]? pages = await browser.PagesAsync(); IPage? page = browser.Pages[0];
IPage? page = pages.First();
if (page == null) if (page == null)
{ {
throw new Exception("Could not get page"); throw new Exception("Could not get page");
} }
string exeDirectory = Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location) ?? string.Empty;
string initScriptsDir = Path.Combine(exeDirectory, _initScriptsDirName);
if (Directory.Exists(initScriptsDir))
{
Log.Information("Loading init scripts from {initScriptsDir}", initScriptsDir);
foreach (string initScript in Directory.GetFiles(initScriptsDir, "*.js"))
{
Log.Debug("Loading init script {initScript}", initScript);
await page.AddInitScriptAsync(initScript);
}
}
Log.Debug("Navigating to OnlyFans."); Log.Debug("Navigating to OnlyFans.");
await page.GoToAsync("https://onlyfans.com"); await page.GotoAsync("https://onlyfans.com");
Log.Debug("Waiting for user to login"); Log.Debug("Waiting for user to login");
await page.WaitForSelectorAsync(".b-feed", new WaitForSelectorOptions { Timeout = LoginTimeout }); await page.WaitForSelectorAsync(".b-feed", new PageWaitForSelectorOptions { Timeout = LoginTimeout });
Log.Debug("Feed element detected (user logged in)"); Log.Debug("Feed element detected (user logged in)");
await page.ReloadAsync(); await page.ReloadAsync();
await page.WaitForNavigationAsync(new NavigationOptions { await page.WaitForNavigationAsync(new PageWaitForNavigationOptions {
WaitUntil = [WaitUntilNavigation.Networkidle2], WaitUntil = WaitUntilState.DOMContentLoaded,
Timeout = FeedLoadTimeout Timeout = FeedLoadTimeout
}); });
Log.Debug("DOM loaded. Getting BC token and cookies ..."); Log.Debug("DOM loaded. Getting BC token and cookies ...");
@ -121,34 +119,40 @@ public class AuthHelper
} }
catch (Exception e) catch (Exception e)
{ {
await browser.CloseAsync();
throw new Exception("Error getting bcToken"); throw new Exception("Error getting bcToken");
} }
Dictionary<string, string> mappedCookies = (await page.GetCookiesAsync()) Dictionary<string, string> mappedCookies = (await browser.CookiesAsync())
.Where(cookie => cookie.Domain.Contains("onlyfans.com")) .Where(cookie => cookie.Domain.Contains("onlyfans.com"))
.ToDictionary(cookie => cookie.Name, cookie => cookie.Value); .ToDictionary(cookie => cookie.Name, cookie => cookie.Value);
mappedCookies.TryGetValue("auth_id", out string? userId); mappedCookies.TryGetValue("auth_id", out string? userId);
if (userId == null) if (userId == null)
{ {
await browser.CloseAsync();
throw new Exception("Could not find 'auth_id' cookie"); throw new Exception("Could not find 'auth_id' cookie");
} }
mappedCookies.TryGetValue("sess", out string? sess); mappedCookies.TryGetValue("sess", out string? sess);
if (sess == null) if (sess == null)
{ {
await browser.CloseAsync();
throw new Exception("Could not find 'sess' cookie"); throw new Exception("Could not find 'sess' cookie");
} }
string? userAgent = await browser.GetUserAgentAsync(); string? userAgent = await page.EvaluateAsync<string>("navigator.userAgent");
if (userAgent == null) if (userAgent == null)
{ {
await browser.CloseAsync();
throw new Exception("Could not get user agent"); throw new Exception("Could not get user agent");
} }
string cookies = string.Join(" ", mappedCookies.Keys.Where(key => _desiredCookies.Contains(key)) string cookies = string.Join(" ", mappedCookies.Keys.Where(key => _desiredCookies.Contains(key))
.Select(key => $"${key}={mappedCookies[key]};")); .Select(key => $"${key}={mappedCookies[key]};"));
await browser.CloseAsync();
return new Auth() return new Auth()
{ {
COOKIE = cookies, COOKIE = cookies,

5
OF DL/OF DL.csproj
View File

@ -18,9 +18,9 @@
<PackageReference Include="BouncyCastle.NetCore" Version="2.2.1" /> <PackageReference Include="BouncyCastle.NetCore" Version="2.2.1" />
<PackageReference Include="HtmlAgilityPack" Version="1.12.0" /> <PackageReference Include="HtmlAgilityPack" Version="1.12.0" />
<PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.3" /> <PackageReference Include="Microsoft.Data.Sqlite" Version="9.0.3" />
<PackageReference Include="Microsoft.Playwright" Version="1.54.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" /> <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="protobuf-net" Version="3.2.46" /> <PackageReference Include="protobuf-net" Version="3.2.46" />
<PackageReference Include="PuppeteerSharp" Version="20.1.3" />
<PackageReference Include="Serilog" Version="4.2.0" /> <PackageReference Include="Serilog" Version="4.2.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" /> <PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" /> <PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
@ -44,6 +44,9 @@
<None Update="rules.json"> <None Update="rules.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None> </None>
<None Update="chromium-scripts/stealth.min.js">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
</ItemGroup> </ItemGroup>
</Project> </Project>

7
OF DL/chromium-scripts/stealth.min.js vendored Normal file
View File

File diff suppressed because one or more lines are too long