Compare commits

..

5 Commits

Author SHA1 Message Date
beee158d65 Bump Nuget packages 2026-03-01 23:29:46 +00:00
ce624ba573
Added exponential backoff retry handling for OF API requests in ApiService and for non‑DRM downloads in DownloadService.
•
Introduced configurable retry constants in Constants.
2026-03-01 23:24:45 +00:00
1945a592d1 Update .gitea/workflows/publish-docs.yml 2026-03-01 22:35:10 +00:00
f950dc258f Merge pull request 'Remove $ from beginning of cookie variables' (#152) from whimsical-c4lic0/OF-DL:fix-invalid-auth-files into master
All checks were successful
Publish Docker image / Build and push Docker image to Gitea Registry (push) Successful in 21m7s
Publish release zip / build (push) Successful in 45s
Reviewed-on: #152
2026-02-27 18:44:03 +00:00
12d7d6793e Remove $ from beining of cookie variables 2026-02-27 12:40:41 -06:00
7 changed files with 229 additions and 69 deletions

View File

@ -4,10 +4,6 @@ on:
push:
tags:
- 'OFDLV*'
paths:
- 'docs/**'
- '.gitea/workflows/publish-docs.yml'
workflow_dispatch:
jobs:
build-and-deploy:

View File

@ -11,4 +11,16 @@ public static class Constants
public const int WidevineMaxRetries = 3;
public const int DrmDownloadMaxRetries = 3;
public const int ApiRetryMaxAttempts = 5;
public const int ApiRetryBaseDelayMs = 500;
public const int ApiRetryMaxDelayMs = 8000;
public const int DownloadRetryMaxAttempts = 4;
public const int DownloadRetryBaseDelayMs = 1000;
public const int DownloadRetryMaxDelayMs = 15000;
}

View File

@ -1,4 +1,7 @@
using System.Globalization;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Security.Cryptography;
using System.Text;
using System.Xml;
@ -177,10 +180,11 @@ public class ApiService(IAuthService authService, IConfigService configService,
{ "limit", Constants.ApiPageSize.ToString() }, { "order", "publish_date_asc" }
};
HttpClient client = new();
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
using HttpResponseMessage response = await client.SendAsync(request);
HttpClient client = GetHttpClient();
using HttpResponseMessage response = await SendWithRetryAsync(
() => BuildHttpRequestMessage(getParams, endpoint),
client,
$"GetUserInfo {endpoint}");
if (!response.IsSuccessStatusCode)
{
@ -216,10 +220,11 @@ public class ApiService(IAuthService authService, IConfigService configService,
try
{
HttpClient client = new();
HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
using HttpResponseMessage response = await client.SendAsync(request);
HttpClient client = GetHttpClient();
using HttpResponseMessage response = await SendWithRetryAsync(
() => BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint),
client,
$"GetUserInfoById {endpoint}");
response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync();
@ -2759,8 +2764,10 @@ public class ApiService(IAuthService authService, IConfigService configService,
{
Log.Debug("Calling BuildHeaderAndExecuteRequests");
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
using HttpResponseMessage response = await client.SendAsync(request);
using HttpResponseMessage response = await SendWithRetryAsync(
() => BuildHttpRequestMessage(getParams, endpoint),
client,
$"BuildHeaderAndExecuteRequests {endpoint}");
response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync();
@ -2769,6 +2776,60 @@ public class ApiService(IAuthService authService, IConfigService configService,
return body;
}
private static bool IsTransientStatusCode(HttpStatusCode statusCode) =>
statusCode == HttpStatusCode.RequestTimeout ||
statusCode == HttpStatusCode.TooManyRequests ||
(int)statusCode >= 500;
private static bool IsTransientException(Exception ex) =>
ex is HttpRequestException or TaskCanceledException or IOException or SocketException;
private static TimeSpan GetRetryDelay(int attempt, int baseDelayMs, int maxDelayMs)
{
double backoffMs = Math.Min(maxDelayMs, baseDelayMs * Math.Pow(2, attempt - 1));
double jitterFactor = 0.5 + Random.Shared.NextDouble();
return TimeSpan.FromMilliseconds(backoffMs * jitterFactor);
}
private async Task<HttpResponseMessage> SendWithRetryAsync(Func<Task<HttpRequestMessage>> requestFactory,
HttpClient client,
string operationName)
{
for (int attempt = 1; attempt <= Constants.ApiRetryMaxAttempts; attempt++)
{
using HttpRequestMessage request = await requestFactory();
try
{
HttpResponseMessage response = await client.SendAsync(request);
if (!IsTransientStatusCode(response.StatusCode) || attempt == Constants.ApiRetryMaxAttempts)
{
return response;
}
Log.Warning(
"Transient API response for {Operation}. StatusCode={StatusCode}. Attempt {Attempt}/{MaxAttempts}.",
operationName,
response.StatusCode,
attempt,
Constants.ApiRetryMaxAttempts);
response.Dispose();
}
catch (Exception ex) when (IsTransientException(ex) && attempt < Constants.ApiRetryMaxAttempts)
{
Log.Warning(ex,
"Transient API exception for {Operation}. Attempt {Attempt}/{MaxAttempts}.",
operationName,
attempt,
Constants.ApiRetryMaxAttempts);
}
TimeSpan delay = GetRetryDelay(attempt, Constants.ApiRetryBaseDelayMs, Constants.ApiRetryMaxDelayMs);
await Task.Delay(delay);
}
throw new Exception($"Retry attempts exhausted for {operationName}");
}
private Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
string endpoint)

View File

@ -314,7 +314,7 @@ public class AuthService(IServiceProvider serviceProvider) : IAuthService
}
string cookies = string.Join(" ", mappedCookies.Keys.Where(key => _desiredCookies.Contains(key))
.Select(key => $"${key}={mappedCookies[key]};"));
.Select(key => $"{key}={mappedCookies[key]};"));
await browser.CloseAsync();

View File

@ -1,5 +1,7 @@
using System.Diagnostics;
using System.Globalization;
using System.Net;
using System.Net.Sockets;
using System.Security.Cryptography;
using System.Text.RegularExpressions;
using FFmpeg.NET;
@ -82,12 +84,31 @@ public class DownloadService(
Uri uri = new(url);
string destinationPath = $"{folder}{subFolder}/";
HttpClient client = new();
for (int attempt = 1; attempt <= Constants.DownloadRetryMaxAttempts; attempt++)
{
try
{
using HttpClient client = new();
HttpRequestMessage request = new() { Method = HttpMethod.Get, RequestUri = uri };
using HttpResponseMessage response =
await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
if (!response.IsSuccessStatusCode)
{
if (IsTransientStatusCode(response.StatusCode) && attempt < Constants.DownloadRetryMaxAttempts)
{
Log.Warning(
"Transient response while downloading profile image. StatusCode={StatusCode}. Attempt {Attempt}/{MaxAttempts}.",
response.StatusCode,
attempt,
Constants.DownloadRetryMaxAttempts);
await Task.Delay(GetRetryDelay(attempt, Constants.DownloadRetryBaseDelayMs,
Constants.DownloadRetryMaxDelayMs));
continue;
}
response.EnsureSuccessStatusCode();
}
using MemoryStream memoryStream = new();
await response.Content.CopyToAsync(memoryStream);
@ -111,6 +132,19 @@ public class DownloadService(
File.SetLastWriteTime(destinationPath,
response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now);
}
return;
}
catch (Exception ex) when (IsTransientException(ex) && attempt < Constants.DownloadRetryMaxAttempts)
{
Log.Warning(ex,
"Transient exception while downloading profile image. Attempt {Attempt}/{MaxAttempts}.",
attempt,
Constants.DownloadRetryMaxAttempts);
await Task.Delay(GetRetryDelay(attempt, Constants.DownloadRetryBaseDelayMs,
Constants.DownloadRetryMaxDelayMs));
}
}
}
private async Task<bool> DownloadDrmMedia(
@ -931,12 +965,39 @@ public class DownloadService(
/// <param name="progressReporter"></param>
/// <returns>A Task resulting in a DateTime indicating the last modified date of the downloaded file.</returns>
private async Task<DateTime> DownloadFile(string url, string destinationPath, IProgressReporter progressReporter)
{
for (int attempt = 1; attempt <= Constants.DownloadRetryMaxAttempts; attempt++)
{
if (attempt > 1 && File.Exists(destinationPath))
{
TryDeleteFile(destinationPath);
}
try
{
using HttpClient client = new();
HttpRequestMessage request = new() { Method = HttpMethod.Get, RequestUri = new Uri(url) };
using HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
using HttpResponseMessage response =
await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
if (!response.IsSuccessStatusCode)
{
if (IsTransientStatusCode(response.StatusCode) && attempt < Constants.DownloadRetryMaxAttempts)
{
Log.Warning(
"Transient response while downloading media. StatusCode={StatusCode}. Attempt {Attempt}/{MaxAttempts}.",
response.StatusCode,
attempt,
Constants.DownloadRetryMaxAttempts);
await Task.Delay(GetRetryDelay(attempt, Constants.DownloadRetryBaseDelayMs,
Constants.DownloadRetryMaxDelayMs));
continue;
}
response.EnsureSuccessStatusCode();
}
Stream body = await response.Content.ReadAsStreamAsync();
// Wrap the body stream with the ThrottledStream to limit read rate.
@ -944,7 +1005,8 @@ public class DownloadService(
configService.CurrentConfig.DownloadLimitInMbPerSec * 1_000_000,
configService.CurrentConfig.LimitDownloadRate))
{
await using FileStream fileStream = new(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None,
await using FileStream fileStream = new(destinationPath, FileMode.Create, FileAccess.Write,
FileShare.None,
16384,
true);
byte[] buffer = new byte[16384];
@ -960,7 +1022,8 @@ public class DownloadService(
}
}
File.SetLastWriteTime(destinationPath, response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now);
File.SetLastWriteTime(destinationPath,
response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now);
if (!configService.CurrentConfig.ShowScrapeSize)
{
progressReporter.ReportProgress(1);
@ -968,6 +1031,34 @@ public class DownloadService(
return response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now;
}
catch (Exception ex) when (IsTransientException(ex) && attempt < Constants.DownloadRetryMaxAttempts)
{
Log.Warning(ex,
"Transient exception while downloading media. Attempt {Attempt}/{MaxAttempts}.",
attempt,
Constants.DownloadRetryMaxAttempts);
await Task.Delay(GetRetryDelay(attempt, Constants.DownloadRetryBaseDelayMs,
Constants.DownloadRetryMaxDelayMs));
}
}
throw new Exception($"Retry attempts exhausted while downloading {url}");
}
private static bool IsTransientStatusCode(HttpStatusCode statusCode) =>
statusCode == HttpStatusCode.RequestTimeout ||
statusCode == HttpStatusCode.TooManyRequests ||
(int)statusCode >= 500;
private static bool IsTransientException(Exception ex) =>
ex is HttpRequestException or TaskCanceledException or IOException or SocketException;
private static TimeSpan GetRetryDelay(int attempt, int baseDelayMs, int maxDelayMs)
{
double backoffMs = Math.Min(maxDelayMs, baseDelayMs * Math.Pow(2, attempt - 1));
double jitterFactor = 0.5 + Random.Shared.NextDouble();
return TimeSpan.FromMilliseconds(backoffMs * jitterFactor);
}
/// <summary>
/// Calculates the total size of a set of URLs by fetching their metadata.

View File

@ -9,11 +9,11 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="coverlet.collector" Version="6.0.4">
<PackageReference Include="coverlet.collector" Version="8.0.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1"/>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.3.0" />
<PackageReference Include="xunit" Version="2.9.3"/>
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5">
<PrivateAssets>all</PrivateAssets>

View File

@ -18,7 +18,7 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="Akka" Version="1.5.60"/>
<PackageReference Include="Akka" Version="1.5.61" />
<PackageReference Include="BouncyCastle.NetCore" Version="2.2.1"/>
<PackageReference Include="HtmlAgilityPack" Version="1.12.4"/>
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.3"/>