Compare commits
5 Commits
OFDLV1.10.
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| beee158d65 | |||
| ce624ba573 | |||
| 1945a592d1 | |||
| f950dc258f | |||
| 12d7d6793e |
@ -4,10 +4,6 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- 'OFDLV*'
|
||||
paths:
|
||||
- 'docs/**'
|
||||
- '.gitea/workflows/publish-docs.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build-and-deploy:
|
||||
|
||||
@ -11,4 +11,16 @@ public static class Constants
|
||||
public const int WidevineMaxRetries = 3;
|
||||
|
||||
public const int DrmDownloadMaxRetries = 3;
|
||||
|
||||
public const int ApiRetryMaxAttempts = 5;
|
||||
|
||||
public const int ApiRetryBaseDelayMs = 500;
|
||||
|
||||
public const int ApiRetryMaxDelayMs = 8000;
|
||||
|
||||
public const int DownloadRetryMaxAttempts = 4;
|
||||
|
||||
public const int DownloadRetryBaseDelayMs = 1000;
|
||||
|
||||
public const int DownloadRetryMaxDelayMs = 15000;
|
||||
}
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Xml;
|
||||
@ -177,10 +180,11 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
||||
{ "limit", Constants.ApiPageSize.ToString() }, { "order", "publish_date_asc" }
|
||||
};
|
||||
|
||||
HttpClient client = new();
|
||||
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
|
||||
|
||||
using HttpResponseMessage response = await client.SendAsync(request);
|
||||
HttpClient client = GetHttpClient();
|
||||
using HttpResponseMessage response = await SendWithRetryAsync(
|
||||
() => BuildHttpRequestMessage(getParams, endpoint),
|
||||
client,
|
||||
$"GetUserInfo {endpoint}");
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
@ -216,10 +220,11 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
||||
|
||||
try
|
||||
{
|
||||
HttpClient client = new();
|
||||
HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
|
||||
|
||||
using HttpResponseMessage response = await client.SendAsync(request);
|
||||
HttpClient client = GetHttpClient();
|
||||
using HttpResponseMessage response = await SendWithRetryAsync(
|
||||
() => BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint),
|
||||
client,
|
||||
$"GetUserInfoById {endpoint}");
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
string body = await response.Content.ReadAsStringAsync();
|
||||
@ -2759,8 +2764,10 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
||||
{
|
||||
Log.Debug("Calling BuildHeaderAndExecuteRequests");
|
||||
|
||||
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
|
||||
using HttpResponseMessage response = await client.SendAsync(request);
|
||||
using HttpResponseMessage response = await SendWithRetryAsync(
|
||||
() => BuildHttpRequestMessage(getParams, endpoint),
|
||||
client,
|
||||
$"BuildHeaderAndExecuteRequests {endpoint}");
|
||||
response.EnsureSuccessStatusCode();
|
||||
string body = await response.Content.ReadAsStringAsync();
|
||||
|
||||
@ -2769,6 +2776,60 @@ public class ApiService(IAuthService authService, IConfigService configService,
|
||||
return body;
|
||||
}
|
||||
|
||||
private static bool IsTransientStatusCode(HttpStatusCode statusCode) =>
|
||||
statusCode == HttpStatusCode.RequestTimeout ||
|
||||
statusCode == HttpStatusCode.TooManyRequests ||
|
||||
(int)statusCode >= 500;
|
||||
|
||||
private static bool IsTransientException(Exception ex) =>
|
||||
ex is HttpRequestException or TaskCanceledException or IOException or SocketException;
|
||||
|
||||
private static TimeSpan GetRetryDelay(int attempt, int baseDelayMs, int maxDelayMs)
|
||||
{
|
||||
double backoffMs = Math.Min(maxDelayMs, baseDelayMs * Math.Pow(2, attempt - 1));
|
||||
double jitterFactor = 0.5 + Random.Shared.NextDouble();
|
||||
return TimeSpan.FromMilliseconds(backoffMs * jitterFactor);
|
||||
}
|
||||
|
||||
private async Task<HttpResponseMessage> SendWithRetryAsync(Func<Task<HttpRequestMessage>> requestFactory,
|
||||
HttpClient client,
|
||||
string operationName)
|
||||
{
|
||||
for (int attempt = 1; attempt <= Constants.ApiRetryMaxAttempts; attempt++)
|
||||
{
|
||||
using HttpRequestMessage request = await requestFactory();
|
||||
try
|
||||
{
|
||||
HttpResponseMessage response = await client.SendAsync(request);
|
||||
if (!IsTransientStatusCode(response.StatusCode) || attempt == Constants.ApiRetryMaxAttempts)
|
||||
{
|
||||
return response;
|
||||
}
|
||||
|
||||
Log.Warning(
|
||||
"Transient API response for {Operation}. StatusCode={StatusCode}. Attempt {Attempt}/{MaxAttempts}.",
|
||||
operationName,
|
||||
response.StatusCode,
|
||||
attempt,
|
||||
Constants.ApiRetryMaxAttempts);
|
||||
response.Dispose();
|
||||
}
|
||||
catch (Exception ex) when (IsTransientException(ex) && attempt < Constants.ApiRetryMaxAttempts)
|
||||
{
|
||||
Log.Warning(ex,
|
||||
"Transient API exception for {Operation}. Attempt {Attempt}/{MaxAttempts}.",
|
||||
operationName,
|
||||
attempt,
|
||||
Constants.ApiRetryMaxAttempts);
|
||||
}
|
||||
|
||||
TimeSpan delay = GetRetryDelay(attempt, Constants.ApiRetryBaseDelayMs, Constants.ApiRetryMaxDelayMs);
|
||||
await Task.Delay(delay);
|
||||
}
|
||||
|
||||
throw new Exception($"Retry attempts exhausted for {operationName}");
|
||||
}
|
||||
|
||||
|
||||
private Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
|
||||
string endpoint)
|
||||
|
||||
@ -314,7 +314,7 @@ public class AuthService(IServiceProvider serviceProvider) : IAuthService
|
||||
}
|
||||
|
||||
string cookies = string.Join(" ", mappedCookies.Keys.Where(key => _desiredCookies.Contains(key))
|
||||
.Select(key => $"${key}={mappedCookies[key]};"));
|
||||
.Select(key => $"{key}={mappedCookies[key]};"));
|
||||
|
||||
await browser.CloseAsync();
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.RegularExpressions;
|
||||
using FFmpeg.NET;
|
||||
@ -82,12 +84,31 @@ public class DownloadService(
|
||||
Uri uri = new(url);
|
||||
string destinationPath = $"{folder}{subFolder}/";
|
||||
|
||||
HttpClient client = new();
|
||||
|
||||
for (int attempt = 1; attempt <= Constants.DownloadRetryMaxAttempts; attempt++)
|
||||
{
|
||||
try
|
||||
{
|
||||
using HttpClient client = new();
|
||||
HttpRequestMessage request = new() { Method = HttpMethod.Get, RequestUri = uri };
|
||||
using HttpResponseMessage response =
|
||||
await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
if (IsTransientStatusCode(response.StatusCode) && attempt < Constants.DownloadRetryMaxAttempts)
|
||||
{
|
||||
Log.Warning(
|
||||
"Transient response while downloading profile image. StatusCode={StatusCode}. Attempt {Attempt}/{MaxAttempts}.",
|
||||
response.StatusCode,
|
||||
attempt,
|
||||
Constants.DownloadRetryMaxAttempts);
|
||||
await Task.Delay(GetRetryDelay(attempt, Constants.DownloadRetryBaseDelayMs,
|
||||
Constants.DownloadRetryMaxDelayMs));
|
||||
continue;
|
||||
}
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
using MemoryStream memoryStream = new();
|
||||
await response.Content.CopyToAsync(memoryStream);
|
||||
@ -111,6 +132,19 @@ public class DownloadService(
|
||||
File.SetLastWriteTime(destinationPath,
|
||||
response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now);
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
catch (Exception ex) when (IsTransientException(ex) && attempt < Constants.DownloadRetryMaxAttempts)
|
||||
{
|
||||
Log.Warning(ex,
|
||||
"Transient exception while downloading profile image. Attempt {Attempt}/{MaxAttempts}.",
|
||||
attempt,
|
||||
Constants.DownloadRetryMaxAttempts);
|
||||
await Task.Delay(GetRetryDelay(attempt, Constants.DownloadRetryBaseDelayMs,
|
||||
Constants.DownloadRetryMaxDelayMs));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> DownloadDrmMedia(
|
||||
@ -931,12 +965,39 @@ public class DownloadService(
|
||||
/// <param name="progressReporter"></param>
|
||||
/// <returns>A Task resulting in a DateTime indicating the last modified date of the downloaded file.</returns>
|
||||
private async Task<DateTime> DownloadFile(string url, string destinationPath, IProgressReporter progressReporter)
|
||||
{
|
||||
for (int attempt = 1; attempt <= Constants.DownloadRetryMaxAttempts; attempt++)
|
||||
{
|
||||
if (attempt > 1 && File.Exists(destinationPath))
|
||||
{
|
||||
TryDeleteFile(destinationPath);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
using HttpClient client = new();
|
||||
HttpRequestMessage request = new() { Method = HttpMethod.Get, RequestUri = new Uri(url) };
|
||||
|
||||
using HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
|
||||
using HttpResponseMessage response =
|
||||
await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
if (IsTransientStatusCode(response.StatusCode) && attempt < Constants.DownloadRetryMaxAttempts)
|
||||
{
|
||||
Log.Warning(
|
||||
"Transient response while downloading media. StatusCode={StatusCode}. Attempt {Attempt}/{MaxAttempts}.",
|
||||
response.StatusCode,
|
||||
attempt,
|
||||
Constants.DownloadRetryMaxAttempts);
|
||||
await Task.Delay(GetRetryDelay(attempt, Constants.DownloadRetryBaseDelayMs,
|
||||
Constants.DownloadRetryMaxDelayMs));
|
||||
continue;
|
||||
}
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
|
||||
Stream body = await response.Content.ReadAsStreamAsync();
|
||||
|
||||
// Wrap the body stream with the ThrottledStream to limit read rate.
|
||||
@ -944,7 +1005,8 @@ public class DownloadService(
|
||||
configService.CurrentConfig.DownloadLimitInMbPerSec * 1_000_000,
|
||||
configService.CurrentConfig.LimitDownloadRate))
|
||||
{
|
||||
await using FileStream fileStream = new(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None,
|
||||
await using FileStream fileStream = new(destinationPath, FileMode.Create, FileAccess.Write,
|
||||
FileShare.None,
|
||||
16384,
|
||||
true);
|
||||
byte[] buffer = new byte[16384];
|
||||
@ -960,7 +1022,8 @@ public class DownloadService(
|
||||
}
|
||||
}
|
||||
|
||||
File.SetLastWriteTime(destinationPath, response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now);
|
||||
File.SetLastWriteTime(destinationPath,
|
||||
response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now);
|
||||
if (!configService.CurrentConfig.ShowScrapeSize)
|
||||
{
|
||||
progressReporter.ReportProgress(1);
|
||||
@ -968,6 +1031,34 @@ public class DownloadService(
|
||||
|
||||
return response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now;
|
||||
}
|
||||
catch (Exception ex) when (IsTransientException(ex) && attempt < Constants.DownloadRetryMaxAttempts)
|
||||
{
|
||||
Log.Warning(ex,
|
||||
"Transient exception while downloading media. Attempt {Attempt}/{MaxAttempts}.",
|
||||
attempt,
|
||||
Constants.DownloadRetryMaxAttempts);
|
||||
await Task.Delay(GetRetryDelay(attempt, Constants.DownloadRetryBaseDelayMs,
|
||||
Constants.DownloadRetryMaxDelayMs));
|
||||
}
|
||||
}
|
||||
|
||||
throw new Exception($"Retry attempts exhausted while downloading {url}");
|
||||
}
|
||||
|
||||
private static bool IsTransientStatusCode(HttpStatusCode statusCode) =>
|
||||
statusCode == HttpStatusCode.RequestTimeout ||
|
||||
statusCode == HttpStatusCode.TooManyRequests ||
|
||||
(int)statusCode >= 500;
|
||||
|
||||
private static bool IsTransientException(Exception ex) =>
|
||||
ex is HttpRequestException or TaskCanceledException or IOException or SocketException;
|
||||
|
||||
private static TimeSpan GetRetryDelay(int attempt, int baseDelayMs, int maxDelayMs)
|
||||
{
|
||||
double backoffMs = Math.Min(maxDelayMs, baseDelayMs * Math.Pow(2, attempt - 1));
|
||||
double jitterFactor = 0.5 + Random.Shared.NextDouble();
|
||||
return TimeSpan.FromMilliseconds(backoffMs * jitterFactor);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates the total size of a set of URLs by fetching their metadata.
|
||||
|
||||
@ -9,11 +9,11 @@
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="coverlet.collector" Version="6.0.4">
|
||||
<PackageReference Include="coverlet.collector" Version="8.0.0">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
|
||||
</PackageReference>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.0.1"/>
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="18.3.0" />
|
||||
<PackageReference Include="xunit" Version="2.9.3"/>
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.5">
|
||||
<PrivateAssets>all</PrivateAssets>
|
||||
|
||||
@ -18,7 +18,7 @@
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Akka" Version="1.5.60"/>
|
||||
<PackageReference Include="Akka" Version="1.5.61" />
|
||||
<PackageReference Include="BouncyCastle.NetCore" Version="2.2.1"/>
|
||||
<PackageReference Include="HtmlAgilityPack" Version="1.12.4"/>
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.3"/>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user