forked from sim0n00ps/OF-DL
Compare commits
No commits in common. "fix-purchased-media-obj-defref" and "master" have entirely different histories.
fix-purcha
...
master
@ -52,20 +52,9 @@ jobs:
|
||||
echo "➤ Creating folder for CDM"
|
||||
mkdir -p cdm/devices/chrome_1610
|
||||
|
||||
echo "➤ Copying ffmpeg"
|
||||
if [ -f /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe ] && [ -f /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE ]; then
|
||||
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
|
||||
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
|
||||
else
|
||||
echo "➤ Downloading ffmpeg package"
|
||||
mkdir -p /tmp/ffmpeg
|
||||
if [ ! -f /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build.zip ]; then
|
||||
curl -L https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-7.1.1-essentials_build.zip -o /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build.zip
|
||||
fi
|
||||
unzip -o /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build.zip -d /tmp/ffmpeg
|
||||
cp /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
|
||||
cp /tmp/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
|
||||
fi
|
||||
echo "➤ Copying ffmpeg from user folder"
|
||||
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/bin/ffmpeg.exe .
|
||||
cp /home/rhys/ffmpeg/ffmpeg-7.1.1-essentials_build/LICENSE LICENSE.ffmpeg
|
||||
|
||||
echo "➤ Creating release zip"
|
||||
zip ../OFDLV${{ steps.version.outputs.version }}.zip OF\ DL.exe e_sqlite3.dll rules.json config.conf cdm ffmpeg.exe LICENSE.ffmpeg
|
||||
@ -81,4 +70,4 @@ jobs:
|
||||
env:
|
||||
GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
GITEA_REPOSITORY: ${{ gitea.repository }}
|
||||
GITEA_SERVER_URL: ${{ gitea.server_url }}
|
||||
GITEA_SERVER_URL: ${{ gitea.server_url }}
|
||||
@ -1,6 +1,6 @@
|
||||
FROM alpine:3.23 AS build
|
||||
FROM alpine:3.20 AS build
|
||||
|
||||
ARG VERSION=dev
|
||||
ARG VERSION
|
||||
|
||||
RUN apk --no-cache --repository community add \
|
||||
dotnet8-sdk
|
||||
@ -21,14 +21,14 @@ RUN /src/out/OF\ DL --non-interactive || true && \
|
||||
mv /src/updated_config.conf /src/config.conf
|
||||
|
||||
|
||||
FROM alpine:3.23 AS final
|
||||
FROM alpine:3.20 AS final
|
||||
|
||||
# Install dependencies
|
||||
RUN apk --no-cache --repository community add \
|
||||
bash \
|
||||
tini \
|
||||
dotnet8-runtime \
|
||||
ffmpeg7 \
|
||||
ffmpeg \
|
||||
udev \
|
||||
ttf-freefont \
|
||||
chromium \
|
||||
|
||||
@ -57,13 +57,6 @@ namespace OF_DL.Entities
|
||||
[ToggleableConfig]
|
||||
public bool LimitDownloadRate { get; set; } = false;
|
||||
public int DownloadLimitInMbPerSec { get; set; } = 4;
|
||||
public int ApiRateLimitRequests { get; set; } = 10;
|
||||
public int ApiRateLimitWindowSeconds { get; set; } = 10;
|
||||
public int MaxLogBodyLength { get; set; } = 2000;
|
||||
public bool EnableJsonResponseBodyRawLogs { get; set; } = false;
|
||||
[ToggleableConfig]
|
||||
public bool EnableFfmpegReport { get; set; } = true;
|
||||
public string FfmpegReportPath { get; set; } = Path.Combine("logs", "diagnostics", "ffmpeg");
|
||||
|
||||
// Indicates if you want to download only on specific dates.
|
||||
[ToggleableConfig]
|
||||
|
||||
@ -51,15 +51,6 @@ namespace OF_DL.Entities
|
||||
public LoggingLevel LoggingLevel { get; set; }
|
||||
|
||||
bool IgnoreOwnMessages { get; set; }
|
||||
|
||||
int ApiRateLimitRequests { get; set; }
|
||||
int ApiRateLimitWindowSeconds { get; set; }
|
||||
|
||||
int MaxLogBodyLength { get; set; }
|
||||
bool EnableJsonResponseBodyRawLogs { get; set; }
|
||||
|
||||
bool EnableFfmpegReport { get; set; }
|
||||
string FfmpegReportPath { get; set; }
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -33,16 +33,6 @@ public class APIHelper : IAPIHelper
|
||||
private static DynamicRules? cachedDynamicRules;
|
||||
private const int MaxAttempts = 30;
|
||||
private const int DelayBetweenAttempts = 3000;
|
||||
private static int ApiRateLimitCount = 10;
|
||||
private static TimeSpan ApiRateLimitWindow = TimeSpan.FromSeconds(10);
|
||||
private static readonly SemaphoreSlim ApiRateLimiter = new(1, 1);
|
||||
private static readonly Queue<DateTime> ApiRequestTimes = new();
|
||||
private static readonly ILogger DiagnosticLog = new LoggerConfiguration()
|
||||
.MinimumLevel.Debug()
|
||||
.WriteTo.File("logs/purchasedtab-diagnostics.txt", rollingInterval: RollingInterval.Day, retainedFileCountLimit: 7)
|
||||
.CreateLogger();
|
||||
private readonly int maxLogBodyLength;
|
||||
private readonly bool enableRawJsonBodyLogs;
|
||||
|
||||
static APIHelper()
|
||||
{
|
||||
@ -57,19 +47,6 @@ public class APIHelper : IAPIHelper
|
||||
this.auth = auth;
|
||||
m_DBHelper = new DBHelper(downloadConfig);
|
||||
this.downloadConfig = downloadConfig;
|
||||
maxLogBodyLength = downloadConfig.MaxLogBodyLength > 0 ? downloadConfig.MaxLogBodyLength : 2000;
|
||||
enableRawJsonBodyLogs = downloadConfig.EnableJsonResponseBodyRawLogs;
|
||||
if (downloadConfig.ApiRateLimitRequests > 0)
|
||||
{
|
||||
ApiRateLimitCount = downloadConfig.ApiRateLimitRequests;
|
||||
}
|
||||
if (downloadConfig.ApiRateLimitWindowSeconds > 0)
|
||||
{
|
||||
ApiRateLimitWindow = TimeSpan.FromSeconds(downloadConfig.ApiRateLimitWindowSeconds);
|
||||
}
|
||||
|
||||
Log.Debug("DiagnosticsEnabled={DiagnosticsEnabled} MaxLogBodyLength={MaxLogBodyLength} EnableRawJson={EnableRawJson} RawLogDir={RawDir}",
|
||||
DiagnosticsEnabled, maxLogBodyLength, enableRawJsonBodyLogs, Path.Combine("logs", "diagnostics", "raw"));
|
||||
}
|
||||
|
||||
|
||||
@ -141,29 +118,16 @@ public class APIHelper : IAPIHelper
|
||||
}
|
||||
|
||||
|
||||
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, ILogger? diagnosticLogger = null, string? rawLabel = null)
|
||||
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
|
||||
{
|
||||
Log.Debug("Calling BuildHeaderAndExecuteRequests");
|
||||
diagnosticLogger ??= CurrentDiagnosticLogger;
|
||||
|
||||
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
|
||||
diagnosticLogger?.Information("Diag request | method={Method} url={Url} headers={Headers}", request.Method, request.RequestUri, FlattenHeaders(request.Headers));
|
||||
|
||||
await EnforceApiRateLimitAsync(diagnosticLogger, request.RequestUri?.ToString());
|
||||
using var response = await client.SendAsync(request);
|
||||
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
||||
{
|
||||
string respBody = await response.Content.ReadAsStringAsync();
|
||||
diagnosticLogger?.Warning("API 429 | url={Url} headers={Headers} respHeaders={RespHeaders} bodyPreview={Body}", request.RequestUri, FlattenHeaders(request.Headers), string.Join("; ", response.Headers.Select(h => $"{h.Key}={string.Join(",", h.Value)}")), TruncateForLog(respBody));
|
||||
Log.Warning("API 429 | url={Url}", request.RequestUri);
|
||||
}
|
||||
diagnosticLogger?.Information("Diag response | status={StatusCode} reason={ReasonPhrase} requestUrl={Url}", response.StatusCode, response.ReasonPhrase, request.RequestUri);
|
||||
response.EnsureSuccessStatusCode();
|
||||
string body = await response.Content.ReadAsStringAsync();
|
||||
|
||||
Log.Debug(body);
|
||||
diagnosticLogger?.Information("Diag response body (truncated) | length={Length} body={Body}", body?.Length ?? 0, TruncateForLog(body));
|
||||
WriteRawBody(rawLabel ?? request.RequestUri?.ToString() ?? endpoint, body);
|
||||
|
||||
return body;
|
||||
}
|
||||
@ -213,120 +177,6 @@ public class APIHelper : IAPIHelper
|
||||
return client;
|
||||
}
|
||||
|
||||
private static string FlattenHeaders(System.Net.Http.Headers.HttpRequestHeaders headers)
|
||||
{
|
||||
return string.Join("; ", headers.Select(h => $"{h.Key}={string.Join(",", h.Value)}"));
|
||||
}
|
||||
|
||||
private bool DiagnosticsEnabled => downloadConfig.LoggingLevel == LoggingLevel.Verbose;
|
||||
|
||||
private ILogger? CurrentDiagnosticLogger => DiagnosticsEnabled ? DiagnosticLog : null;
|
||||
|
||||
private static string TruncateString(string? value, int maxLength = 1024)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
return value.Length <= maxLength ? value : $"{value[..maxLength]}...[truncated]";
|
||||
}
|
||||
|
||||
private string TruncateForLog(string? value, int? overrideMax = null)
|
||||
{
|
||||
int max = overrideMax.HasValue ? Math.Min(overrideMax.Value, maxLogBodyLength) : maxLogBodyLength;
|
||||
return TruncateString(value, max);
|
||||
}
|
||||
|
||||
private void DiagInfo(string messageTemplate, params object[] propertyValues)
|
||||
{
|
||||
if (!DiagnosticsEnabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
DiagnosticLog.Information(messageTemplate, propertyValues);
|
||||
}
|
||||
|
||||
private void DiagWarning(string messageTemplate, params object[] propertyValues)
|
||||
{
|
||||
if (!DiagnosticsEnabled)
|
||||
{
|
||||
return;
|
||||
}
|
||||
DiagnosticLog.Warning(messageTemplate, propertyValues);
|
||||
}
|
||||
|
||||
private string SanitizeForFile(string input)
|
||||
{
|
||||
var invalids = Path.GetInvalidFileNameChars();
|
||||
var safe = string.Concat(input.Select(ch => invalids.Contains(ch) ? '_' : ch));
|
||||
return string.IsNullOrWhiteSpace(safe) ? "response" : safe;
|
||||
}
|
||||
|
||||
private void WriteRawBody(string? label, string? body)
|
||||
{
|
||||
if (!enableRawJsonBodyLogs || string.IsNullOrEmpty(body))
|
||||
{
|
||||
return;
|
||||
}
|
||||
try
|
||||
{
|
||||
string dir = Path.Combine("logs", "diagnostics", "raw");
|
||||
Directory.CreateDirectory(dir);
|
||||
string namePart = !string.IsNullOrEmpty(label) ? SanitizeForFile(label) : "response";
|
||||
string fileName = $"{DateTime.UtcNow:yyyyMMddTHHmmssfff}-{namePart}.json";
|
||||
string fullPath = Path.Combine(dir, fileName);
|
||||
File.WriteAllText(fullPath, body);
|
||||
Log.Debug("Wrote raw body log to {Path}", fullPath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log.Debug("Failed to write raw body log: {Message}", ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task EnforceApiRateLimitAsync(ILogger? diagnosticLogger, string? context = null)
|
||||
{
|
||||
if (ApiRateLimitCount <= 0 || ApiRateLimitWindow.TotalSeconds <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
while (true)
|
||||
{
|
||||
await ApiRateLimiter.WaitAsync();
|
||||
double waitSeconds = 0;
|
||||
try
|
||||
{
|
||||
var now = DateTime.UtcNow;
|
||||
while (ApiRequestTimes.Count > 0 && (now - ApiRequestTimes.Peek()) > ApiRateLimitWindow)
|
||||
{
|
||||
ApiRequestTimes.Dequeue();
|
||||
}
|
||||
|
||||
if (ApiRequestTimes.Count < ApiRateLimitCount)
|
||||
{
|
||||
ApiRequestTimes.Enqueue(now);
|
||||
return;
|
||||
}
|
||||
|
||||
waitSeconds = (ApiRateLimitWindow - (now - ApiRequestTimes.Peek())).TotalSeconds;
|
||||
}
|
||||
finally
|
||||
{
|
||||
ApiRateLimiter.Release();
|
||||
}
|
||||
|
||||
if (waitSeconds < 0.1)
|
||||
{
|
||||
waitSeconds = 0.1;
|
||||
}
|
||||
|
||||
diagnosticLogger?.Information("API rate limit sleep | waitSeconds={WaitSeconds} currentCount={CurrentCount} context={Context}", waitSeconds, ApiRequestTimes.Count, context ?? string.Empty);
|
||||
Log.Information("API rate limit sleep | waitSeconds={WaitSeconds} currentCount={CurrentCount} context={Context}", waitSeconds, ApiRequestTimes.Count, context ?? string.Empty);
|
||||
await Task.Delay(TimeSpan.FromSeconds(waitSeconds));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// this one is used during initialization only
|
||||
@ -382,7 +232,6 @@ public class APIHelper : IAPIHelper
|
||||
HttpClient client = new();
|
||||
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
|
||||
|
||||
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
|
||||
using var response = await client.SendAsync(request);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
@ -392,7 +241,6 @@ public class APIHelper : IAPIHelper
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
WriteRawBody($"user-{endpoint.Replace("/", "_")}", body);
|
||||
user = JsonConvert.DeserializeObject<Entities.User>(body, m_JsonSerializerSettings);
|
||||
return user;
|
||||
}
|
||||
@ -410,24 +258,22 @@ public class APIHelper : IAPIHelper
|
||||
return null;
|
||||
}
|
||||
|
||||
public async Task<JObject> GetUserInfoById(string endpoint)
|
||||
public async Task<JObject> GetUserInfoById(string endpoint)
|
||||
{
|
||||
try
|
||||
{
|
||||
try
|
||||
{
|
||||
HttpClient client = new();
|
||||
HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
|
||||
HttpClient client = new();
|
||||
HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
|
||||
|
||||
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
|
||||
using var response = await client.SendAsync(request);
|
||||
using var response = await client.SendAsync(request);
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
WriteRawBody($"user-list-{endpoint.Replace("/", "_")}", body);
|
||||
response.EnsureSuccessStatusCode();
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
|
||||
//if the content creator doesnt exist, we get a 200 response, but the content isnt usable
|
||||
//so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
|
||||
//which means we wont get loads of exceptions
|
||||
if (body.Equals("[]"))
|
||||
//if the content creator doesnt exist, we get a 200 response, but the content isnt usable
|
||||
//so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
|
||||
//which means we wont get loads of exceptions
|
||||
if (body.Equals("[]"))
|
||||
return null;
|
||||
|
||||
JObject jObject = JObject.Parse(body);
|
||||
@ -779,10 +625,10 @@ public class APIHelper : IAPIHelper
|
||||
{
|
||||
Highlights newhighlights = new();
|
||||
|
||||
Log.Debug("Media Highlights - " + endpoint);
|
||||
Log.Debug("Media Highlights - " + endpoint);
|
||||
|
||||
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
||||
newhighlights = JsonConvert.DeserializeObject<Highlights>(loopbody, m_JsonSerializerSettings);
|
||||
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
||||
newhighlights = JsonConvert.DeserializeObject<Highlights>(loopbody, m_JsonSerializerSettings);
|
||||
|
||||
highlights.list.AddRange(newhighlights.list);
|
||||
if (!newhighlights.hasMore)
|
||||
@ -815,11 +661,9 @@ public class APIHelper : IAPIHelper
|
||||
highlight_request.Headers.Add(keyValuePair.Key, keyValuePair.Value);
|
||||
}
|
||||
|
||||
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, highlight_request.RequestUri?.ToString());
|
||||
using var highlightResponse = await highlight_client.SendAsync(highlight_request);
|
||||
highlightResponse.EnsureSuccessStatusCode();
|
||||
var highlightBody = await highlightResponse.Content.ReadAsStringAsync();
|
||||
WriteRawBody($"highlight-{highlight_id}", highlightBody);
|
||||
highlightMedia = JsonConvert.DeserializeObject<HighlightMedia>(highlightBody, m_JsonSerializerSettings);
|
||||
if (highlightMedia != null)
|
||||
{
|
||||
@ -2008,7 +1852,7 @@ public class APIHelper : IAPIHelper
|
||||
{ "skip_users", "all" }
|
||||
};
|
||||
|
||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-initial-{username}");
|
||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
||||
paidMessages = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
|
||||
ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.list.Count}[/]");
|
||||
ctx.Spinner(Spinner.Known.Dots);
|
||||
@ -2018,10 +1862,21 @@ public class APIHelper : IAPIHelper
|
||||
getParams["offset"] = paidMessages.list.Count.ToString();
|
||||
while (true)
|
||||
{
|
||||
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
||||
Purchased newpaidMessages = new();
|
||||
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-page-{getParams["offset"]}-{username}");
|
||||
if (!string.IsNullOrEmpty(loopbody))
|
||||
Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
|
||||
HttpClient loopclient = GetHttpClient(config);
|
||||
|
||||
HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
|
||||
|
||||
foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
|
||||
{
|
||||
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
|
||||
}
|
||||
using (var loopresponse = await loopclient.SendAsync(looprequest))
|
||||
{
|
||||
loopresponse.EnsureSuccessStatusCode();
|
||||
var loopbody = await loopresponse.Content.ReadAsStringAsync();
|
||||
newpaidMessages = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
|
||||
}
|
||||
paidMessages.list.AddRange(newpaidMessages.list);
|
||||
@ -2229,18 +2084,28 @@ public class APIHelper : IAPIHelper
|
||||
{ "skip_users", "all" }
|
||||
};
|
||||
|
||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, "posts-paid-all-initial");
|
||||
WriteRawBody("posts-paid-all-initial", body);
|
||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
||||
purchased = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
|
||||
if (purchased != null && purchased.hasMore)
|
||||
{
|
||||
getParams["offset"] = purchased.list.Count.ToString();
|
||||
while (true)
|
||||
{
|
||||
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
||||
Purchased newPurchased = new();
|
||||
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"posts-paid-all-page-{getParams["offset"]}");
|
||||
if (!string.IsNullOrEmpty(loopbody))
|
||||
Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
|
||||
HttpClient loopclient = GetHttpClient(config);
|
||||
|
||||
HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
|
||||
|
||||
foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
|
||||
{
|
||||
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
|
||||
}
|
||||
using (var loopresponse = await loopclient.SendAsync(looprequest))
|
||||
{
|
||||
loopresponse.EnsureSuccessStatusCode();
|
||||
var loopbody = await loopresponse.Content.ReadAsStringAsync();
|
||||
newPurchased = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
|
||||
}
|
||||
purchased.list.AddRange(newPurchased.list);
|
||||
@ -2379,8 +2244,6 @@ public class APIHelper : IAPIHelper
|
||||
{
|
||||
Log.Debug($"Calling GetPurchasedTab - {endpoint}");
|
||||
|
||||
string diagContext = $"purchasedtab-{DateTime.UtcNow:yyyyMMddTHHmmssfff}-{Guid.NewGuid():N}";
|
||||
|
||||
try
|
||||
{
|
||||
Dictionary<long, List<Purchased.List>> userPurchases = new Dictionary<long, List<Purchased.List>>();
|
||||
@ -2395,30 +2258,29 @@ public class APIHelper : IAPIHelper
|
||||
{ "skip_users", "all" }
|
||||
};
|
||||
|
||||
string initialQueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
||||
var initialHeaders = GetDynamicHeaders("/api2/v2" + endpoint, initialQueryParams);
|
||||
DiagInfo("PurchasedTab start | ctx={Context} endpoint={Endpoint} folder={Folder} usersCount={UsersCount} limit={Limit} query={Query} headers={Headers}", diagContext, endpoint, folder, users?.Count ?? 0, post_limit, initialQueryParams, string.Join(", ", initialHeaders.Select(kvp => $"{kvp.Key}={kvp.Value}")));
|
||||
|
||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, diagContext);
|
||||
DiagInfo("PurchasedTab initial response | ctx={Context} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, body?.Length ?? 0, TruncateForLog(body));
|
||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
||||
purchased = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
|
||||
DiagInfo("PurchasedTab parsed initial | ctx={Context} items={Items} hasMore={HasMore}", diagContext, purchased?.list?.Count ?? 0, purchased?.hasMore ?? false);
|
||||
if (purchased == null || purchased.list == null)
|
||||
{
|
||||
DiagWarning("PurchasedTab parsed initial null list | ctx={Context} bodyPreview={BodyPreview}", diagContext, TruncateForLog(body));
|
||||
}
|
||||
if (purchased != null && purchased.hasMore)
|
||||
{
|
||||
getParams["offset"] = purchased.list.Count.ToString();
|
||||
while (true)
|
||||
{
|
||||
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
||||
Purchased newPurchased = new();
|
||||
DiagInfo("PurchasedTab page request | ctx={Context} offset={Offset} limit={Limit}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit);
|
||||
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"{diagContext}-page-{getParams["offset"]}");
|
||||
if (!string.IsNullOrEmpty(loopbody))
|
||||
Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
|
||||
HttpClient loopclient = GetHttpClient(config);
|
||||
|
||||
HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
|
||||
|
||||
foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
|
||||
{
|
||||
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
|
||||
}
|
||||
using (var loopresponse = await loopclient.SendAsync(looprequest))
|
||||
{
|
||||
loopresponse.EnsureSuccessStatusCode();
|
||||
var loopbody = await loopresponse.Content.ReadAsStringAsync();
|
||||
newPurchased = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
|
||||
DiagInfo("PurchasedTab page parsed | ctx={Context} offset={Offset} items={Items} hasMore={HasMore} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", newPurchased?.list?.Count ?? 0, newPurchased?.hasMore ?? false, loopbody?.Length ?? 0, TruncateForLog(loopbody));
|
||||
}
|
||||
purchased.list.AddRange(newPurchased.list);
|
||||
if (!newPurchased.hasMore)
|
||||
@ -2456,17 +2318,6 @@ public class APIHelper : IAPIHelper
|
||||
{
|
||||
PurchasedTabCollection purchasedTabCollection = new PurchasedTabCollection();
|
||||
JObject userObject = await GetUserInfoById($"/users/list?x[]={user.Key}");
|
||||
string resolvedUsername = $"Deleted User - {user.Key}";
|
||||
if (userObject == null || userObject[user.Key.ToString()] == null || userObject[user.Key.ToString()]["username"] == null)
|
||||
{
|
||||
DiagWarning("PurchasedTab user lookup missing username | ctx={Context} userId={UserId} rawUser={RawUser}", diagContext, user.Key, userObject != null ? TruncateForLog(userObject.ToString()) : "null");
|
||||
}
|
||||
else
|
||||
{
|
||||
resolvedUsername = userObject[user.Key.ToString()]["username"].ToString();
|
||||
DiagInfo("PurchasedTab user lookup success | ctx={Context} userId={UserId} username={Username} rawUser={RawUser}", diagContext, user.Key, resolvedUsername, TruncateForLog(userObject.ToString()));
|
||||
}
|
||||
WriteRawBody($"{diagContext}-user-{user.Key}", userObject?.ToString());
|
||||
purchasedTabCollection.UserId = user.Key;
|
||||
purchasedTabCollection.Username = userObject is not null && !string.IsNullOrEmpty(userObject[user.Key.ToString()]["username"].ToString()) ? userObject[user.Key.ToString()]["username"].ToString() : $"Deleted User - {user.Key}";
|
||||
string path = System.IO.Path.Combine(folder, purchasedTabCollection.Username);
|
||||
@ -2474,16 +2325,6 @@ public class APIHelper : IAPIHelper
|
||||
{
|
||||
foreach (Purchased.List purchase in user.Value)
|
||||
{
|
||||
DiagInfo("PurchasedTab processing purchase | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} mediaNull={MediaNull} mediaCount={MediaCount} postedAt={PostedAt} createdAt={CreatedAt} fromUserId={FromUserId} authorId={AuthorId}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.media == null, purchase.media?.Count ?? 0, purchase.postedAt, purchase.createdAt, purchase.fromUser?.id, purchase.author?.id);
|
||||
DiagInfo("PurchasedTab purchase raw | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} rawPurchase={RawPurchase}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, TruncateForLog(JsonConvert.SerializeObject(purchase)));
|
||||
WriteRawBody($"{diagContext}-purchase-{purchase.id}", JsonConvert.SerializeObject(purchase));
|
||||
if (purchase.media == null)
|
||||
{
|
||||
string textPreview = TruncateForLog(purchase.text, 100);
|
||||
DiagWarning("PurchasedTab purchase media null, setting empty list | ctx={Context} userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt} textPreview={TextPreview}", diagContext, user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.createdAt, purchase.postedAt, textPreview);
|
||||
Log.Warning("PurchasedTab purchase media null, setting empty list | userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt} textPreview={TextPreview}", user.Key, resolvedUsername, purchase.id, purchase.responseType, purchase.createdAt, purchase.postedAt, textPreview);
|
||||
purchase.media = new List<Messages.Medium>();
|
||||
}
|
||||
switch (purchase.responseType)
|
||||
{
|
||||
case "post":
|
||||
@ -2767,7 +2608,6 @@ public class APIHelper : IAPIHelper
|
||||
request.Headers.Add("user-agent", auth.USER_AGENT);
|
||||
request.Headers.Add("Accept", "*/*");
|
||||
request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.COOKIE};");
|
||||
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
|
||||
using (var response = await client.SendAsync(request))
|
||||
{
|
||||
response.EnsureSuccessStatusCode();
|
||||
@ -2813,7 +2653,6 @@ public class APIHelper : IAPIHelper
|
||||
request.Headers.Add("user-agent", auth.USER_AGENT);
|
||||
request.Headers.Add("Accept", "*/*");
|
||||
request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {auth.COOKIE};");
|
||||
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
|
||||
using (var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead))
|
||||
{
|
||||
response.EnsureSuccessStatusCode();
|
||||
@ -2870,7 +2709,6 @@ public class APIHelper : IAPIHelper
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json")
|
||||
};
|
||||
|
||||
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
|
||||
using var response = await client.SendAsync(request);
|
||||
|
||||
Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}");
|
||||
@ -2950,7 +2788,6 @@ public class APIHelper : IAPIHelper
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json")
|
||||
};
|
||||
|
||||
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
|
||||
using var response = await client.SendAsync(request);
|
||||
|
||||
if (!response.IsSuccessStatusCode)
|
||||
|
||||
@ -632,11 +632,8 @@ public class DownloadHelper : IDownloadHelper
|
||||
|
||||
// Configure ffmpeg log level and optional report file location
|
||||
bool ffmpegDebugLogging = Log.IsEnabled(Serilog.Events.LogEventLevel.Debug);
|
||||
bool enableReport = downloadConfig.EnableFfmpegReport;
|
||||
|
||||
bool includeReport = enableReport && (ffmpegDebugLogging || downloadConfig.LoggingLevel is LoggingLevel.Verbose or LoggingLevel.Debug);
|
||||
|
||||
string logLevelArgs = includeReport
|
||||
string logLevelArgs = ffmpegDebugLogging || downloadConfig.LoggingLevel is LoggingLevel.Verbose or LoggingLevel.Debug
|
||||
? "-loglevel debug -report"
|
||||
: downloadConfig.LoggingLevel switch
|
||||
{
|
||||
@ -647,12 +644,10 @@ public class DownloadHelper : IDownloadHelper
|
||||
_ => string.Empty
|
||||
};
|
||||
|
||||
if (includeReport)
|
||||
if (logLevelArgs.Contains("-report", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
string reportDir = string.IsNullOrWhiteSpace(downloadConfig.FfmpegReportPath)
|
||||
? Path.Combine("logs", "diagnostics", "ffmpeg")
|
||||
: downloadConfig.FfmpegReportPath;
|
||||
string logDir = Path.GetFullPath(reportDir);
|
||||
// Direct ffmpeg report files into the same logs directory Serilog uses (relative to current working directory)
|
||||
string logDir = Path.GetFullPath(Path.Combine(Environment.CurrentDirectory, "logs"));
|
||||
Directory.CreateDirectory(logDir);
|
||||
string ffReportPath = Path.Combine(logDir, "ffmpeg-%p-%t.log"); // ffmpeg will replace %p/%t
|
||||
Environment.SetEnvironmentVariable("FFREPORT", $"file={ffReportPath}:level=32");
|
||||
|
||||
@ -35,32 +35,6 @@ public class Program
|
||||
private static Auth? auth = null;
|
||||
private static LoggingLevelSwitch levelSwitch = new LoggingLevelSwitch();
|
||||
|
||||
private static int SafeGetInt(Akka.Configuration.Config hoconConfig, string path, int defaultValue)
|
||||
{
|
||||
try
|
||||
{
|
||||
var value = hoconConfig.GetString(path);
|
||||
return string.IsNullOrWhiteSpace(value) ? defaultValue : hoconConfig.GetInt(path);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool SafeGetBool(Akka.Configuration.Config hoconConfig, string path, bool defaultValue)
|
||||
{
|
||||
try
|
||||
{
|
||||
var value = hoconConfig.GetString(path);
|
||||
return string.IsNullOrWhiteSpace(value) ? defaultValue : hoconConfig.GetBoolean(path);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task LoadAuthFromBrowser()
|
||||
{
|
||||
bool runningInDocker = Environment.GetEnvironmentVariable("OFDL_DOCKER") != null;
|
||||
@ -245,15 +219,11 @@ public class Program
|
||||
hoconConfig.AppendLine($" Timeout = {(jsonConfig.Timeout.HasValue ? jsonConfig.Timeout.Value : -1)}");
|
||||
hoconConfig.AppendLine($" LimitDownloadRate = {jsonConfig.LimitDownloadRate.ToString().ToLower()}");
|
||||
hoconConfig.AppendLine($" DownloadLimitInMbPerSec = {jsonConfig.DownloadLimitInMbPerSec}");
|
||||
hoconConfig.AppendLine($" ApiRateLimitRequests = {jsonConfig.ApiRateLimitRequests}");
|
||||
hoconConfig.AppendLine($" ApiRateLimitWindowSeconds = {jsonConfig.ApiRateLimitWindowSeconds}");
|
||||
hoconConfig.AppendLine("}");
|
||||
|
||||
hoconConfig.AppendLine("# Logging/Debug Settings");
|
||||
hoconConfig.AppendLine("Logging {");
|
||||
hoconConfig.AppendLine($" LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\"");
|
||||
hoconConfig.AppendLine($" MaxLogBodyLength = {jsonConfig.MaxLogBodyLength}");
|
||||
hoconConfig.AppendLine($" EnableJsonResponseBodyRawLogs = {jsonConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
|
||||
hoconConfig.AppendLine("}");
|
||||
|
||||
File.WriteAllText("config.conf", hoconConfig.ToString());
|
||||
@ -348,13 +318,9 @@ public class Program
|
||||
Timeout = string.IsNullOrWhiteSpace(hoconConfig.GetString("Performance.Timeout")) ? -1 : hoconConfig.GetInt("Performance.Timeout"),
|
||||
LimitDownloadRate = hoconConfig.GetBoolean("Performance.LimitDownloadRate"),
|
||||
DownloadLimitInMbPerSec = hoconConfig.GetInt("Performance.DownloadLimitInMbPerSec"),
|
||||
ApiRateLimitRequests = SafeGetInt(hoconConfig, "Performance.ApiRateLimitRequests", 10),
|
||||
ApiRateLimitWindowSeconds = SafeGetInt(hoconConfig, "Performance.ApiRateLimitWindowSeconds", 10),
|
||||
|
||||
// Logging/Debug Settings
|
||||
LoggingLevel = Enum.Parse<LoggingLevel>(hoconConfig.GetString("Logging.LoggingLevel"), true),
|
||||
MaxLogBodyLength = SafeGetInt(hoconConfig, "Logging.MaxLogBodyLength", 2000),
|
||||
EnableJsonResponseBodyRawLogs = SafeGetBool(hoconConfig, "Logging.EnableJsonResponseBodyRawLogs", false)
|
||||
LoggingLevel = Enum.Parse<LoggingLevel>(hoconConfig.GetString("Logging.LoggingLevel"), true)
|
||||
};
|
||||
|
||||
ValidateFileNameFormat(config.PaidPostFileNameFormat, "PaidPostFileNameFormat");
|
||||
@ -501,15 +467,11 @@ public class Program
|
||||
hoconConfig.AppendLine($" Timeout = {(jsonConfig.Timeout.HasValue ? jsonConfig.Timeout.Value : -1)}");
|
||||
hoconConfig.AppendLine($" LimitDownloadRate = {jsonConfig.LimitDownloadRate.ToString().ToLower()}");
|
||||
hoconConfig.AppendLine($" DownloadLimitInMbPerSec = {jsonConfig.DownloadLimitInMbPerSec}");
|
||||
hoconConfig.AppendLine($" ApiRateLimitRequests = {jsonConfig.ApiRateLimitRequests}");
|
||||
hoconConfig.AppendLine($" ApiRateLimitWindowSeconds = {jsonConfig.ApiRateLimitWindowSeconds}");
|
||||
hoconConfig.AppendLine("}");
|
||||
|
||||
hoconConfig.AppendLine("# Logging/Debug Settings");
|
||||
hoconConfig.AppendLine("Logging {");
|
||||
hoconConfig.AppendLine($" LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\"");
|
||||
hoconConfig.AppendLine($" MaxLogBodyLength = {jsonConfig.MaxLogBodyLength}");
|
||||
hoconConfig.AppendLine($" EnableJsonResponseBodyRawLogs = {jsonConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
|
||||
hoconConfig.AppendLine("}");
|
||||
|
||||
File.WriteAllText("config.conf", hoconConfig.ToString());
|
||||
@ -3081,15 +3043,11 @@ public class Program
|
||||
hoconConfig.AppendLine($" Timeout = {(newConfig.Timeout.HasValue ? newConfig.Timeout.Value : -1)}");
|
||||
hoconConfig.AppendLine($" LimitDownloadRate = {newConfig.LimitDownloadRate.ToString().ToLower()}");
|
||||
hoconConfig.AppendLine($" DownloadLimitInMbPerSec = {newConfig.DownloadLimitInMbPerSec}");
|
||||
hoconConfig.AppendLine($" ApiRateLimitRequests = {newConfig.ApiRateLimitRequests}");
|
||||
hoconConfig.AppendLine($" ApiRateLimitWindowSeconds = {newConfig.ApiRateLimitWindowSeconds}");
|
||||
hoconConfig.AppendLine("}");
|
||||
|
||||
hoconConfig.AppendLine("# Logging/Debug Settings");
|
||||
hoconConfig.AppendLine("Logging {");
|
||||
hoconConfig.AppendLine($" LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\"");
|
||||
hoconConfig.AppendLine($" MaxLogBodyLength = {newConfig.MaxLogBodyLength}");
|
||||
hoconConfig.AppendLine($" EnableJsonResponseBodyRawLogs = {newConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
|
||||
hoconConfig.AppendLine("}");
|
||||
|
||||
File.WriteAllText("config.conf", hoconConfig.ToString());
|
||||
@ -3247,15 +3205,11 @@ public class Program
|
||||
hoconConfig.AppendLine($" Timeout = {(newConfig.Timeout.HasValue ? newConfig.Timeout.Value : -1)}");
|
||||
hoconConfig.AppendLine($" LimitDownloadRate = {newConfig.LimitDownloadRate.ToString().ToLower()}");
|
||||
hoconConfig.AppendLine($" DownloadLimitInMbPerSec = {newConfig.DownloadLimitInMbPerSec}");
|
||||
hoconConfig.AppendLine($" ApiRateLimitRequests = {newConfig.ApiRateLimitRequests}");
|
||||
hoconConfig.AppendLine($" ApiRateLimitWindowSeconds = {newConfig.ApiRateLimitWindowSeconds}");
|
||||
hoconConfig.AppendLine("}");
|
||||
|
||||
hoconConfig.AppendLine("# Logging/Debug Settings");
|
||||
hoconConfig.AppendLine("Logging {");
|
||||
hoconConfig.AppendLine($" LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\"");
|
||||
hoconConfig.AppendLine($" MaxLogBodyLength = {newConfig.MaxLogBodyLength}");
|
||||
hoconConfig.AppendLine($" EnableJsonResponseBodyRawLogs = {newConfig.EnableJsonResponseBodyRawLogs.ToString().ToLower()}");
|
||||
hoconConfig.AppendLine("}");
|
||||
|
||||
File.WriteAllText("config.conf", hoconConfig.ToString());
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user