nyc_tk/OF-DL
1
0
Fork 0
forked from sim0n00ps/OF-DL
Code Pull Requests Activity

Compare commits

base: nyc_tk:e6075be886f6534099f38b39e65f8997732111c1
Branches Tags
nyc_tk:master
nyc_tk:fix-purchased-media-obj-deref-error
nyc_tk:docker-update-ffmpeg
nyc_tk:fix-purchased-media-obj-defref
sim0n00ps:master
sim0n00ps:avalonia-gui
sim0n00ps:already_downloaded_error
sim0n00ps:ffmpeg_additional_logging
sim0n00ps:header_update
sim0n00ps:version_timeout
sim0n00ps:ffmpeg_version
nyc_tk:OFDLV1.9.14
nyc_tk:OFDLV1.9.13
nyc_tk:OFDLV1.9.12
nyc_tk:OFDLV1.9.11
nyc_tk:OFDLV1.9.10
nyc_tk:OFDLV1.9.9
nyc_tk:OFDLV1.9.8
nyc_tk:OFDLV1.9.7
sim0n00ps:OFDLV1.10.0
sim0n00ps:OFDLV1.9.10
sim0n00ps:OFDLV1.9.11
sim0n00ps:OFDLV1.9.12
sim0n00ps:OFDLV1.9.13
sim0n00ps:OFDLV1.9.14
sim0n00ps:OFDLV1.9.15
sim0n00ps:OFDLV1.9.16
sim0n00ps:OFDLV1.9.17
sim0n00ps:OFDLV1.9.18
sim0n00ps:OFDLV1.9.19
sim0n00ps:OFDLV1.9.20
sim0n00ps:OFDLV1.9.7
sim0n00ps:OFDLV1.9.8
sim0n00ps:OFDLV1.9.9
..
compare: nyc_tk:0dccb0ca9c8136af1e5cf09ac335e80d9a41cef0
Branches Tags
nyc_tk:fix-purchased-media-obj-deref-error
nyc_tk:docker-update-ffmpeg
nyc_tk:fix-purchased-media-obj-defref
nyc_tk:master
sim0n00ps:master
sim0n00ps:avalonia-gui
sim0n00ps:already_downloaded_error
sim0n00ps:ffmpeg_additional_logging
sim0n00ps:header_update
sim0n00ps:version_timeout
sim0n00ps:ffmpeg_version
nyc_tk:OFDLV1.9.14
nyc_tk:OFDLV1.9.13
nyc_tk:OFDLV1.9.12
nyc_tk:OFDLV1.9.11
nyc_tk:OFDLV1.9.10
nyc_tk:OFDLV1.9.9
nyc_tk:OFDLV1.9.8
nyc_tk:OFDLV1.9.7
sim0n00ps:OFDLV1.10.0
sim0n00ps:OFDLV1.9.10
sim0n00ps:OFDLV1.9.11
sim0n00ps:OFDLV1.9.12
sim0n00ps:OFDLV1.9.13
sim0n00ps:OFDLV1.9.14
sim0n00ps:OFDLV1.9.15
sim0n00ps:OFDLV1.9.16
sim0n00ps:OFDLV1.9.17
sim0n00ps:OFDLV1.9.18
sim0n00ps:OFDLV1.9.19
sim0n00ps:OFDLV1.9.20
sim0n00ps:OFDLV1.9.7
sim0n00ps:OFDLV1.9.8
sim0n00ps:OFDLV1.9.9

No commits in common. "e6075be886f6534099f38b39e65f8997732111c1" and "0dccb0ca9c8136af1e5cf09ac335e80d9a41cef0" have entirely different histories.
e6075be886 ... 0dccb0ca9c

1 changed files with 64 additions and 31 deletions
Show Stats Expand all files Collapse all files

95
OF DL/Helpers/APIHelper.cs
View File

@ -144,7 +144,6 @@ public class APIHelper : IAPIHelper
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, ILogger? diagnosticLogger = null, string? rawLabel = null)
{
Log.Debug("Calling BuildHeaderAndExecuteRequests");
diagnosticLogger ??= CurrentDiagnosticLogger;
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
diagnosticLogger?.Information("Diag request | method={Method} url={Url} headers={Headers}", request.Method, request.RequestUri, FlattenHeaders(request.Headers));
@ -392,7 +391,6 @@ public class APIHelper : IAPIHelper
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
WriteRawBody($"user-{endpoint.Replace("/", "_")}", body);
user = JsonConvert.DeserializeObject<Entities.User>(body, m_JsonSerializerSettings);
return user;
}
@ -410,24 +408,23 @@ public class APIHelper : IAPIHelper
return null;
}
public async Task<JObject> GetUserInfoById(string endpoint)
public async Task<JObject> GetUserInfoById(string endpoint)
{
try
{
try
{
HttpClient client = new();
HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
HttpClient client = new();
HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
using var response = await client.SendAsync(request);
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString());
using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
WriteRawBody($"user-list-{endpoint.Replace("/", "_")}", body);
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
//if the content creator doesnt exist, we get a 200 response, but the content isnt usable
//so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
//which means we wont get loads of exceptions
if (body.Equals("[]"))
//if the content creator doesnt exist, we get a 200 response, but the content isnt usable
//so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
//which means we wont get loads of exceptions
if (body.Equals("[]"))
return null;
JObject jObject = JObject.Parse(body);
@ -779,10 +776,10 @@ public class APIHelper : IAPIHelper
{
Highlights newhighlights = new();
Log.Debug("Media Highlights - " + endpoint);
Log.Debug("Media Highlights - " + endpoint);
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
newhighlights = JsonConvert.DeserializeObject<Highlights>(loopbody, m_JsonSerializerSettings);
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
newhighlights = JsonConvert.DeserializeObject<Highlights>(loopbody, m_JsonSerializerSettings);
highlights.list.AddRange(newhighlights.list);
if (!newhighlights.hasMore)
@ -815,11 +812,10 @@ public class APIHelper : IAPIHelper
highlight_request.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, highlight_request.RequestUri?.ToString());
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger);
using var highlightResponse = await highlight_client.SendAsync(highlight_request);
highlightResponse.EnsureSuccessStatusCode();
var highlightBody = await highlightResponse.Content.ReadAsStringAsync();
WriteRawBody($"highlight-{highlight_id}", highlightBody);
highlightMedia = JsonConvert.DeserializeObject<HighlightMedia>(highlightBody, m_JsonSerializerSettings);
if (highlightMedia != null)
{
@ -2008,7 +2004,7 @@ public class APIHelper : IAPIHelper
{ "skip_users", "all" }
};
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-initial-{username}");
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
paidMessages = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.list.Count}[/]");
ctx.Spinner(Spinner.Known.Dots);
@ -2018,10 +2014,22 @@ public class APIHelper : IAPIHelper
getParams["offset"] = paidMessages.list.Count.ToString();
while (true)
{
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
Purchased newpaidMessages = new();
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-page-{getParams["offset"]}-{username}");
if (!string.IsNullOrEmpty(loopbody))
Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
HttpClient loopclient = GetHttpClient(config);
HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
{
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString());
using (var loopresponse = await loopclient.SendAsync(looprequest))
{
loopresponse.EnsureSuccessStatusCode();
var loopbody = await loopresponse.Content.ReadAsStringAsync();
newpaidMessages = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
}
paidMessages.list.AddRange(newpaidMessages.list);
@ -2229,18 +2237,29 @@ public class APIHelper : IAPIHelper
{ "skip_users", "all" }
};
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, "posts-paid-all-initial");
WriteRawBody("posts-paid-all-initial", body);
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
purchased = JsonConvert.DeserializeObject<Purchased>(body, m_JsonSerializerSettings);
if (purchased != null && purchased.hasMore)
{
getParams["offset"] = purchased.list.Count.ToString();
while (true)
{
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
Purchased newPurchased = new();
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"posts-paid-all-page-{getParams["offset"]}");
if (!string.IsNullOrEmpty(loopbody))
Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
HttpClient loopclient = GetHttpClient(config);
HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
{
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger);
using (var loopresponse = await loopclient.SendAsync(looprequest))
{
loopresponse.EnsureSuccessStatusCode();
var loopbody = await loopresponse.Content.ReadAsStringAsync();
newPurchased = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
}
purchased.list.AddRange(newPurchased.list);
@ -2412,13 +2431,27 @@ public class APIHelper : IAPIHelper
getParams["offset"] = purchased.list.Count.ToString();
while (true)
{
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
Purchased newPurchased = new();
DiagInfo("PurchasedTab page request | ctx={Context} offset={Offset} limit={Limit}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit);
var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"{diagContext}-page-{getParams["offset"]}");
if (!string.IsNullOrEmpty(loopbody))
Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
HttpClient loopclient = GetHttpClient(config);
HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
{
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
DiagInfo("PurchasedTab page request | ctx={Context} url={Url} offset={Offset} limit={Limit} headers={Headers}", diagContext, looprequest.RequestUri, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit, FlattenHeaders(looprequest.Headers));
await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString());
using (var loopresponse = await loopclient.SendAsync(looprequest))
{
DiagInfo("PurchasedTab page response | ctx={Context} status={Status} reason={Reason} url={Url}", diagContext, loopresponse.StatusCode, loopresponse.ReasonPhrase, looprequest.RequestUri);
loopresponse.EnsureSuccessStatusCode();
var loopbody = await loopresponse.Content.ReadAsStringAsync();
newPurchased = JsonConvert.DeserializeObject<Purchased>(loopbody, m_JsonSerializerSettings);
DiagInfo("PurchasedTab page parsed | ctx={Context} offset={Offset} items={Items} hasMore={HasMore} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", newPurchased?.list?.Count ?? 0, newPurchased?.hasMore ?? false, loopbody?.Length ?? 0, TruncateForLog(loopbody));
WriteRawBody($"{diagContext}-page-{getParams["offset"]}", loopbody);
}
purchased.list.AddRange(newPurchased.list);
if (!newPurchased.hasMore)