diff --git a/OF DL/Helpers/APIHelper.cs b/OF DL/Helpers/APIHelper.cs index d9d7562..89a18f1 100644 --- a/OF DL/Helpers/APIHelper.cs +++ b/OF DL/Helpers/APIHelper.cs @@ -144,6 +144,7 @@ public class APIHelper : IAPIHelper private async Task BuildHeaderAndExecuteRequests(Dictionary getParams, string endpoint, HttpClient client, ILogger? diagnosticLogger = null, string? rawLabel = null) { Log.Debug("Calling BuildHeaderAndExecuteRequests"); + diagnosticLogger ??= CurrentDiagnosticLogger; HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint); diagnosticLogger?.Information("Diag request | method={Method} url={Url} headers={Headers}", request.Method, request.RequestUri, FlattenHeaders(request.Headers)); @@ -409,23 +410,24 @@ public class APIHelper : IAPIHelper return null; } - public async Task GetUserInfoById(string endpoint) - { - try + public async Task GetUserInfoById(string endpoint) { - HttpClient client = new(); - HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary(), endpoint); + try + { + HttpClient client = new(); + HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary(), endpoint); - await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString()); - using var response = await client.SendAsync(request); + await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, request.RequestUri?.ToString()); + using var response = await client.SendAsync(request); - response.EnsureSuccessStatusCode(); - var body = await response.Content.ReadAsStringAsync(); + response.EnsureSuccessStatusCode(); + var body = await response.Content.ReadAsStringAsync(); + WriteRawBody($"user-list-{endpoint.Replace("/", "_")}", body); - //if the content creator doesnt exist, we get a 200 response, but the content isnt usable - //so let's not throw an exception, since "content creator no longer exists" is handled elsewhere - //which means we wont get loads of exceptions - if (body.Equals("[]")) + //if the content creator doesnt exist, we get a 200 response, but the content isnt usable + //so let's not throw an exception, since "content creator no longer exists" is handled elsewhere + //which means we wont get loads of exceptions + if (body.Equals("[]")) return null; JObject jObject = JObject.Parse(body); @@ -777,10 +779,10 @@ public class APIHelper : IAPIHelper { Highlights newhighlights = new(); - Log.Debug("Media Highlights - " + endpoint); + Log.Debug("Media Highlights - " + endpoint); - var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); - newhighlights = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); + var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); + newhighlights = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); highlights.list.AddRange(newhighlights.list); if (!newhighlights.hasMore) @@ -817,6 +819,7 @@ public class APIHelper : IAPIHelper using var highlightResponse = await highlight_client.SendAsync(highlight_request); highlightResponse.EnsureSuccessStatusCode(); var highlightBody = await highlightResponse.Content.ReadAsStringAsync(); + WriteRawBody($"highlight-{highlight_id}", highlightBody); highlightMedia = JsonConvert.DeserializeObject(highlightBody, m_JsonSerializerSettings); if (highlightMedia != null) { @@ -2005,7 +2008,7 @@ public class APIHelper : IAPIHelper { "skip_users", "all" } }; - var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); + var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-initial-{username}"); paidMessages = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); @@ -2015,22 +2018,10 @@ public class APIHelper : IAPIHelper getParams["offset"] = paidMessages.list.Count.ToString(); while (true) { - string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); Purchased newpaidMessages = new(); - Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams); - HttpClient loopclient = GetHttpClient(config); - - HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}"); - - foreach (KeyValuePair keyValuePair in loopheaders) + var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"paid-messages-page-{getParams["offset"]}-{username}"); + if (!string.IsNullOrEmpty(loopbody)) { - looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); - } - await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString()); - using (var loopresponse = await loopclient.SendAsync(looprequest)) - { - loopresponse.EnsureSuccessStatusCode(); - var loopbody = await loopresponse.Content.ReadAsStringAsync(); newpaidMessages = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); } paidMessages.list.AddRange(newpaidMessages.list); @@ -2246,24 +2237,11 @@ public class APIHelper : IAPIHelper getParams["offset"] = purchased.list.Count.ToString(); while (true) { - string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); Purchased newPurchased = new(); - Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams); - HttpClient loopclient = GetHttpClient(config); - - HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}"); - - foreach (KeyValuePair keyValuePair in loopheaders) + var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"posts-paid-all-page-{getParams["offset"]}"); + if (!string.IsNullOrEmpty(loopbody)) { - looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); - } - await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString()); - using (var loopresponse = await loopclient.SendAsync(looprequest)) - { - loopresponse.EnsureSuccessStatusCode(); - var loopbody = await loopresponse.Content.ReadAsStringAsync(); newPurchased = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); - WriteRawBody($"posts-paid-all-page-{getParams["offset"]}", loopbody); } purchased.list.AddRange(newPurchased.list); if (!newPurchased.hasMore) @@ -2434,27 +2412,13 @@ public class APIHelper : IAPIHelper getParams["offset"] = purchased.list.Count.ToString(); while (true) { - string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); Purchased newPurchased = new(); - Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams); - HttpClient loopclient = GetHttpClient(config); - - HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}"); - - foreach (KeyValuePair keyValuePair in loopheaders) + DiagInfo("PurchasedTab page request | ctx={Context} offset={Offset} limit={Limit}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit); + var loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config), CurrentDiagnosticLogger, $"{diagContext}-page-{getParams["offset"]}"); + if (!string.IsNullOrEmpty(loopbody)) { - looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); - } - DiagInfo("PurchasedTab page request | ctx={Context} url={Url} offset={Offset} limit={Limit} headers={Headers}", diagContext, looprequest.RequestUri, getParams.ContainsKey("offset") ? getParams["offset"] : "0", post_limit, FlattenHeaders(looprequest.Headers)); - await EnforceApiRateLimitAsync(CurrentDiagnosticLogger, looprequest.RequestUri?.ToString()); - using (var loopresponse = await loopclient.SendAsync(looprequest)) - { - DiagInfo("PurchasedTab page response | ctx={Context} status={Status} reason={Reason} url={Url}", diagContext, loopresponse.StatusCode, loopresponse.ReasonPhrase, looprequest.RequestUri); - loopresponse.EnsureSuccessStatusCode(); - var loopbody = await loopresponse.Content.ReadAsStringAsync(); newPurchased = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); DiagInfo("PurchasedTab page parsed | ctx={Context} offset={Offset} items={Items} hasMore={HasMore} bodyLength={Length} bodyPreview={BodyPreview}", diagContext, getParams.ContainsKey("offset") ? getParams["offset"] : "0", newPurchased?.list?.Count ?? 0, newPurchased?.hasMore ?? false, loopbody?.Length ?? 0, TruncateForLog(loopbody)); - WriteRawBody($"{diagContext}-page-{getParams["offset"]}", loopbody); } purchased.list.AddRange(newPurchased.list); if (!newPurchased.hasMore)