reptail/OF-DL
1
0
Fork 0
forked from sim0n00ps/OF-DL
Code Pull Requests Activity

Compare commits

base: reptail:5b285acb3750ea8d7d12a035dd18118a20d0d26e
Branches Tags
reptail:master
reptail:dev
reptail:improvements
sim0n00ps:master
reptail:OFDLV1.9.9
reptail:OFDLV1.9.8
reptail:OFDLV1.9.7
sim0n00ps:OFDLV1.9.10
sim0n00ps:OFDLV1.9.11
sim0n00ps:OFDLV1.9.12
sim0n00ps:OFDLV1.9.7
sim0n00ps:OFDLV1.9.8
sim0n00ps:OFDLV1.9.9
..
compare: reptail:d83ad2ec548572b25e1333a73401735bc5ba61a0
Branches Tags
reptail:dev
reptail:master
reptail:improvements
sim0n00ps:master
reptail:OFDLV1.9.9
reptail:OFDLV1.9.8
reptail:OFDLV1.9.7
sim0n00ps:OFDLV1.9.10
sim0n00ps:OFDLV1.9.11
sim0n00ps:OFDLV1.9.12
sim0n00ps:OFDLV1.9.7
sim0n00ps:OFDLV1.9.8
sim0n00ps:OFDLV1.9.9

No commits in common. "5b285acb3750ea8d7d12a035dd18118a20d0d26e" and "d83ad2ec548572b25e1333a73401735bc5ba61a0" have entirely different histories.
5b285acb37 ... d83ad2ec54

2 changed files with 56 additions and 84 deletions
Show Stats Expand all files Collapse all files

119
OF DL/Helpers/APIHelper.cs
View File

@ -13,7 +13,6 @@ using OF_DL.Enumerations;
using OF_DL.Enumurations;
using Serilog;
using Spectre.Console;
using System.Diagnostics;
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
@ -117,38 +116,18 @@ public class APIHelper : IAPIHelper
}
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, int attemptNumber = 1)
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
{
const int MAX_ATTEMPTS = 10;
const int DELAY_BEFORE_RETRY = 1000;
Log.Debug("Calling BuildHeaderAndExecuteRequests");
Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", attemptNumber);
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync();
try
{
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
using var response = await client.SendAsync(request);
Log.Debug(body);
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
Debugger.Break();
response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync();
Log.Debug(body);
return body;
}
catch (HttpRequestException ex)
{
if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && attemptNumber <= MAX_ATTEMPTS)
{
await Task.Delay(DELAY_BEFORE_RETRY);
return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, ++attemptNumber);
}
throw;
}
return body;
}
@ -319,44 +298,47 @@ public class APIHelper : IAPIHelper
try
{
Dictionary<string, int> users = new();
int limit = 25;
int offset = 0;
getParams["limit"] = limit.ToString();
getParams["offset"] = offset.ToString();
Subscriptions subscriptions = new();
Log.Debug("Calling GetAllSubscrptions");
while (true)
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
if (subscriptions != null && subscriptions.hasMore)
{
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
getParams["offset"] = subscriptions.list.Count.ToString();
if (string.IsNullOrWhiteSpace(body))
break;
Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
if (subscriptions?.list is null)
break;
foreach (Subscriptions.List item in subscriptions.list)
while (true)
{
if (users.ContainsKey(item.username))
continue;
Subscriptions newSubscriptions = new();
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
bool isRestricted = item.isRestricted ?? false;
bool isRestrictedButAllowed = isRestricted && includeRestricted;
if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
{
newSubscriptions = JsonConvert.DeserializeObject<Subscriptions>(loopbody, m_JsonSerializerSettings);
}
else
{
break;
}
if (!isRestricted || isRestrictedButAllowed)
users.Add(item.username, item.id);
subscriptions.list.AddRange(newSubscriptions.list);
if (!newSubscriptions.hasMore)
{
break;
}
getParams["offset"] = subscriptions.list.Count.ToString();
}
}
if (!subscriptions.hasMore)
break;
offset += limit;
getParams["offset"] = offset.ToString();
foreach (Subscriptions.List subscription in subscriptions.list)
{
if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted))
&& !users.ContainsKey(subscription.username))
{
users.Add(subscription.username, subscription.id);
}
}
return users;
@ -379,6 +361,8 @@ public class APIHelper : IAPIHelper
{
Dictionary<string, string> getParams = new()
{
{ "offset", "0" },
{ "limit", "50" },
{ "type", "active" },
{ "format", "infinite"}
};
@ -392,6 +376,8 @@ public class APIHelper : IAPIHelper
Dictionary<string, string> getParams = new()
{
{ "offset", "0" },
{ "limit", "50" },
{ "type", "expired" },
{ "format", "infinite"}
};
@ -474,16 +460,12 @@ public class APIHelper : IAPIHelper
try
{
int limit = 50;
int offset = 0;
Dictionary<string, string> getParams = new()
{
{ "limit", limit.ToString() },
{ "offset", offset.ToString() },
{ "format", "infinite"}
{ "limit", "50" }
};
List<string> users = new();
while (true)
@ -494,8 +476,6 @@ public class APIHelper : IAPIHelper
break;
}
UserList? userList = JsonConvert.DeserializeObject<UserList>(body, m_JsonSerializerSettings);
List<UsersList>? usersList = JsonConvert.DeserializeObject<List<UsersList>>(body);
if (usersList == null || usersList.Count <= 0)
@ -517,7 +497,6 @@ public class APIHelper : IAPIHelper
getParams["offset"] = Convert.ToString(offset);
}
return users;
}
catch (Exception ex)
@ -1171,7 +1150,7 @@ public class APIHelper : IAPIHelper
}
break;
case VideoResolution._240:
if (medium.videoSources != null)
if(medium.videoSources != null)
{
if (!string.IsNullOrEmpty(medium.videoSources._240))
{
@ -1198,7 +1177,7 @@ public class APIHelper : IAPIHelper
}
}
break;
}
}
else if (medium.canView && medium.files != null && medium.files.drm != null)
@ -2155,11 +2134,11 @@ public class APIHelper : IAPIHelper
{
JObject user = await GetUserInfoById($"/users/list?x[]={purchase.fromUser.id}");
if (user is null)
if(user is null)
{
if (!config.BypassContentForCreatorsWhoNoLongerExist)
{
if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}"))
{
purchasedTabUsers.Add($"Deleted User - {purchase.fromUser.id}", purchase.fromUser.id);
}
@ -2209,7 +2188,7 @@ public class APIHelper : IAPIHelper
{
if (!config.BypassContentForCreatorsWhoNoLongerExist)
{
if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
if(!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}"))
{
purchasedTabUsers.Add($"Deleted User - {purchase.author.id}", purchase.author.id);
}

21
OF DL/Program.cs
View File

@ -844,9 +844,7 @@ public class Program
{
DateTime startTime = DateTime.Now;
Dictionary<string, int> users = new();
AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
Log.Debug("Subscriptions: ");
@ -862,8 +860,7 @@ public class Program
{
Log.Debug("Inactive Subscriptions: ");
AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
foreach (KeyValuePair<string, int> expiredSub in expiredSubs)
{
if (!users.ContainsKey(expiredSub.Key))
@ -1027,11 +1024,9 @@ public class Program
Log.Debug($"Download path: {p}");
List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
int userNum = 1;
int userCount = purchasedTabCollections.Count;
foreach (PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
{
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]");
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]");
string path = "";
if (!string.IsNullOrEmpty(Config.DownloadPath))
{
@ -1140,10 +1135,8 @@ public class Program
}
else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
{
//Iterate over each user in the list of users
int userNum = 1;
int userCount = hasSelectedUsersKVP.Value.Count;
foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
//Iterate over each user in the list of users
foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
{
int paidPostCount = 0;
int postCount = 0;
@ -1153,7 +1146,7 @@ public class Program
int highlightsCount = 0;
int messagesCount = 0;
int paidMessagesCount = 0;
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]");
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]");
Log.Debug($"Scraping Data for {user.Key}");