forked from sim0n00ps/OF-DL
3393 lines
159 KiB
C#
3393 lines
159 KiB
C#
using System.Globalization;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Xml.Linq;
|
|
using Newtonsoft.Json;
|
|
using Newtonsoft.Json.Linq;
|
|
using OF_DL.Models;
|
|
using OF_DL.Models.Entities.Common;
|
|
using OF_DL.Enumerations;
|
|
using ArchivedDtos = OF_DL.Models.Dtos.Archived;
|
|
using HighlightDtos = OF_DL.Models.Dtos.Highlights;
|
|
using ListDtos = OF_DL.Models.Dtos.Lists;
|
|
using MessageDtos = OF_DL.Models.Dtos.Messages;
|
|
using PostDtos = OF_DL.Models.Dtos.Posts;
|
|
using PurchasedDtos = OF_DL.Models.Dtos.Purchased;
|
|
using StoriesDtos = OF_DL.Models.Dtos.Stories;
|
|
using StreamsDtos = OF_DL.Models.Dtos.Streams;
|
|
using UserDtos = OF_DL.Models.Dtos.Users;
|
|
using SubscriptionsDtos = OF_DL.Models.Dtos.Subscriptions;
|
|
using ArchivedEntities = OF_DL.Models.Entities.Archived;
|
|
using HighlightEntities = OF_DL.Models.Entities.Highlights;
|
|
using ListEntities = OF_DL.Models.Entities.Lists;
|
|
using MessageEntities = OF_DL.Models.Entities.Messages;
|
|
using PostEntities = OF_DL.Models.Entities.Posts;
|
|
using PurchasedEntities = OF_DL.Models.Entities.Purchased;
|
|
using StoryEntities = OF_DL.Models.Entities.Stories;
|
|
using StreamEntities = OF_DL.Models.Entities.Streams;
|
|
using SubscriptionEntities = OF_DL.Models.Entities.Subscriptions;
|
|
using UserEntities = OF_DL.Models.Entities.Users;
|
|
using OF_DL.Models.Mappers;
|
|
using OF_DL.Models.OfdlApi;
|
|
using OF_DL.Widevine;
|
|
using Serilog;
|
|
using static OF_DL.Utils.HttpUtil;
|
|
using Constants = OF_DL.Helpers.Constants;
|
|
using SinglePostCollection = OF_DL.Models.Entities.Posts.SinglePostCollection;
|
|
|
|
namespace OF_DL.Services;
|
|
|
|
public class ApiService(IAuthService authService, IConfigService configService, IDBService dbService)
|
|
: IAPIService
|
|
{
|
|
private const int MaxAttempts = 30;
|
|
private const int DelayBetweenAttempts = 3000;
|
|
private static readonly JsonSerializerSettings s_mJsonSerializerSettings;
|
|
private static DateTime? s_cachedDynamicRulesExpiration;
|
|
private static DynamicRules? s_cachedDynamicRules;
|
|
|
|
static ApiService() =>
|
|
s_mJsonSerializerSettings = new JsonSerializerSettings { MissingMemberHandling = MissingMemberHandling.Ignore };
|
|
|
|
|
|
public Dictionary<string, string> GetDynamicHeaders(string path, string queryParams)
|
|
{
|
|
Log.Debug("Calling GetDynamicHeaders");
|
|
Log.Debug("Path: {Path}", path);
|
|
Log.Debug("Query Params: {QueryParams}", queryParams);
|
|
|
|
DynamicRules? root;
|
|
|
|
//Check if we have a cached version of the dynamic rules
|
|
if (s_cachedDynamicRules != null && s_cachedDynamicRulesExpiration.HasValue &&
|
|
DateTime.UtcNow < s_cachedDynamicRulesExpiration)
|
|
{
|
|
Log.Debug("Using cached dynamic rules");
|
|
root = s_cachedDynamicRules;
|
|
}
|
|
else
|
|
{
|
|
//Get rules from GitHub and fallback to local file
|
|
string? dynamicRulesJson = GetDynamicRules();
|
|
if (!string.IsNullOrEmpty(dynamicRulesJson))
|
|
{
|
|
Log.Debug("Using dynamic rules from GitHub");
|
|
root = JsonConvert.DeserializeObject<DynamicRules>(dynamicRulesJson);
|
|
|
|
// Cache the GitHub response for 15 minutes
|
|
s_cachedDynamicRules = root;
|
|
s_cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(15);
|
|
}
|
|
else
|
|
{
|
|
Log.Debug("Using dynamic rules from local file");
|
|
root = JsonConvert.DeserializeObject<DynamicRules>(File.ReadAllText("rules.json"));
|
|
|
|
// Cache the dynamic rules from local file to prevent unnecessary disk
|
|
// operations and frequent call to GitHub. Since the GitHub dynamic rules
|
|
// are preferred to the local file, the cache time is shorter than when dynamic rules
|
|
// are successfully retrieved from GitHub.
|
|
s_cachedDynamicRules = root;
|
|
s_cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(5);
|
|
}
|
|
}
|
|
|
|
if (root == null)
|
|
{
|
|
throw new Exception("Unable to parse dynamic rules. Root is null");
|
|
}
|
|
|
|
if (root.ChecksumConstant == null || root.ChecksumIndexes.Count == 0 || root.Prefix == null ||
|
|
root.Suffix == null || root.AppToken == null)
|
|
{
|
|
throw new Exception("Invalid dynamic rules. Missing required fields");
|
|
}
|
|
|
|
if (authService.CurrentAuth == null)
|
|
{
|
|
throw new Exception("Auth service is null");
|
|
}
|
|
|
|
if (authService.CurrentAuth.UserId == null || authService.CurrentAuth.Cookie == null ||
|
|
authService.CurrentAuth.UserAgent == null || authService.CurrentAuth.XBc == null)
|
|
{
|
|
throw new Exception("Auth service is missing required fields");
|
|
}
|
|
|
|
DateTimeOffset dto = DateTime.UtcNow;
|
|
long timestamp = dto.ToUnixTimeMilliseconds();
|
|
|
|
string input = $"{root.StaticParam}\n{timestamp}\n{path + queryParams}\n{authService.CurrentAuth.UserId}";
|
|
byte[] inputBytes = Encoding.UTF8.GetBytes(input);
|
|
byte[] hashBytes = SHA1.HashData(inputBytes);
|
|
string hashString = BitConverter.ToString(hashBytes).Replace("-", "").ToLower();
|
|
|
|
int checksum = root.ChecksumIndexes.Aggregate(0, (current, number) => current + hashString[number]) +
|
|
root.ChecksumConstant.Value;
|
|
string sign = $"{root.Prefix}:{hashString}:{checksum.ToString("X").ToLower()}:{root.Suffix}";
|
|
|
|
Dictionary<string, string> headers = new()
|
|
{
|
|
{ "accept", "application/json, text/plain" },
|
|
{ "app-token", root.AppToken },
|
|
{ "cookie", authService.CurrentAuth.Cookie },
|
|
{ "sign", sign },
|
|
{ "time", timestamp.ToString() },
|
|
{ "user-id", authService.CurrentAuth.UserId },
|
|
{ "user-agent", authService.CurrentAuth.UserAgent },
|
|
{ "x-bc", authService.CurrentAuth.XBc }
|
|
};
|
|
return headers;
|
|
}
|
|
|
|
|
|
public async Task<UserEntities.User?> GetUserInfo(string endpoint)
|
|
{
|
|
Log.Debug($"Calling GetUserInfo: {endpoint}");
|
|
|
|
try
|
|
{
|
|
UserEntities.User user = new();
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "limit", postLimit.ToString() }, { "order", "publish_date_asc" }
|
|
};
|
|
|
|
HttpClient client = new();
|
|
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
|
|
|
|
using HttpResponseMessage response = await client.SendAsync(request);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
return user;
|
|
}
|
|
|
|
response.EnsureSuccessStatusCode();
|
|
string body = await response.Content.ReadAsStringAsync();
|
|
UserDtos.UserDto? userDto =
|
|
JsonConvert.DeserializeObject<UserDtos.UserDto>(body, s_mJsonSerializerSettings);
|
|
user = UserMapper.FromDto(userDto) ?? new UserEntities.User();
|
|
return user;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public async Task<JObject?> GetUserInfoById(string endpoint)
|
|
{
|
|
try
|
|
{
|
|
HttpClient client = new();
|
|
HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary<string, string>(), endpoint);
|
|
|
|
using HttpResponseMessage response = await client.SendAsync(request);
|
|
|
|
response.EnsureSuccessStatusCode();
|
|
string body = await response.Content.ReadAsStringAsync();
|
|
|
|
//if the content creator doesnt exist, we get a 200 response, but the content isnt usable
|
|
//so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
|
|
//which means we wont get loads of exceptions
|
|
if (body.Equals("[]"))
|
|
{
|
|
return null;
|
|
}
|
|
|
|
JObject jObject = JObject.Parse(body);
|
|
|
|
return jObject;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public async Task<Dictionary<string, long>?> GetActiveSubscriptions(string endpoint, bool includeRestricted)
|
|
{
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "offset", "0" }, { "limit", "50" }, { "type", "active" }, { "format", "infinite" }
|
|
};
|
|
|
|
return await GetAllSubscriptions(getParams, endpoint, includeRestricted);
|
|
}
|
|
|
|
|
|
public async Task<Dictionary<string, long>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted)
|
|
{
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "offset", "0" }, { "limit", "50" }, { "type", "expired" }, { "format", "infinite" }
|
|
};
|
|
|
|
Log.Debug("Calling GetExpiredSubscriptions");
|
|
|
|
return await GetAllSubscriptions(getParams, endpoint, includeRestricted);
|
|
}
|
|
|
|
|
|
public async Task<Dictionary<string, long>?> GetLists(string endpoint)
|
|
{
|
|
Log.Debug("Calling GetLists");
|
|
|
|
try
|
|
{
|
|
int offset = 0;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "offset", offset.ToString() },
|
|
{ "skip_users", "all" },
|
|
{ "limit", "50" },
|
|
{ "format", "infinite" }
|
|
};
|
|
Dictionary<string, long> lists = new();
|
|
while (true)
|
|
{
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
|
|
|
if (body == null)
|
|
{
|
|
break;
|
|
}
|
|
|
|
ListDtos.UserListDto? userListDto = JsonConvert.DeserializeObject<ListDtos.UserListDto>(body);
|
|
ListEntities.UserList userList = UserListsMapper.FromDto(userListDto);
|
|
|
|
foreach (ListEntities.UserListItem listItem in userList.List)
|
|
{
|
|
if (IsStringOnlyDigits(listItem.Id) && !lists.ContainsKey(listItem.Name))
|
|
{
|
|
lists.Add(listItem.Name, Convert.ToInt32(listItem.Id));
|
|
}
|
|
}
|
|
|
|
if (userList.HasMore)
|
|
{
|
|
offset += 50;
|
|
getParams["offset"] = Convert.ToString(offset);
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
|
|
return lists;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
|
|
public async Task<List<string>?> GetListUsers(string endpoint)
|
|
{
|
|
Log.Debug($"Calling GetListUsers - {endpoint}");
|
|
|
|
try
|
|
{
|
|
int offset = 0;
|
|
Dictionary<string, string> getParams = new() { { "offset", offset.ToString() }, { "limit", "50" } };
|
|
List<string> users = new();
|
|
|
|
while (true)
|
|
{
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
|
if (body == null)
|
|
{
|
|
break;
|
|
}
|
|
|
|
List<ListDtos.UsersListDto>? usersListDto =
|
|
JsonConvert.DeserializeObject<List<ListDtos.UsersListDto>>(body);
|
|
List<ListEntities.UsersList> usersList = UserListsMapper.FromDto(usersListDto);
|
|
|
|
if (usersList.Count <= 0)
|
|
{
|
|
break;
|
|
}
|
|
|
|
foreach (ListEntities.UsersList ul in usersList)
|
|
{
|
|
users.Add(ul.Username);
|
|
}
|
|
|
|
if (users.Count < 50)
|
|
{
|
|
break;
|
|
}
|
|
|
|
offset += 50;
|
|
getParams["offset"] = Convert.ToString(offset);
|
|
}
|
|
|
|
return users;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
|
|
public async Task<Dictionary<long, string>?> GetMedia(MediaType mediatype,
|
|
string endpoint,
|
|
string? username,
|
|
string folder,
|
|
List<long> paid_post_ids)
|
|
{
|
|
Log.Debug($"Calling GetMedia - {username}");
|
|
|
|
try
|
|
{
|
|
Dictionary<long, string> return_urls = new();
|
|
const int postLimit = 50;
|
|
int limit = 5;
|
|
int offset = 0;
|
|
|
|
Dictionary<string, string> getParams = new();
|
|
|
|
switch (mediatype)
|
|
{
|
|
case MediaType.Stories:
|
|
getParams = new Dictionary<string, string>
|
|
{
|
|
{ "limit", postLimit.ToString() }, { "order", "publish_date_desc" }, { "skip_users", "all" }
|
|
};
|
|
break;
|
|
|
|
case MediaType.Highlights:
|
|
getParams = new Dictionary<string, string>
|
|
{
|
|
{ "limit", limit.ToString() }, { "offset", offset.ToString() }, { "skip_users", "all" }
|
|
};
|
|
break;
|
|
}
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
|
|
|
if (string.IsNullOrWhiteSpace(body))
|
|
{
|
|
Log.Warning("GetMedia returned empty response for {Endpoint}", endpoint);
|
|
return return_urls;
|
|
}
|
|
|
|
if (mediatype == MediaType.Stories)
|
|
{
|
|
Log.Debug("Media Stories - " + endpoint);
|
|
|
|
List<StoriesDtos.StoryDto>? storiesDto =
|
|
DeserializeJson<List<StoriesDtos.StoryDto>>(body, s_mJsonSerializerSettings);
|
|
List<StoryEntities.Stories> stories = StoriesMapper.FromDto(storiesDto);
|
|
|
|
foreach (StoryEntities.Stories story in stories)
|
|
{
|
|
DateTime? storyCreatedAt = story.Media.Count > 0 ? story.Media[0].CreatedAt : null;
|
|
if (storyCreatedAt.HasValue)
|
|
{
|
|
await dbService.AddStory(folder, story.Id, "", "0", false, false, storyCreatedAt.Value);
|
|
}
|
|
else if (story.CreatedAt.HasValue)
|
|
{
|
|
await dbService.AddStory(folder, story.Id, "", "0", false, false, story.CreatedAt.Value);
|
|
}
|
|
else
|
|
{
|
|
await dbService.AddStory(folder, story.Id, "", "0", false, false, DateTime.Now);
|
|
}
|
|
|
|
if (story.Media.Count > 0)
|
|
{
|
|
foreach (StoryEntities.Medium medium in story.Media)
|
|
{
|
|
string? mediaUrl = medium.Files.Full?.Url;
|
|
if (string.IsNullOrEmpty(mediaUrl))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string? mediaType = ResolveMediaType(medium.Type);
|
|
if (mediaType == null)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
await dbService.AddMedia(folder, medium.Id, story.Id, mediaUrl, null, null, null,
|
|
"Stories", mediaType, false, false, null);
|
|
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.CanView && !return_urls.ContainsKey(medium.Id))
|
|
{
|
|
return_urls.Add(medium.Id, mediaUrl);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (mediatype == MediaType.Highlights)
|
|
{
|
|
List<string> highlightIds = [];
|
|
HighlightDtos.HighlightsDto? highlightsDto =
|
|
DeserializeJson<HighlightDtos.HighlightsDto>(body, s_mJsonSerializerSettings);
|
|
HighlightEntities.Highlights highlights = HighlightsMapper.FromDto(highlightsDto);
|
|
|
|
if (highlights.HasMore)
|
|
{
|
|
offset += 5;
|
|
getParams["offset"] = offset.ToString();
|
|
while (true)
|
|
{
|
|
Log.Debug("Media Highlights - " + endpoint);
|
|
|
|
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
if (string.IsNullOrWhiteSpace(loopbody))
|
|
{
|
|
Log.Warning("Received empty body from API");
|
|
break;
|
|
}
|
|
|
|
HighlightDtos.HighlightsDto? newHighlightsDto =
|
|
DeserializeJson<HighlightDtos.HighlightsDto>(loopbody, s_mJsonSerializerSettings);
|
|
HighlightEntities.Highlights newHighlights = HighlightsMapper.FromDto(newHighlightsDto);
|
|
|
|
highlights.List.AddRange(newHighlights.List);
|
|
if (!newHighlights.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
offset += 5;
|
|
getParams["offset"] = offset.ToString();
|
|
}
|
|
}
|
|
|
|
foreach (HighlightEntities.ListItem list in highlights.List)
|
|
{
|
|
if (!highlightIds.Contains(list.Id.ToString()))
|
|
{
|
|
highlightIds.Add(list.Id.ToString());
|
|
}
|
|
}
|
|
|
|
foreach (string highlight_id in highlightIds)
|
|
{
|
|
Dictionary<string, string> highlight_headers =
|
|
GetDynamicHeaders("/api2/v2/stories/highlights/" + highlight_id, "");
|
|
|
|
HttpClient highlight_client = GetHttpClient();
|
|
|
|
HttpRequestMessage highlight_request = new(HttpMethod.Get,
|
|
$"https://onlyfans.com/api2/v2/stories/highlights/{highlight_id}");
|
|
|
|
foreach (KeyValuePair<string, string> keyValuePair in highlight_headers)
|
|
{
|
|
highlight_request.Headers.Add(keyValuePair.Key, keyValuePair.Value);
|
|
}
|
|
|
|
using HttpResponseMessage highlightResponse = await highlight_client.SendAsync(highlight_request);
|
|
highlightResponse.EnsureSuccessStatusCode();
|
|
string highlightBody = await highlightResponse.Content.ReadAsStringAsync();
|
|
HighlightDtos.HighlightMediaDto? highlightMediaDto =
|
|
DeserializeJson<HighlightDtos.HighlightMediaDto>(highlightBody, s_mJsonSerializerSettings);
|
|
HighlightEntities.HighlightMedia highlightMedia = HighlightsMapper.FromDto(highlightMediaDto);
|
|
|
|
foreach (HighlightEntities.Story item in highlightMedia.Stories)
|
|
{
|
|
DateTime? createdAt = item.Media != null && item.Media.Count > 0
|
|
? item.Media[0].CreatedAt
|
|
: null;
|
|
|
|
if (createdAt.HasValue)
|
|
{
|
|
await dbService.AddStory(folder, item.Id, "", "0", false, false, createdAt.Value);
|
|
}
|
|
else if (item.CreatedAt.HasValue)
|
|
{
|
|
await dbService.AddStory(folder, item.Id, "", "0", false, false, item.CreatedAt.Value);
|
|
}
|
|
else
|
|
{
|
|
await dbService.AddStory(folder, item.Id, "", "0", false, false, DateTime.Now);
|
|
}
|
|
|
|
if (item.Media != null && item.Media.Count > 0 && item.Media[0].CanView)
|
|
{
|
|
string? storyUrl = item.Media[0].Files?.Full?.Url;
|
|
string storyUrlValue = storyUrl ?? string.Empty;
|
|
foreach (HighlightEntities.Medium medium in item.Media)
|
|
{
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
await dbService.AddMedia(folder, medium.Id, item.Id, storyUrlValue, null, null, null,
|
|
"Stories", mediaType, false, false, null);
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!return_urls.ContainsKey(medium.Id) && !string.IsNullOrEmpty(storyUrl))
|
|
{
|
|
return_urls.Add(medium.Id, storyUrl);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return return_urls;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
|
|
public async Task<PurchasedEntities.PaidPostCollection> GetPaidPosts(string endpoint, string folder,
|
|
string username,
|
|
List<long> paid_post_ids, IStatusReporter statusReporter)
|
|
{
|
|
Log.Debug($"Calling GetPaidPosts - {username}");
|
|
|
|
try
|
|
{
|
|
PurchasedEntities.PaidPostCollection paidPostCollection = new();
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "limit", postLimit.ToString() },
|
|
{ "skip_users", "all" },
|
|
{ "order", "publish_date_desc" },
|
|
{ "format", "infinite" },
|
|
{ "author", username }
|
|
};
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
PurchasedDtos.PurchasedDto? paidPostsDto =
|
|
DeserializeJson<PurchasedDtos.PurchasedDto>(body, s_mJsonSerializerSettings);
|
|
PurchasedEntities.Purchased paidPosts = PurchasedMapper.FromDto(paidPostsDto);
|
|
statusReporter.ReportStatus($"Getting Paid Posts - Found {paidPosts.List.Count}");
|
|
if (paidPosts.HasMore)
|
|
{
|
|
getParams["offset"] = paidPosts.List.Count.ToString();
|
|
while (true)
|
|
{
|
|
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
PurchasedDtos.PurchasedDto? newPaidPostsDto =
|
|
DeserializeJson<PurchasedDtos.PurchasedDto>(loopbody, s_mJsonSerializerSettings);
|
|
PurchasedEntities.Purchased newPaidPosts = PurchasedMapper.FromDto(newPaidPostsDto);
|
|
|
|
paidPosts.List.AddRange(newPaidPosts.List);
|
|
statusReporter.ReportStatus($"Getting Paid Posts - Found {paidPosts.List.Count}");
|
|
if (!newPaidPosts.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + postLimit);
|
|
}
|
|
}
|
|
|
|
List<PurchasedEntities.ListItem> paidPostList = paidPosts.List;
|
|
foreach (PurchasedEntities.ListItem purchase in paidPostList)
|
|
{
|
|
if (purchase.ResponseType == "post" && purchase.Media != null && purchase.Media.Count > 0)
|
|
{
|
|
List<long> previewids = new();
|
|
if (purchase.Previews != null)
|
|
{
|
|
for (int i = 0; i < purchase.Previews.Count; i++)
|
|
{
|
|
if (purchase.Previews[i] is long previewId)
|
|
{
|
|
if (!previewids.Contains(previewId))
|
|
{
|
|
previewids.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (purchase.Preview != null)
|
|
{
|
|
for (int i = 0; i < purchase.Preview.Count; i++)
|
|
{
|
|
if (purchase.Preview[i] is long previewId)
|
|
{
|
|
if (!previewids.Contains(previewId))
|
|
{
|
|
previewids.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
DateTime createdAt = purchase.CreatedAt ?? purchase.PostedAt ?? DateTime.Now;
|
|
bool isArchived = purchase.IsArchived ?? false;
|
|
await dbService.AddPost(folder, purchase.Id, purchase.Text ?? "",
|
|
purchase.Price ?? "0",
|
|
purchase.Price != null && purchase.IsOpened, isArchived, createdAt);
|
|
paidPostCollection.PaidPostObjects.Add(purchase);
|
|
foreach (MessageEntities.Medium medium in purchase.Media)
|
|
{
|
|
if (!previewids.Contains(medium.Id))
|
|
{
|
|
paid_post_ids.Add(medium.Id);
|
|
}
|
|
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string mediaType = ResolveMediaType(medium.Type) ?? "";
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = previewids.Contains(medium.Id);
|
|
|
|
if (previewids.Count > 0)
|
|
{
|
|
bool has = previewids.Any(cus => cus.Equals(medium.Id));
|
|
if (!has && medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, purchase.Id, fullUrl,
|
|
null, null, null, "Posts",
|
|
mediaType, isPreview, false, null);
|
|
paidPostCollection.PaidPosts.Add(medium.Id, fullUrl);
|
|
paidPostCollection.PaidPostMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (!has && medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, purchase.Id,
|
|
manifestDash, null, null, null, "Posts", mediaType, isPreview, false, null);
|
|
paidPostCollection.PaidPosts.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
|
|
paidPostCollection.PaidPostMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, purchase.Id, fullUrl,
|
|
null, null, null, "Posts",
|
|
mediaType, isPreview, false, null);
|
|
paidPostCollection.PaidPosts.Add(medium.Id, fullUrl);
|
|
paidPostCollection.PaidPostMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, purchase.Id,
|
|
manifestDash, null, null, null, "Posts", mediaType, isPreview, false, null);
|
|
paidPostCollection.PaidPosts.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
|
|
paidPostCollection.PaidPostMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return paidPostCollection;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return new PurchasedEntities.PaidPostCollection();
|
|
}
|
|
|
|
|
|
public async Task<PostEntities.PostCollection> GetPosts(string endpoint, string folder, List<long> paid_post_ids,
|
|
IStatusReporter statusReporter)
|
|
{
|
|
Log.Debug($"Calling GetPosts - {endpoint}");
|
|
|
|
try
|
|
{
|
|
PostEntities.PostCollection postCollection = new();
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "limit", postLimit.ToString() },
|
|
{ "order", "publish_date_desc" },
|
|
{ "format", "infinite" },
|
|
{ "skip_users", "all" }
|
|
};
|
|
|
|
DownloadDateSelection downloadDateSelection = DownloadDateSelection.before;
|
|
DateTime? downloadAsOf = null;
|
|
|
|
if (configService.CurrentConfig.DownloadOnlySpecificDates &&
|
|
configService.CurrentConfig.CustomDate.HasValue)
|
|
{
|
|
downloadDateSelection = configService.CurrentConfig.DownloadDateSelection;
|
|
downloadAsOf = configService.CurrentConfig.CustomDate;
|
|
}
|
|
else if (configService.CurrentConfig.DownloadPostsIncrementally)
|
|
{
|
|
DateTime? mostRecentPostDate = await dbService.GetMostRecentPostDate(folder);
|
|
if (mostRecentPostDate.HasValue)
|
|
{
|
|
downloadDateSelection = DownloadDateSelection.after;
|
|
downloadAsOf = mostRecentPostDate.Value.AddMinutes(-5); // Back track a little for a margin of error
|
|
}
|
|
}
|
|
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
downloadAsOf);
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
|
PostDtos.PostDto? postsDto =
|
|
DeserializeJson<PostDtos.PostDto>(body, s_mJsonSerializerSettings);
|
|
PostEntities.Post posts = PostMapper.FromDto(postsDto);
|
|
statusReporter.ReportStatus($"Getting Posts - Found {posts.List.Count}");
|
|
if (posts.HasMore)
|
|
{
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
posts.TailMarker);
|
|
|
|
while (true)
|
|
{
|
|
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
PostDtos.PostDto? newPostsDto =
|
|
DeserializeJson<PostDtos.PostDto>(loopbody, s_mJsonSerializerSettings);
|
|
PostEntities.Post newposts = PostMapper.FromDto(newPostsDto);
|
|
|
|
posts.List.AddRange(newposts.List);
|
|
statusReporter.ReportStatus($"Getting Posts - Found {posts.List.Count}");
|
|
if (!newposts.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
newposts.TailMarker);
|
|
}
|
|
}
|
|
|
|
List<PostEntities.ListItem> postList = posts.List;
|
|
foreach (PostEntities.ListItem post in postList)
|
|
{
|
|
if (configService.CurrentConfig.SkipAds)
|
|
{
|
|
if (!string.IsNullOrEmpty(post.RawText) &&
|
|
(post.RawText.Contains("#ad") || post.RawText.Contains("/trial/") ||
|
|
post.RawText.Contains("#announcement")))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(post.Text) &&
|
|
(post.Text.Contains("#ad") || post.Text.Contains("/trial/") ||
|
|
post.Text.Contains("#announcement")))
|
|
{
|
|
continue;
|
|
}
|
|
}
|
|
|
|
List<long> postPreviewIds = new();
|
|
if (post.Preview != null && post.Preview.Count > 0)
|
|
{
|
|
for (int i = 0; i < post.Preview.Count; i++)
|
|
{
|
|
if (post.Preview[i] is long previewId)
|
|
{
|
|
if (!postPreviewIds.Contains(previewId))
|
|
{
|
|
postPreviewIds.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
await dbService.AddPost(folder, post.Id, !string.IsNullOrEmpty(post.RawText) ? post.RawText : "",
|
|
post.Price ?? "0", post.Price != null && post.IsOpened,
|
|
post.IsArchived, post.PostedAt);
|
|
postCollection.PostObjects.Add(post);
|
|
if (post.Media != null && post.Media.Count > 0)
|
|
{
|
|
foreach (PostEntities.Medium medium in post.Media)
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
string? previewUrl = medium.Files?.Preview?.Url;
|
|
bool isPreview = postPreviewIds.Contains(medium.Id);
|
|
|
|
if (medium.CanView && medium.Files?.Drm == null)
|
|
{
|
|
bool has = paid_post_ids.Any(cus => cus.Equals(medium.Id));
|
|
if (!has && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (!postCollection.Posts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, post.Id, fullUrl, null, null, null,
|
|
"Posts", mediaType, isPreview, false, null);
|
|
postCollection.Posts.Add(medium.Id, fullUrl);
|
|
postCollection.PostMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (!has && string.IsNullOrEmpty(fullUrl) && !string.IsNullOrEmpty(previewUrl))
|
|
{
|
|
if (!postCollection.Posts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, post.Id, previewUrl, null, null, null,
|
|
"Posts", mediaType, isPreview, false, null);
|
|
postCollection.Posts.Add(medium.Id, previewUrl);
|
|
postCollection.PostMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
else if (medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
|
|
out string cloudFrontSignature, out string cloudFrontKeyPairId))
|
|
{
|
|
bool has = paid_post_ids.Any(cus => cus.Equals(medium.Id));
|
|
if (!has && !postCollection.Posts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, post.Id, manifestDash, null, null, null,
|
|
"Posts", mediaType, isPreview, false, null);
|
|
postCollection.Posts.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{post.Id}");
|
|
postCollection.PostMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return postCollection;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return new PostEntities.PostCollection();
|
|
}
|
|
|
|
public async Task<SinglePostCollection> GetPost(string endpoint, string folder)
|
|
{
|
|
Log.Debug($"Calling GetPost - {endpoint}");
|
|
|
|
try
|
|
{
|
|
SinglePostCollection singlePostCollection = new();
|
|
Dictionary<string, string> getParams = new() { { "skip_users", "all" } };
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
|
PostDtos.SinglePostDto? singlePostDto =
|
|
DeserializeJson<PostDtos.SinglePostDto>(body, s_mJsonSerializerSettings);
|
|
PostEntities.SinglePost singlePost = PostMapper.FromDto(singlePostDto);
|
|
|
|
if (singlePostDto != null)
|
|
{
|
|
List<long> postPreviewIds = new();
|
|
if (singlePost.Preview != null && singlePost.Preview.Count > 0)
|
|
{
|
|
for (int i = 0; i < singlePost.Preview.Count; i++)
|
|
{
|
|
if (singlePost.Preview[i] is long previewId)
|
|
{
|
|
if (!postPreviewIds.Contains(previewId))
|
|
{
|
|
postPreviewIds.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
await dbService.AddPost(folder, singlePost.Id,
|
|
!string.IsNullOrEmpty(singlePost.Text) ? singlePost.Text : "",
|
|
singlePost.Price ?? "0",
|
|
singlePost.Price != null && singlePost.IsOpened, singlePost.IsArchived,
|
|
singlePost.PostedAt);
|
|
singlePostCollection.SinglePostObjects.Add(singlePost);
|
|
if (singlePost.Media != null && singlePost.Media.Count > 0)
|
|
{
|
|
foreach (PostEntities.Medium medium in singlePost.Media)
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
bool isPreview = postPreviewIds.Contains(medium.Id);
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
string? previewUrl = medium.Files?.Preview?.Url;
|
|
|
|
if (medium.CanView && medium.Files?.Drm == null)
|
|
{
|
|
switch (configService.CurrentConfig.DownloadVideoResolution)
|
|
{
|
|
case VideoResolution.source:
|
|
if (!string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, singlePost.Id,
|
|
fullUrl, null, null, null, "Posts", mediaType, isPreview, false, null);
|
|
singlePostCollection.SinglePosts.Add(medium.Id, fullUrl);
|
|
singlePostCollection.SinglePostMedia.Add(medium);
|
|
}
|
|
}
|
|
|
|
break;
|
|
case VideoResolution._240:
|
|
string? video240 = medium.VideoSources?._240;
|
|
if (!string.IsNullOrEmpty(video240))
|
|
{
|
|
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, singlePost.Id, video240, null,
|
|
null, null, "Posts", mediaType, isPreview, false, null);
|
|
singlePostCollection.SinglePosts.Add(medium.Id, video240);
|
|
singlePostCollection.SinglePostMedia.Add(medium);
|
|
}
|
|
}
|
|
|
|
break;
|
|
case VideoResolution._720:
|
|
string? video720 = medium.VideoSources?._720;
|
|
if (!string.IsNullOrEmpty(video720))
|
|
{
|
|
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, singlePost.Id, video720, null,
|
|
null, null, "Posts", mediaType, isPreview, false, null);
|
|
singlePostCollection.SinglePosts.Add(medium.Id, video720);
|
|
singlePostCollection.SinglePostMedia.Add(medium);
|
|
}
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
else if (medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
|
|
out string cloudFrontSignature, out string cloudFrontKeyPairId))
|
|
{
|
|
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, singlePost.Id, manifestDash, null, null,
|
|
null, "Posts", mediaType, isPreview, false, null);
|
|
singlePostCollection.SinglePosts.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{singlePost.Id}");
|
|
singlePostCollection.SinglePostMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (!string.IsNullOrEmpty(previewUrl) && medium.Files?.Full == null)
|
|
{
|
|
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, singlePost.Id, previewUrl, null, null, null,
|
|
"Posts", mediaType, isPreview, false, null);
|
|
singlePostCollection.SinglePosts.Add(medium.Id, previewUrl);
|
|
singlePostCollection.SinglePostMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return singlePostCollection;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return new SinglePostCollection();
|
|
}
|
|
|
|
public async Task<StreamEntities.StreamsCollection> GetStreams(string endpoint, string folder,
|
|
List<long> paid_post_ids,
|
|
IStatusReporter statusReporter)
|
|
{
|
|
Log.Debug($"Calling GetStreams - {endpoint}");
|
|
|
|
try
|
|
{
|
|
StreamEntities.StreamsCollection streamsCollection = new();
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "limit", postLimit.ToString() },
|
|
{ "order", "publish_date_desc" },
|
|
{ "format", "infinite" },
|
|
{ "skip_users", "all" }
|
|
};
|
|
|
|
DownloadDateSelection downloadDateSelection = DownloadDateSelection.before;
|
|
if (configService.CurrentConfig.DownloadOnlySpecificDates &&
|
|
configService.CurrentConfig.CustomDate.HasValue)
|
|
{
|
|
downloadDateSelection = configService.CurrentConfig.DownloadDateSelection;
|
|
}
|
|
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
configService.CurrentConfig.CustomDate);
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
|
StreamsDtos.StreamsDto? streamsDto =
|
|
DeserializeJson<StreamsDtos.StreamsDto>(body, s_mJsonSerializerSettings);
|
|
StreamEntities.Streams streams = StreamsMapper.FromDto(streamsDto);
|
|
statusReporter.ReportStatus($"Getting Streams - Found {streams.List.Count}");
|
|
if (streams.HasMore)
|
|
{
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
streams.TailMarker);
|
|
|
|
while (true)
|
|
{
|
|
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
StreamsDtos.StreamsDto? newStreamsDto =
|
|
DeserializeJson<StreamsDtos.StreamsDto>(loopbody, s_mJsonSerializerSettings);
|
|
StreamEntities.Streams newstreams = StreamsMapper.FromDto(newStreamsDto);
|
|
|
|
streams.List.AddRange(newstreams.List);
|
|
statusReporter.ReportStatus($"Getting Streams - Found {streams.List.Count}");
|
|
if (!newstreams.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
newstreams.TailMarker);
|
|
}
|
|
}
|
|
|
|
List<StreamEntities.ListItem> streamList = streams.List;
|
|
foreach (StreamEntities.ListItem stream in streamList)
|
|
{
|
|
List<long> streamPreviewIds = new();
|
|
if (stream.Preview != null && stream.Preview.Count > 0)
|
|
{
|
|
for (int i = 0; i < stream.Preview.Count; i++)
|
|
{
|
|
if (stream.Preview[i] is long previewId)
|
|
{
|
|
if (!streamPreviewIds.Contains(previewId))
|
|
{
|
|
streamPreviewIds.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
await dbService.AddPost(folder, stream.Id, !string.IsNullOrEmpty(stream.Text) ? stream.Text : "",
|
|
stream.Price ?? "0", stream.Price != null && stream.IsOpened,
|
|
stream.IsArchived, stream.PostedAt);
|
|
streamsCollection.StreamObjects.Add(stream);
|
|
if (stream.Media != null && stream.Media.Count > 0)
|
|
{
|
|
foreach (StreamEntities.Medium medium in stream.Media)
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = streamPreviewIds.Contains(medium.Id);
|
|
|
|
if (medium.CanView && medium.Files?.Drm == null)
|
|
{
|
|
bool has = paid_post_ids.Any(cus => cus.Equals(medium.Id));
|
|
if (!has && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (!streamsCollection.Streams.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, stream.Id, fullUrl, null, null, null,
|
|
"Posts", mediaType, isPreview, false, null);
|
|
streamsCollection.Streams.Add(medium.Id, fullUrl);
|
|
streamsCollection.StreamMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
else if (medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
|
|
out string cloudFrontSignature, out string cloudFrontKeyPairId))
|
|
{
|
|
bool has = paid_post_ids.Any(cus => cus.Equals(medium.Id));
|
|
if (!has && !streamsCollection.Streams.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, stream.Id, manifestDash, null, null, null,
|
|
"Posts", mediaType, isPreview, false, null);
|
|
streamsCollection.Streams.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{stream.Id}");
|
|
streamsCollection.StreamMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return streamsCollection;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return new StreamEntities.StreamsCollection();
|
|
}
|
|
|
|
|
|
public async Task<ArchivedEntities.ArchivedCollection> GetArchived(string endpoint, string folder,
|
|
IStatusReporter statusReporter)
|
|
{
|
|
Log.Debug($"Calling GetArchived - {endpoint}");
|
|
|
|
try
|
|
{
|
|
ArchivedEntities.ArchivedCollection archivedCollection = new();
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "limit", postLimit.ToString() },
|
|
{ "order", "publish_date_desc" },
|
|
{ "skip_users", "all" },
|
|
{ "format", "infinite" },
|
|
{ "label", "archived" },
|
|
{ "counters", "1" }
|
|
};
|
|
|
|
DownloadDateSelection downloadDateSelection = DownloadDateSelection.before;
|
|
if (configService.CurrentConfig.DownloadOnlySpecificDates &&
|
|
configService.CurrentConfig.CustomDate.HasValue)
|
|
{
|
|
downloadDateSelection = configService.CurrentConfig.DownloadDateSelection;
|
|
}
|
|
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
configService.CurrentConfig.CustomDate);
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
if (body == null)
|
|
{
|
|
throw new Exception("Failed to retrieve archived posts. Received null response.");
|
|
}
|
|
|
|
ArchivedDtos.ArchivedDto? archivedDto =
|
|
DeserializeJson<ArchivedDtos.ArchivedDto>(body, s_mJsonSerializerSettings);
|
|
ArchivedEntities.Archived archived = ArchivedMapper.FromDto(archivedDto);
|
|
statusReporter.ReportStatus($"Getting Archived Posts - Found {archived.List.Count}");
|
|
if (archived.HasMore)
|
|
{
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
archived.TailMarker);
|
|
while (true)
|
|
{
|
|
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
if (loopbody == null)
|
|
{
|
|
throw new Exception("Failed to retrieve archived posts. Received null response.");
|
|
}
|
|
|
|
ArchivedDtos.ArchivedDto? newarchivedDto =
|
|
DeserializeJson<ArchivedDtos.ArchivedDto>(loopbody, s_mJsonSerializerSettings);
|
|
ArchivedEntities.Archived newarchived = ArchivedMapper.FromDto(newarchivedDto);
|
|
|
|
archived.List.AddRange(newarchived.List);
|
|
statusReporter.ReportStatus($"Getting Archived Posts - Found {archived.List.Count}");
|
|
if (!newarchived.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
newarchived.TailMarker);
|
|
}
|
|
}
|
|
|
|
foreach (ArchivedEntities.ListItem archive in archived.List)
|
|
{
|
|
List<long> previewids = new();
|
|
if (archive.Preview != null)
|
|
{
|
|
for (int i = 0; i < archive.Preview.Count; i++)
|
|
{
|
|
if (archive.Preview[i] is long previewId)
|
|
{
|
|
if (!previewids.Contains(previewId))
|
|
{
|
|
previewids.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
await dbService.AddPost(folder, archive.Id, archive.Text != null ? archive.Text : "",
|
|
archive.Price ?? "0",
|
|
archive.Price != null && archive.IsOpened, archive.IsArchived, archive.PostedAt);
|
|
archivedCollection.ArchivedPostObjects.Add(archive);
|
|
if (archive.Media != null && archive.Media.Count > 0)
|
|
{
|
|
foreach (ArchivedEntities.Medium medium in archive.Media)
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = previewids.Contains(medium.Id);
|
|
|
|
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (!archivedCollection.ArchivedPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, archive.Id, fullUrl, null, null, null,
|
|
"Posts", mediaType, isPreview, false, null);
|
|
archivedCollection.ArchivedPosts.Add(medium.Id, fullUrl);
|
|
archivedCollection.ArchivedPostMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
|
|
out string cloudFrontSignature, out string cloudFrontKeyPairId))
|
|
{
|
|
if (!archivedCollection.ArchivedPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, archive.Id, manifestDash, null, null, null,
|
|
"Posts", mediaType, isPreview, false, null);
|
|
archivedCollection.ArchivedPosts.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{archive.Id}");
|
|
archivedCollection.ArchivedPostMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return archivedCollection;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return new ArchivedEntities.ArchivedCollection();
|
|
}
|
|
|
|
|
|
public async Task<MessageEntities.MessageCollection> GetMessages(string endpoint, string folder,
|
|
IStatusReporter statusReporter)
|
|
{
|
|
Log.Debug($"Calling GetMessages - {endpoint}");
|
|
|
|
try
|
|
{
|
|
MessageEntities.MessageCollection messageCollection = new();
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "limit", postLimit.ToString() }, { "order", "desc" }, { "skip_users", "all" }
|
|
};
|
|
int currentUserId = GetCurrentUserIdOrDefault();
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
MessageDtos.MessagesDto? messagesDto =
|
|
DeserializeJson<MessageDtos.MessagesDto>(body, s_mJsonSerializerSettings);
|
|
MessageEntities.Messages messages = MessagesMapper.FromDto(messagesDto);
|
|
statusReporter.ReportStatus($"Getting Messages - Found {messages.List.Count}");
|
|
if (messages.HasMore)
|
|
{
|
|
getParams["id"] = messages.List[^1].Id.ToString();
|
|
while (true)
|
|
{
|
|
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
MessageDtos.MessagesDto? newMessagesDto =
|
|
DeserializeJson<MessageDtos.MessagesDto>(loopbody, s_mJsonSerializerSettings);
|
|
MessageEntities.Messages newMessages = MessagesMapper.FromDto(newMessagesDto);
|
|
|
|
messages.List.AddRange(newMessages.List);
|
|
statusReporter.ReportStatus($"Getting Messages - Found {messages.List.Count}");
|
|
if (!newMessages.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
getParams["id"] = newMessages.List[newMessages.List.Count - 1].Id.ToString();
|
|
}
|
|
}
|
|
|
|
foreach (MessageEntities.ListItem list in messages.List)
|
|
{
|
|
if (configService.CurrentConfig.SkipAds)
|
|
{
|
|
if (!string.IsNullOrEmpty(list.Text) &&
|
|
(list.Text.Contains("#ad") || list.Text.Contains("/trial/")))
|
|
{
|
|
continue;
|
|
}
|
|
}
|
|
|
|
List<long> messagePreviewIds = new();
|
|
if (list.Previews != null && list.Previews.Count > 0)
|
|
{
|
|
for (int i = 0; i < list.Previews.Count; i++)
|
|
{
|
|
if (list.Previews[i] is long previewId)
|
|
{
|
|
if (!messagePreviewIds.Contains(previewId))
|
|
{
|
|
messagePreviewIds.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!configService.CurrentConfig.IgnoreOwnMessages || list.FromUser?.Id != currentUserId)
|
|
{
|
|
DateTime createdAt = list.CreatedAt ?? DateTime.Now;
|
|
await dbService.AddMessage(folder, list.Id, list.Text ?? "", list.Price ?? "0",
|
|
list.CanPurchaseReason == "opened" ? true :
|
|
list.CanPurchaseReason != "opened" ? false : (bool?)null ?? false, false,
|
|
createdAt,
|
|
list.FromUser?.Id ?? int.MinValue);
|
|
messageCollection.MessageObjects.Add(list);
|
|
if (list.CanPurchaseReason != "opened" && list.Media != null && list.Media.Count > 0)
|
|
{
|
|
foreach (MessageEntities.Medium medium in list.Media ?? new List<MessageEntities.Medium>())
|
|
{
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = messagePreviewIds.Contains(medium.Id);
|
|
|
|
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!messageCollection.Messages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, list.Id, fullUrl, null, null, null,
|
|
"Messages", mediaType, isPreview, false, null);
|
|
messageCollection.Messages.Add(medium.Id, fullUrl);
|
|
messageCollection.MessageMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!messageCollection.Messages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, list.Id, manifestDash, null, null, null,
|
|
"Messages", mediaType, isPreview, false, null);
|
|
messageCollection.Messages.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{list.Id}");
|
|
messageCollection.MessageMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (messagePreviewIds.Count > 0)
|
|
{
|
|
foreach (MessageEntities.Medium medium in list.Media ?? new List<MessageEntities.Medium>())
|
|
{
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = messagePreviewIds.Contains(medium.Id);
|
|
|
|
if (medium.CanView && !string.IsNullOrEmpty(fullUrl) && isPreview)
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!messageCollection.Messages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, list.Id, fullUrl, null, null, null,
|
|
"Messages", mediaType, isPreview, false, null);
|
|
messageCollection.Messages.Add(medium.Id, fullUrl);
|
|
messageCollection.MessageMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (medium.CanView && isPreview &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!messageCollection.Messages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, list.Id, manifestDash, null, null, null,
|
|
"Messages", mediaType, isPreview, false, null);
|
|
messageCollection.Messages.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{list.Id}");
|
|
messageCollection.MessageMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return messageCollection;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return new MessageEntities.MessageCollection();
|
|
}
|
|
|
|
public async Task<PurchasedEntities.SinglePaidMessageCollection> GetPaidMessage(string endpoint, string folder)
|
|
{
|
|
Log.Debug($"Calling GetPaidMessage - {endpoint}");
|
|
|
|
try
|
|
{
|
|
PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection = new();
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new() { { "limit", postLimit.ToString() }, { "order", "desc" } };
|
|
int currentUserId = GetCurrentUserIdOrDefault();
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
MessageDtos.SingleMessageDto? messageDto =
|
|
DeserializeJson<MessageDtos.SingleMessageDto>(body, s_mJsonSerializerSettings);
|
|
MessageEntities.SingleMessage message = MessagesMapper.FromDto(messageDto);
|
|
|
|
if (!configService.CurrentConfig.IgnoreOwnMessages || message.FromUser?.Id != currentUserId)
|
|
{
|
|
DateTime createdAt = message.CreatedAt ?? DateTime.Now;
|
|
await dbService.AddMessage(folder, message.Id, message.Text ?? "",
|
|
message.Price?.ToString() ?? "0", true, false,
|
|
createdAt,
|
|
message.FromUser?.Id ?? int.MinValue);
|
|
singlePaidMessageCollection.SingleMessageObjects.Add(message);
|
|
List<long> messagePreviewIds = new();
|
|
if (message.Previews != null && message.Previews.Count > 0)
|
|
{
|
|
for (int i = 0; i < message.Previews.Count; i++)
|
|
{
|
|
if (message.Previews[i] is long previewId)
|
|
{
|
|
if (!messagePreviewIds.Contains(previewId))
|
|
{
|
|
messagePreviewIds.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (message.Media != null && message.Media.Count > 0)
|
|
{
|
|
foreach (MessageEntities.Medium medium in message.Media)
|
|
{
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = messagePreviewIds.Contains(medium.Id);
|
|
|
|
if (!isPreview && medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!singlePaidMessageCollection.SingleMessages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, message.Id, fullUrl, null, null, null,
|
|
"Messages", mediaType, isPreview, false, null);
|
|
singlePaidMessageCollection.SingleMessages.Add(medium.Id, fullUrl);
|
|
singlePaidMessageCollection.SingleMessageMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (isPreview && medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!singlePaidMessageCollection.PreviewSingleMessages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, message.Id, fullUrl, null, null, null,
|
|
"Messages", mediaType, isPreview, false, null);
|
|
singlePaidMessageCollection.PreviewSingleMessages.Add(medium.Id, fullUrl);
|
|
singlePaidMessageCollection.PreviewSingleMessageMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (!isPreview && medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
|
|
out string cloudFrontSignature, out string cloudFrontKeyPairId))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!singlePaidMessageCollection.SingleMessages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, message.Id, manifestDash, null, null, null,
|
|
"Messages", mediaType, isPreview, false, null);
|
|
singlePaidMessageCollection.SingleMessages.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{message.Id}");
|
|
singlePaidMessageCollection.SingleMessageMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (isPreview && medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string previewManifestDash,
|
|
out string previewCloudFrontPolicy, out string previewCloudFrontSignature,
|
|
out string previewCloudFrontKeyPairId))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!singlePaidMessageCollection.PreviewSingleMessages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, message.Id, previewManifestDash, null, null,
|
|
null, "Messages", mediaType, isPreview, false, null);
|
|
singlePaidMessageCollection.PreviewSingleMessages.Add(medium.Id,
|
|
$"{previewManifestDash},{previewCloudFrontPolicy},{previewCloudFrontSignature},{previewCloudFrontKeyPairId},{medium.Id},{message.Id}");
|
|
singlePaidMessageCollection.PreviewSingleMessageMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return singlePaidMessageCollection;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return new PurchasedEntities.SinglePaidMessageCollection();
|
|
}
|
|
|
|
|
|
public async Task<PurchasedEntities.PaidMessageCollection> GetPaidMessages(string endpoint, string folder,
|
|
string username,
|
|
IStatusReporter statusReporter)
|
|
{
|
|
Log.Debug($"Calling GetPaidMessages - {username}");
|
|
|
|
try
|
|
{
|
|
PurchasedEntities.PaidMessageCollection paidMessageCollection = new();
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "limit", postLimit.ToString() },
|
|
{ "order", "publish_date_desc" },
|
|
{ "format", "infinite" },
|
|
{ "author", username },
|
|
{ "skip_users", "all" }
|
|
};
|
|
int currentUserId = GetCurrentUserIdOrDefault();
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
PurchasedDtos.PurchasedDto? paidMessagesDto =
|
|
DeserializeJson<PurchasedDtos.PurchasedDto>(body, s_mJsonSerializerSettings);
|
|
PurchasedEntities.Purchased paidMessages = PurchasedMapper.FromDto(paidMessagesDto);
|
|
statusReporter.ReportStatus($"Getting Paid Messages - Found {paidMessages.List.Count}");
|
|
if (paidMessages.HasMore)
|
|
{
|
|
getParams["offset"] = paidMessages.List.Count.ToString();
|
|
while (true)
|
|
{
|
|
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
|
PurchasedEntities.Purchased newpaidMessages;
|
|
Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
|
|
HttpClient loopclient = GetHttpClient();
|
|
|
|
HttpRequestMessage looprequest =
|
|
new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
|
|
|
|
foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
|
|
{
|
|
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
|
|
}
|
|
|
|
using (HttpResponseMessage loopresponse = await loopclient.SendAsync(looprequest))
|
|
{
|
|
loopresponse.EnsureSuccessStatusCode();
|
|
string loopbody = await loopresponse.Content.ReadAsStringAsync();
|
|
PurchasedDtos.PurchasedDto? newPaidMessagesDto =
|
|
DeserializeJson<PurchasedDtos.PurchasedDto>(loopbody, s_mJsonSerializerSettings);
|
|
newpaidMessages = PurchasedMapper.FromDto(newPaidMessagesDto);
|
|
}
|
|
|
|
paidMessages.List.AddRange(newpaidMessages.List);
|
|
statusReporter.ReportStatus($"Getting Paid Messages - Found {paidMessages.List.Count}");
|
|
if (!newpaidMessages.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + postLimit);
|
|
}
|
|
}
|
|
|
|
List<PurchasedEntities.ListItem> paidMessageList =
|
|
paidMessages.List;
|
|
if (paidMessageList.Count > 0)
|
|
{
|
|
foreach (PurchasedEntities.ListItem purchase in paidMessageList
|
|
.Where(p => p.ResponseType == "message")
|
|
.OrderByDescending(p => p.PostedAt ?? p.CreatedAt))
|
|
{
|
|
long fromUserId = purchase.FromUser?.Id ?? long.MinValue;
|
|
if (!configService.CurrentConfig.IgnoreOwnMessages || fromUserId != currentUserId)
|
|
{
|
|
DateTime createdAt = purchase.PostedAt ?? purchase.CreatedAt ?? DateTime.Now;
|
|
await dbService.AddMessage(folder, purchase.Id,
|
|
purchase.Text ?? "",
|
|
purchase.Price ?? "0", true, false, createdAt,
|
|
fromUserId);
|
|
|
|
paidMessageCollection.PaidMessageObjects.Add(purchase);
|
|
if (purchase.Media != null && purchase.Media.Count > 0)
|
|
{
|
|
List<long> previewids = new();
|
|
if (purchase.Previews != null)
|
|
{
|
|
for (int i = 0; i < purchase.Previews.Count; i++)
|
|
{
|
|
if (purchase.Previews[i] is long previewId)
|
|
{
|
|
if (!previewids.Contains(previewId))
|
|
{
|
|
previewids.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (purchase.Preview != null)
|
|
{
|
|
for (int i = 0; i < purchase.Preview.Count; i++)
|
|
{
|
|
if (purchase.Preview[i] is long previewId)
|
|
{
|
|
if (!previewids.Contains(previewId))
|
|
{
|
|
previewids.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach (MessageEntities.Medium medium in purchase.Media)
|
|
{
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = previewids.Contains(medium.Id);
|
|
|
|
if (previewids.Count > 0)
|
|
{
|
|
bool has = previewids.Any(cus => cus.Equals(medium.Id));
|
|
if (!has && medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, purchase.Id,
|
|
fullUrl, null, null, null, "Messages", mediaType, isPreview, false,
|
|
null);
|
|
paidMessageCollection.PaidMessages.Add(medium.Id, fullUrl);
|
|
paidMessageCollection.PaidMessageMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (!has && medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, purchase.Id,
|
|
manifestDash, null, null, null, "Messages", mediaType, isPreview, false,
|
|
null);
|
|
paidMessageCollection.PaidMessages.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
|
|
paidMessageCollection.PaidMessageMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, purchase.Id,
|
|
fullUrl, null, null, null, "Messages", mediaType, isPreview, false,
|
|
null);
|
|
paidMessageCollection.PaidMessages.Add(medium.Id, fullUrl);
|
|
paidMessageCollection.PaidMessageMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(folder, medium.Id, purchase.Id,
|
|
manifestDash, null, null, null, "Messages", mediaType, isPreview, false,
|
|
null);
|
|
paidMessageCollection.PaidMessages.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
|
|
paidMessageCollection.PaidMessageMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return paidMessageCollection;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return new PurchasedEntities.PaidMessageCollection();
|
|
}
|
|
|
|
public async Task<Dictionary<string, long>> GetPurchasedTabUsers(string endpoint, Dictionary<string, long> users)
|
|
{
|
|
Log.Debug($"Calling GetPurchasedTabUsers - {endpoint}");
|
|
|
|
try
|
|
{
|
|
Dictionary<string, long> purchasedTabUsers = new();
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "limit", postLimit.ToString() },
|
|
{ "order", "publish_date_desc" },
|
|
{ "format", "infinite" },
|
|
{ "skip_users", "all" }
|
|
};
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
if (body == null)
|
|
{
|
|
throw new Exception("Failed to get purchased tab users. null body returned.");
|
|
}
|
|
|
|
PurchasedDtos.PurchasedDto? purchasedDto =
|
|
DeserializeJson<PurchasedDtos.PurchasedDto>(body, s_mJsonSerializerSettings);
|
|
PurchasedEntities.Purchased purchased = PurchasedMapper.FromDto(purchasedDto);
|
|
if (purchased.HasMore)
|
|
{
|
|
getParams["offset"] = purchased.List.Count.ToString();
|
|
while (true)
|
|
{
|
|
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
|
PurchasedEntities.Purchased newPurchased;
|
|
Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
|
|
HttpClient loopclient = GetHttpClient();
|
|
|
|
HttpRequestMessage looprequest =
|
|
new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
|
|
|
|
foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
|
|
{
|
|
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
|
|
}
|
|
|
|
using (HttpResponseMessage loopresponse = await loopclient.SendAsync(looprequest))
|
|
{
|
|
loopresponse.EnsureSuccessStatusCode();
|
|
string loopbody = await loopresponse.Content.ReadAsStringAsync();
|
|
PurchasedDtos.PurchasedDto? newPurchasedDto =
|
|
DeserializeJson<PurchasedDtos.PurchasedDto>(loopbody, s_mJsonSerializerSettings);
|
|
newPurchased = PurchasedMapper.FromDto(newPurchasedDto);
|
|
}
|
|
|
|
purchased.List.AddRange(newPurchased.List);
|
|
if (!newPurchased.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + postLimit);
|
|
}
|
|
}
|
|
|
|
if (purchased.List.Count > 0)
|
|
{
|
|
foreach (PurchasedEntities.ListItem purchase in
|
|
purchased.List.OrderByDescending(p => p.PostedAt ?? p.CreatedAt))
|
|
{
|
|
long fromUserId = purchase.FromUser?.Id ?? 0;
|
|
long authorId = purchase.Author?.Id ?? 0;
|
|
|
|
if (fromUserId != 0)
|
|
{
|
|
if (users.Values.Contains(fromUserId))
|
|
{
|
|
string? matchedUsername = users.FirstOrDefault(x => x.Value == fromUserId).Key;
|
|
if (!string.IsNullOrEmpty(matchedUsername))
|
|
{
|
|
if (!purchasedTabUsers.ContainsKey(matchedUsername))
|
|
{
|
|
purchasedTabUsers.Add(matchedUsername, fromUserId);
|
|
}
|
|
}
|
|
else if (!purchasedTabUsers.ContainsKey($"Deleted User - {fromUserId}"))
|
|
{
|
|
purchasedTabUsers.Add($"Deleted User - {fromUserId}", fromUserId);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
JObject? user = await GetUserInfoById($"/users/list?x[]={fromUserId}");
|
|
string? fetchedUsername = user?[fromUserId.ToString()]?["username"]?.ToString();
|
|
|
|
if (string.IsNullOrEmpty(fetchedUsername))
|
|
{
|
|
if (!configService.CurrentConfig.BypassContentForCreatorsWhoNoLongerExist &&
|
|
!purchasedTabUsers.ContainsKey($"Deleted User - {fromUserId}"))
|
|
{
|
|
purchasedTabUsers.Add($"Deleted User - {fromUserId}", fromUserId);
|
|
}
|
|
|
|
Log.Debug("Content creator not longer exists - {0}", fromUserId);
|
|
}
|
|
else if (!purchasedTabUsers.ContainsKey(fetchedUsername))
|
|
{
|
|
purchasedTabUsers.Add(fetchedUsername, fromUserId);
|
|
}
|
|
}
|
|
}
|
|
else if (authorId != 0)
|
|
{
|
|
if (users.ContainsValue(authorId))
|
|
{
|
|
string? matchedUsername = users.FirstOrDefault(x => x.Value == authorId).Key;
|
|
if (!string.IsNullOrEmpty(matchedUsername))
|
|
{
|
|
if (!purchasedTabUsers.ContainsKey(matchedUsername) &&
|
|
users.ContainsKey(matchedUsername))
|
|
{
|
|
purchasedTabUsers.Add(matchedUsername, authorId);
|
|
}
|
|
}
|
|
else if (!purchasedTabUsers.ContainsKey($"Deleted User - {authorId}"))
|
|
{
|
|
purchasedTabUsers.Add($"Deleted User - {authorId}", authorId);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
JObject? user = await GetUserInfoById($"/users/list?x[]={authorId}");
|
|
string? fetchedUsername = user?[authorId.ToString()]?["username"]?.ToString();
|
|
|
|
if (string.IsNullOrEmpty(fetchedUsername))
|
|
{
|
|
if (!configService.CurrentConfig.BypassContentForCreatorsWhoNoLongerExist &&
|
|
!purchasedTabUsers.ContainsKey($"Deleted User - {authorId}"))
|
|
{
|
|
purchasedTabUsers.Add($"Deleted User - {authorId}", authorId);
|
|
}
|
|
|
|
Log.Debug("Content creator not longer exists - {0}", authorId);
|
|
}
|
|
else if (!purchasedTabUsers.ContainsKey(fetchedUsername) &&
|
|
users.ContainsKey(fetchedUsername))
|
|
{
|
|
purchasedTabUsers.Add(fetchedUsername, authorId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return purchasedTabUsers;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return new Dictionary<string, long>();
|
|
}
|
|
|
|
public async Task<List<PurchasedEntities.PurchasedTabCollection>> GetPurchasedTab(string endpoint, string folder,
|
|
Dictionary<string, long> users)
|
|
{
|
|
Log.Debug($"Calling GetPurchasedTab - {endpoint}");
|
|
|
|
try
|
|
{
|
|
Dictionary<long, List<PurchasedEntities.ListItem>> userPurchases = new();
|
|
List<PurchasedEntities.PurchasedTabCollection> purchasedTabCollections = [];
|
|
const int postLimit = 50;
|
|
Dictionary<string, string> getParams = new()
|
|
{
|
|
{ "limit", postLimit.ToString() },
|
|
{ "order", "publish_date_desc" },
|
|
{ "format", "infinite" },
|
|
{ "skip_users", "all" }
|
|
};
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
|
|
PurchasedDtos.PurchasedDto? purchasedDto =
|
|
DeserializeJson<PurchasedDtos.PurchasedDto>(body, s_mJsonSerializerSettings);
|
|
PurchasedEntities.Purchased purchased = PurchasedMapper.FromDto(purchasedDto);
|
|
if (purchased.HasMore)
|
|
{
|
|
getParams["offset"] = purchased.List.Count.ToString();
|
|
while (true)
|
|
{
|
|
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
|
PurchasedEntities.Purchased newPurchased;
|
|
Dictionary<string, string> loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
|
|
HttpClient loopclient = GetHttpClient();
|
|
|
|
HttpRequestMessage looprequest =
|
|
new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}");
|
|
|
|
foreach (KeyValuePair<string, string> keyValuePair in loopheaders)
|
|
{
|
|
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
|
|
}
|
|
|
|
using (HttpResponseMessage loopresponse = await loopclient.SendAsync(looprequest))
|
|
{
|
|
loopresponse.EnsureSuccessStatusCode();
|
|
string loopbody = await loopresponse.Content.ReadAsStringAsync();
|
|
PurchasedDtos.PurchasedDto? newPurchasedDto =
|
|
DeserializeJson<PurchasedDtos.PurchasedDto>(loopbody, s_mJsonSerializerSettings);
|
|
newPurchased = PurchasedMapper.FromDto(newPurchasedDto);
|
|
}
|
|
|
|
purchased.List.AddRange(newPurchased.List);
|
|
if (!newPurchased.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + postLimit);
|
|
}
|
|
}
|
|
|
|
if (purchased.List.Count > 0)
|
|
{
|
|
foreach (PurchasedEntities.ListItem purchase in
|
|
purchased.List.OrderByDescending(p => p.PostedAt ?? p.CreatedAt))
|
|
{
|
|
if (purchase.FromUser != null)
|
|
{
|
|
if (!userPurchases.ContainsKey(purchase.FromUser.Id))
|
|
{
|
|
userPurchases.Add(purchase.FromUser.Id, new List<PurchasedEntities.ListItem>());
|
|
}
|
|
|
|
userPurchases[purchase.FromUser.Id].Add(purchase);
|
|
}
|
|
else if (purchase.Author != null)
|
|
{
|
|
if (!userPurchases.ContainsKey(purchase.Author.Id))
|
|
{
|
|
userPurchases.Add(purchase.Author.Id, new List<PurchasedEntities.ListItem>());
|
|
}
|
|
|
|
userPurchases[purchase.Author.Id].Add(purchase);
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach (KeyValuePair<long, List<PurchasedEntities.ListItem>> user in userPurchases)
|
|
{
|
|
PurchasedEntities.PurchasedTabCollection purchasedTabCollection = new();
|
|
JObject? userObject = await GetUserInfoById($"/users/list?x[]={user.Key}");
|
|
purchasedTabCollection.UserId = user.Key;
|
|
string? fetchedUsername = userObject?[user.Key.ToString()]?["username"]?.ToString();
|
|
purchasedTabCollection.Username = !string.IsNullOrEmpty(fetchedUsername)
|
|
? fetchedUsername
|
|
: $"Deleted User - {user.Key}";
|
|
string path = Path.Combine(folder, purchasedTabCollection.Username);
|
|
if (Path.Exists(path))
|
|
{
|
|
foreach (PurchasedEntities.ListItem purchase in user.Value)
|
|
{
|
|
if (purchase.Media == null)
|
|
{
|
|
Log.Warning(
|
|
"PurchasedTab purchase media null, setting empty list | userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt}",
|
|
user.Key, purchasedTabCollection.Username, purchase.Id, purchase.ResponseType,
|
|
purchase.CreatedAt, purchase.PostedAt);
|
|
purchase.Media = new List<MessageEntities.Medium>();
|
|
}
|
|
|
|
switch (purchase.ResponseType)
|
|
{
|
|
case "post":
|
|
List<long> previewids = new();
|
|
if (purchase.Previews != null)
|
|
{
|
|
for (int i = 0; i < purchase.Previews.Count; i++)
|
|
{
|
|
if (purchase.Previews[i] is long previewId)
|
|
{
|
|
if (!previewids.Contains(previewId))
|
|
{
|
|
previewids.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (purchase.Preview != null)
|
|
{
|
|
for (int i = 0; i < purchase.Preview.Count; i++)
|
|
{
|
|
if (purchase.Preview[i] is long previewId)
|
|
{
|
|
if (!previewids.Contains(previewId))
|
|
{
|
|
previewids.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
DateTime createdAt = purchase.CreatedAt ?? purchase.PostedAt ?? DateTime.Now;
|
|
bool isArchived = purchase.IsArchived ?? false;
|
|
await dbService.AddPost(path, purchase.Id,
|
|
purchase.Text ?? "",
|
|
purchase.Price ?? "0",
|
|
purchase.Price != null && purchase.IsOpened,
|
|
isArchived,
|
|
createdAt);
|
|
purchasedTabCollection.PaidPosts.PaidPostObjects.Add(purchase);
|
|
foreach (MessageEntities.Medium medium in purchase.Media)
|
|
{
|
|
if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = previewids.Contains(medium.Id);
|
|
|
|
if (previewids.Count > 0)
|
|
{
|
|
bool has = previewids.Any(cus => cus.Equals(medium.Id));
|
|
if (!has && medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(path, medium.Id, purchase.Id, fullUrl, null,
|
|
null, null, "Posts", mediaType, isPreview, false, null);
|
|
purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id, fullUrl);
|
|
purchasedTabCollection.PaidPosts.PaidPostMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (!has && medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(path, medium.Id, purchase.Id, manifestDash,
|
|
null, null, null, "Posts", mediaType, isPreview, false, null);
|
|
purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
|
|
purchasedTabCollection.PaidPosts.PaidPostMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(path, medium.Id, purchase.Id, fullUrl, null,
|
|
null, null, "Posts", mediaType, isPreview, false, null);
|
|
purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id, fullUrl);
|
|
purchasedTabCollection.PaidPosts.PaidPostMedia.Add(medium);
|
|
}
|
|
}
|
|
else if (medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id))
|
|
{
|
|
await dbService.AddMedia(path, medium.Id, purchase.Id, manifestDash,
|
|
null, null, null, "Posts", mediaType, isPreview, false, null);
|
|
purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
|
|
purchasedTabCollection.PaidPosts.PaidPostMedia.Add(medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
break;
|
|
case "message":
|
|
DateTime messageCreatedAt = purchase.PostedAt ?? purchase.CreatedAt ?? DateTime.Now;
|
|
long fromUserId = purchase.FromUser?.Id ?? long.MinValue;
|
|
await dbService.AddMessage(path, purchase.Id,
|
|
purchase.Text ?? "",
|
|
purchase.Price ?? "0", true, false,
|
|
messageCreatedAt, fromUserId);
|
|
|
|
purchasedTabCollection.PaidMessages.PaidMessageObjects.Add(purchase);
|
|
if (purchase.Media != null && purchase.Media.Count > 0)
|
|
{
|
|
List<long> paidMessagePreviewids = new();
|
|
if (purchase.Previews != null)
|
|
{
|
|
for (int i = 0; i < purchase.Previews.Count; i++)
|
|
{
|
|
if (purchase.Previews[i] is long previewId)
|
|
{
|
|
if (!paidMessagePreviewids.Contains(previewId))
|
|
{
|
|
paidMessagePreviewids.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else if (purchase.Preview != null)
|
|
{
|
|
for (int i = 0; i < purchase.Preview.Count; i++)
|
|
{
|
|
if (purchase.Preview[i] is long previewId)
|
|
{
|
|
if (!paidMessagePreviewids.Contains(previewId))
|
|
{
|
|
paidMessagePreviewids.Add(previewId);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach (MessageEntities.Medium medium in purchase.Media)
|
|
{
|
|
if (paidMessagePreviewids.Count > 0)
|
|
{
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = paidMessagePreviewids.Contains(medium.Id);
|
|
bool has = paidMessagePreviewids.Any(cus => cus.Equals(medium.Id));
|
|
if (!has && medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (medium.Type == "photo" &&
|
|
!configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" &&
|
|
!configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" &&
|
|
!configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey(
|
|
medium.Id))
|
|
{
|
|
await dbService.AddMedia(path, medium.Id, purchase.Id,
|
|
fullUrl, null, null, null, "Messages", mediaType,
|
|
isPreview, false, null);
|
|
purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id,
|
|
fullUrl);
|
|
purchasedTabCollection.PaidMessages.PaidMessageMedia.Add(
|
|
medium);
|
|
}
|
|
}
|
|
else if (!has && medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (medium.Type == "photo" &&
|
|
!configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" &&
|
|
!configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" &&
|
|
!configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey(
|
|
medium.Id))
|
|
{
|
|
await dbService.AddMedia(path, medium.Id, purchase.Id,
|
|
manifestDash, null, null, null, "Messages", mediaType,
|
|
isPreview, false, null);
|
|
purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
|
|
purchasedTabCollection.PaidMessages.PaidMessageMedia.Add(
|
|
medium);
|
|
}
|
|
}
|
|
}
|
|
else
|
|
{
|
|
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
|
|
string? fullUrl = medium.Files?.Full?.Url;
|
|
bool isPreview = paidMessagePreviewids.Contains(medium.Id);
|
|
|
|
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
|
|
{
|
|
if (medium.Type == "photo" &&
|
|
!configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" &&
|
|
!configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" &&
|
|
!configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey(
|
|
medium.Id))
|
|
{
|
|
await dbService.AddMedia(path, medium.Id, purchase.Id,
|
|
fullUrl, null, null, null, "Messages", mediaType,
|
|
isPreview, false, null);
|
|
purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id,
|
|
fullUrl);
|
|
purchasedTabCollection.PaidMessages.PaidMessageMedia.Add(
|
|
medium);
|
|
}
|
|
}
|
|
else if (medium.CanView &&
|
|
TryGetDrmInfo(medium.Files, out string manifestDash,
|
|
out string cloudFrontPolicy, out string cloudFrontSignature,
|
|
out string cloudFrontKeyPairId))
|
|
{
|
|
if (medium.Type == "photo" &&
|
|
!configService.CurrentConfig.DownloadImages)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "video" &&
|
|
!configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (medium.Type == "audio" &&
|
|
!configService.CurrentConfig.DownloadAudios)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey(
|
|
medium.Id))
|
|
{
|
|
await dbService.AddMedia(path, medium.Id, purchase.Id,
|
|
manifestDash, null, null, null, "Messages", mediaType,
|
|
isPreview, false, null);
|
|
purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id,
|
|
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
|
|
purchasedTabCollection.PaidMessages.PaidMessageMedia.Add(
|
|
medium);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
purchasedTabCollections.Add(purchasedTabCollection);
|
|
}
|
|
}
|
|
|
|
return purchasedTabCollections;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return [];
|
|
}
|
|
|
|
|
|
public async Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp)
|
|
{
|
|
try
|
|
{
|
|
Auth? currentAuth = authService.CurrentAuth;
|
|
if (currentAuth == null || currentAuth.UserAgent == null || currentAuth.Cookie == null)
|
|
{
|
|
throw new Exception("Auth service is missing required fields");
|
|
}
|
|
|
|
string? pssh;
|
|
|
|
HttpClient client = new();
|
|
HttpRequestMessage request = new(HttpMethod.Get, mpdUrl);
|
|
request.Headers.Add("user-agent", currentAuth.UserAgent);
|
|
request.Headers.Add("Accept", "*/*");
|
|
request.Headers.Add("Cookie",
|
|
$"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {currentAuth.Cookie};");
|
|
using (HttpResponseMessage response = await client.SendAsync(request))
|
|
{
|
|
response.EnsureSuccessStatusCode();
|
|
string body = await response.Content.ReadAsStringAsync();
|
|
XNamespace ns = "urn:mpeg:dash:schema:mpd:2011";
|
|
XNamespace cenc = "urn:mpeg:cenc:2013";
|
|
XDocument xmlDoc = XDocument.Parse(body);
|
|
IEnumerable<XElement> psshElements = xmlDoc.Descendants(cenc + "pssh");
|
|
pssh = psshElements.ElementAt(1).Value;
|
|
}
|
|
|
|
return pssh;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return string.Empty;
|
|
}
|
|
|
|
|
|
public async Task<DateTime> GetDRMMPDLastModified(string mpdUrl, string policy, string signature, string kvp)
|
|
{
|
|
Log.Debug("Calling GetDRMMPDLastModified");
|
|
Log.Debug($"mpdUrl: {mpdUrl}");
|
|
Log.Debug($"policy: {policy}");
|
|
Log.Debug($"signature: {signature}");
|
|
Log.Debug($"kvp: {kvp}");
|
|
|
|
try
|
|
{
|
|
DateTime lastmodified;
|
|
|
|
Auth? currentAuth = authService.CurrentAuth;
|
|
if (currentAuth == null || currentAuth.UserAgent == null || currentAuth.Cookie == null)
|
|
{
|
|
throw new Exception("Auth service is missing required fields");
|
|
}
|
|
|
|
HttpClient client = new();
|
|
HttpRequestMessage request = new(HttpMethod.Get, mpdUrl);
|
|
request.Headers.Add("user-agent", currentAuth.UserAgent);
|
|
request.Headers.Add("Accept", "*/*");
|
|
request.Headers.Add("Cookie",
|
|
$"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {currentAuth.Cookie};");
|
|
using (HttpResponseMessage response =
|
|
await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead))
|
|
{
|
|
response.EnsureSuccessStatusCode();
|
|
lastmodified = response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now;
|
|
|
|
Log.Debug($"Last modified: {lastmodified}");
|
|
}
|
|
|
|
return lastmodified;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return DateTime.Now;
|
|
}
|
|
|
|
public async Task<string> GetDecryptionKeyOFDL(Dictionary<string, string> drmHeaders, string licenceUrl,
|
|
string pssh)
|
|
{
|
|
Log.Debug("Calling GetDecryptionOFDL");
|
|
|
|
try
|
|
{
|
|
HttpClient client = new();
|
|
int attempt = 0;
|
|
|
|
OfdlRequest ofdlRequest = new()
|
|
{
|
|
Pssh = pssh, LicenseUrl = licenceUrl, Headers = JsonConvert.SerializeObject(drmHeaders)
|
|
};
|
|
|
|
string json = JsonConvert.SerializeObject(ofdlRequest);
|
|
|
|
Log.Debug("Posting to ofdl.tools: {Json}", json);
|
|
|
|
while (attempt < MaxAttempts)
|
|
{
|
|
attempt++;
|
|
|
|
HttpRequestMessage request = new(HttpMethod.Post, "https://ofdl.tools/WV")
|
|
{
|
|
Content = new StringContent(json, Encoding.UTF8, "application/json")
|
|
};
|
|
|
|
using HttpResponseMessage response = await client.SendAsync(request);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
string body = await response.Content.ReadAsStringAsync();
|
|
|
|
if (!body.TrimStart().StartsWith('{'))
|
|
{
|
|
return body;
|
|
}
|
|
|
|
Log.Debug($"Received JSON object instead of string. Retrying... Attempt {attempt} of {MaxAttempts}");
|
|
await Task.Delay(DelayBetweenAttempts);
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return string.Empty;
|
|
}
|
|
|
|
public async Task<string> GetDecryptionKeyCDM(Dictionary<string, string> drmHeaders, string licenceURL,
|
|
string pssh)
|
|
{
|
|
Log.Debug("Calling GetDecryptionKeyCDM");
|
|
|
|
try
|
|
{
|
|
byte[] resp1 = await PostData(licenceURL, drmHeaders, [0x08, 0x04]);
|
|
string certDataB64 = Convert.ToBase64String(resp1);
|
|
CDMApi cdm = new();
|
|
byte[]? challenge = cdm.GetChallenge(pssh, certDataB64);
|
|
if (challenge == null)
|
|
{
|
|
throw new Exception("Failed to get challenge from CDM");
|
|
}
|
|
|
|
byte[] resp2 = await PostData(licenceURL, drmHeaders, challenge);
|
|
string licenseB64 = Convert.ToBase64String(resp2);
|
|
Log.Debug("resp1: {Resp1}", resp1);
|
|
Log.Debug("certDataB64: {CertDataB64}", certDataB64);
|
|
Log.Debug("challenge: {Challenge}", challenge);
|
|
Log.Debug("resp2: {Resp2}", resp2);
|
|
Log.Debug("licenseB64: {LicenseB64}", licenseB64);
|
|
cdm.ProvideLicense(licenseB64);
|
|
List<ContentKey> keys = cdm.GetKeys();
|
|
if (keys.Count > 0)
|
|
{
|
|
Log.Debug("GetDecryptionKeyCDM Key: {ContentKey}", keys[0]);
|
|
return keys[0].ToString();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return string.Empty;
|
|
}
|
|
|
|
|
|
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint,
|
|
HttpClient client)
|
|
{
|
|
Log.Debug("Calling BuildHeaderAndExecuteRequests");
|
|
|
|
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
|
|
using HttpResponseMessage response = await client.SendAsync(request);
|
|
response.EnsureSuccessStatusCode();
|
|
string body = await response.Content.ReadAsStringAsync();
|
|
|
|
Log.Debug(body);
|
|
|
|
return body;
|
|
}
|
|
|
|
|
|
private Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
|
|
string endpoint)
|
|
{
|
|
Log.Debug("Calling BuildHttpRequestMessage");
|
|
|
|
string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
|
|
|
Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
|
|
|
|
HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{queryParams}");
|
|
|
|
Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}");
|
|
|
|
foreach (KeyValuePair<string, string> keyValuePair in headers)
|
|
{
|
|
request.Headers.Add(keyValuePair.Key, keyValuePair.Value);
|
|
}
|
|
|
|
return Task.FromResult(request);
|
|
}
|
|
|
|
private static double ConvertToUnixTimestampWithMicrosecondPrecision(DateTime date)
|
|
{
|
|
DateTime origin = new(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
|
|
TimeSpan diff = date.ToUniversalTime() - origin;
|
|
|
|
return
|
|
diff.TotalSeconds; // This gives the number of seconds. If you need milliseconds, use diff.TotalMilliseconds
|
|
}
|
|
|
|
public static bool IsStringOnlyDigits(string input) => input.All(char.IsDigit);
|
|
|
|
|
|
private HttpClient GetHttpClient()
|
|
{
|
|
HttpClient client = new();
|
|
if (configService.CurrentConfig.Timeout != null && configService.CurrentConfig.Timeout > 0)
|
|
{
|
|
client.Timeout = TimeSpan.FromSeconds(configService.CurrentConfig.Timeout.Value);
|
|
}
|
|
|
|
return client;
|
|
}
|
|
|
|
private static T? DeserializeJson<T>(string? body, JsonSerializerSettings? settings = null)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(body))
|
|
{
|
|
return default;
|
|
}
|
|
|
|
return settings == null
|
|
? JsonConvert.DeserializeObject<T>(body)
|
|
: JsonConvert.DeserializeObject<T>(body, settings);
|
|
}
|
|
|
|
private static string? ResolveMediaType(string? type) =>
|
|
type switch
|
|
{
|
|
"photo" => "Images",
|
|
"video" => "Videos",
|
|
"gif" => "Videos",
|
|
"audio" => "Audios",
|
|
_ => null
|
|
};
|
|
|
|
private static bool TryGetDrmInfo(Files? files, out string manifestDash, out string cloudFrontPolicy,
|
|
out string cloudFrontSignature, out string cloudFrontKeyPairId)
|
|
{
|
|
manifestDash = string.Empty;
|
|
cloudFrontPolicy = string.Empty;
|
|
cloudFrontSignature = string.Empty;
|
|
cloudFrontKeyPairId = string.Empty;
|
|
|
|
string? dash = files?.Drm?.Manifest?.Dash;
|
|
Dash? signatureDash = files?.Drm?.Signature?.Dash;
|
|
if (string.IsNullOrEmpty(dash) || signatureDash == null)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(signatureDash.CloudFrontPolicy) ||
|
|
string.IsNullOrEmpty(signatureDash.CloudFrontSignature) ||
|
|
string.IsNullOrEmpty(signatureDash.CloudFrontKeyPairId))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
manifestDash = dash;
|
|
cloudFrontPolicy = signatureDash.CloudFrontPolicy;
|
|
cloudFrontSignature = signatureDash.CloudFrontSignature;
|
|
cloudFrontKeyPairId = signatureDash.CloudFrontKeyPairId;
|
|
return true;
|
|
}
|
|
|
|
private int GetCurrentUserIdOrDefault()
|
|
{
|
|
if (authService.CurrentAuth?.UserId == null)
|
|
{
|
|
return int.MinValue;
|
|
}
|
|
|
|
return int.TryParse(authService.CurrentAuth.UserId, out int userId) ? userId : int.MinValue;
|
|
}
|
|
|
|
|
|
/// <summary>
|
|
/// this one is used during initialization only
|
|
/// if the config option is not available, then no modifications will be done on the getParams
|
|
/// </summary>
|
|
/// <param name="downloadDateSelection"></param>
|
|
/// <param name="getParams"></param>
|
|
/// <param name="dt"></param>
|
|
private static void UpdateGetParamsForDateSelection(DownloadDateSelection downloadDateSelection,
|
|
ref Dictionary<string, string> getParams, DateTime? dt)
|
|
{
|
|
//if (config.DownloadOnlySpecificDates && dt.HasValue)
|
|
//{
|
|
if (dt.HasValue)
|
|
{
|
|
UpdateGetParamsForDateSelection(
|
|
downloadDateSelection,
|
|
ref getParams,
|
|
ConvertToUnixTimestampWithMicrosecondPrecision(dt.Value)
|
|
.ToString("0.000000", CultureInfo.InvariantCulture)
|
|
);
|
|
}
|
|
//}
|
|
}
|
|
|
|
private static void UpdateGetParamsForDateSelection(DownloadDateSelection downloadDateSelection,
|
|
ref Dictionary<string, string> getParams, string? unixTimeStampInMicrosec)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(unixTimeStampInMicrosec))
|
|
{
|
|
return;
|
|
}
|
|
|
|
switch (downloadDateSelection)
|
|
{
|
|
case DownloadDateSelection.before:
|
|
getParams["beforePublishTime"] = unixTimeStampInMicrosec;
|
|
break;
|
|
case DownloadDateSelection.after:
|
|
getParams["order"] = "publish_date_asc";
|
|
getParams["afterPublishTime"] = unixTimeStampInMicrosec;
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
public async Task<Dictionary<string, long>?> GetAllSubscriptions(Dictionary<string, string> getParams,
|
|
string endpoint, bool includeRestricted)
|
|
{
|
|
try
|
|
{
|
|
Dictionary<string, long> users = new();
|
|
|
|
Log.Debug("Calling GetAllSubscrptions");
|
|
|
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
|
|
|
SubscriptionsDtos.SubscriptionsDto? subscriptionsDto =
|
|
DeserializeJson<SubscriptionsDtos.SubscriptionsDto>(body, s_mJsonSerializerSettings);
|
|
SubscriptionEntities.Subscriptions subscriptions = SubscriptionsMapper.FromDto(subscriptionsDto);
|
|
if (subscriptions.HasMore)
|
|
{
|
|
getParams["offset"] = subscriptions.List.Count.ToString();
|
|
|
|
while (true)
|
|
{
|
|
SubscriptionEntities.Subscriptions newSubscriptions;
|
|
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
|
|
|
if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
|
|
{
|
|
SubscriptionsDtos.SubscriptionsDto? newSubscriptionsDto =
|
|
DeserializeJson<SubscriptionsDtos.SubscriptionsDto>(loopbody, s_mJsonSerializerSettings);
|
|
newSubscriptions = SubscriptionsMapper.FromDto(newSubscriptionsDto);
|
|
}
|
|
else
|
|
{
|
|
break;
|
|
}
|
|
|
|
subscriptions.List.AddRange(newSubscriptions.List);
|
|
if (!newSubscriptions.HasMore)
|
|
{
|
|
break;
|
|
}
|
|
|
|
getParams["offset"] = subscriptions.List.Count.ToString();
|
|
}
|
|
}
|
|
|
|
foreach (SubscriptionEntities.ListItem subscription in subscriptions.List)
|
|
{
|
|
if ((!(subscription.IsRestricted ?? false) ||
|
|
((subscription.IsRestricted ?? false) && includeRestricted))
|
|
&& !users.ContainsKey(subscription.Username))
|
|
{
|
|
users.Add(subscription.Username, subscription.Id);
|
|
}
|
|
}
|
|
|
|
return users;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public static string? GetDynamicRules()
|
|
{
|
|
Log.Debug("Calling GetDynamicRules");
|
|
try
|
|
{
|
|
HttpClient client = new();
|
|
HttpRequestMessage request = new(HttpMethod.Get,
|
|
"https://git.ofdl.tools/sim0n00ps/dynamic-rules/raw/branch/main/rules.json");
|
|
using HttpResponseMessage response = client.Send(request);
|
|
|
|
if (!response.IsSuccessStatusCode)
|
|
{
|
|
Log.Debug("GetDynamicRules did not return a Success Status Code");
|
|
return null;
|
|
}
|
|
|
|
string body = response.Content.ReadAsStringAsync().Result;
|
|
|
|
Log.Debug("GetDynamicRules Response: ");
|
|
Log.Debug(body);
|
|
|
|
return body;
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
if (ex.InnerException != null)
|
|
{
|
|
Console.WriteLine("\nInner Exception:");
|
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message,
|
|
ex.InnerException.StackTrace);
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
}
|