using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Xml;
using System.Xml.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using OF_DL.Models;
using OF_DL.Models.Entities.Common;
using OF_DL.Enumerations;
using OF_DL.Helpers;
using ArchivedDtos = OF_DL.Models.Dtos.Archived;
using HighlightDtos = OF_DL.Models.Dtos.Highlights;
using ListDtos = OF_DL.Models.Dtos.Lists;
using MessageDtos = OF_DL.Models.Dtos.Messages;
using PostDtos = OF_DL.Models.Dtos.Posts;
using PurchasedDtos = OF_DL.Models.Dtos.Purchased;
using StoriesDtos = OF_DL.Models.Dtos.Stories;
using StreamsDtos = OF_DL.Models.Dtos.Streams;
using UserDtos = OF_DL.Models.Dtos.Users;
using SubscriptionsDtos = OF_DL.Models.Dtos.Subscriptions;
using ArchivedEntities = OF_DL.Models.Entities.Archived;
using HighlightEntities = OF_DL.Models.Entities.Highlights;
using ListEntities = OF_DL.Models.Entities.Lists;
using MessageEntities = OF_DL.Models.Entities.Messages;
using PostEntities = OF_DL.Models.Entities.Posts;
using PurchasedEntities = OF_DL.Models.Entities.Purchased;
using StoryEntities = OF_DL.Models.Entities.Stories;
using StreamEntities = OF_DL.Models.Entities.Streams;
using SubscriptionEntities = OF_DL.Models.Entities.Subscriptions;
using UserEntities = OF_DL.Models.Entities.Users;
using OF_DL.Models.Mappers;
using OF_DL.Models.OfdlApi;
using OF_DL.Widevine;
using Serilog;
using static OF_DL.Utils.HttpUtil;
using Constants = OF_DL.Helpers.Constants;
using SinglePostCollection = OF_DL.Models.Entities.Posts.SinglePostCollection;
namespace OF_DL.Services;
public class ApiService(IAuthService authService, IConfigService configService, IDbService dbService)
: IApiService
{
private const int MaxAttempts = 30;
private const int DelayBetweenAttempts = 3000;
private static readonly JsonSerializerSettings s_mJsonSerializerSettings;
private static DateTime? s_cachedDynamicRulesExpiration;
private static DynamicRules? s_cachedDynamicRules;
static ApiService() =>
s_mJsonSerializerSettings = new JsonSerializerSettings { MissingMemberHandling = MissingMemberHandling.Ignore };
///
/// Builds signed headers for API requests using dynamic rules.
///
/// The API path.
/// The query string.
/// Signed headers required by the API.
public Dictionary GetDynamicHeaders(string path, string queryParams)
{
Log.Debug("Calling GetDynamicHeaders");
Log.Debug("Path: {Path}", path);
Log.Debug("Query Params: {QueryParams}", queryParams);
DynamicRules? root;
//Check if we have a cached version of the dynamic rules
if (s_cachedDynamicRules != null && s_cachedDynamicRulesExpiration.HasValue &&
DateTime.UtcNow < s_cachedDynamicRulesExpiration)
{
Log.Debug("Using cached dynamic rules");
root = s_cachedDynamicRules;
}
else
{
// Get rules from GitHub and fallback to a local file
string? dynamicRulesJson = GetDynamicRules();
if (!string.IsNullOrEmpty(dynamicRulesJson))
{
Log.Debug("Using dynamic rules from GitHub");
root = JsonConvert.DeserializeObject(dynamicRulesJson);
// Cache the GitHub response for 15 minutes
s_cachedDynamicRules = root;
s_cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(15);
}
else
{
Log.Debug("Using dynamic rules from local file");
root = JsonConvert.DeserializeObject(File.ReadAllText("rules.json"));
// Cache the dynamic rules from a local file to prevent unnecessary disk
// operations and frequent call to GitHub. Since the GitHub dynamic rules
// are preferred to the local file, the cache time is shorter than when dynamic rules
// are successfully retrieved from GitHub.
s_cachedDynamicRules = root;
s_cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(5);
}
}
if (root == null)
{
throw new Exception("Unable to parse dynamic rules. Root is null");
}
if (root.ChecksumConstant == null || root.ChecksumIndexes.Count == 0 || root.Prefix == null ||
root.Suffix == null || root.AppToken == null)
{
throw new Exception("Invalid dynamic rules. Missing required fields");
}
if (authService.CurrentAuth == null)
{
throw new Exception("Auth service is null");
}
if (authService.CurrentAuth.UserId == null || authService.CurrentAuth.Cookie == null ||
authService.CurrentAuth.UserAgent == null || authService.CurrentAuth.XBc == null)
{
throw new Exception("Auth service is missing required fields");
}
DateTimeOffset dto = DateTime.UtcNow;
long timestamp = dto.ToUnixTimeMilliseconds();
string input = $"{root.StaticParam}\n{timestamp}\n{path + queryParams}\n{authService.CurrentAuth.UserId}";
byte[] inputBytes = Encoding.UTF8.GetBytes(input);
byte[] hashBytes = SHA1.HashData(inputBytes);
string hashString = BitConverter.ToString(hashBytes).Replace("-", "").ToLower();
int checksum = root.ChecksumIndexes.Aggregate(0, (current, number) => current + hashString[number]) +
root.ChecksumConstant.Value;
string sign = $"{root.Prefix}:{hashString}:{checksum.ToString("X").ToLower()}:{root.Suffix}";
Dictionary headers = new()
{
{ "accept", "application/json, text/plain" },
{ "app-token", root.AppToken },
{ "cookie", authService.CurrentAuth.Cookie },
{ "sign", sign },
{ "time", timestamp.ToString() },
{ "user-id", authService.CurrentAuth.UserId },
{ "user-agent", authService.CurrentAuth.UserAgent },
{ "x-bc", authService.CurrentAuth.XBc }
};
return headers;
}
private bool HasSignedRequestAuth()
{
Auth? currentAuth = authService.CurrentAuth;
return currentAuth is { UserId: not null, Cookie: not null, UserAgent: not null, XBc: not null };
}
///
/// Retrieves user information from the API.
///
/// The user endpoint.
///
/// The user entity when available.
public async Task GetUserInfo(string endpoint, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetUserInfo: {endpoint}");
if (!HasSignedRequestAuth())
{
return null;
}
try
{
cancellationToken.ThrowIfCancellationRequested();
UserEntities.User user = new();
Dictionary getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() }, { "order", "publish_date_asc" }
};
HttpClient client = new();
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
using HttpResponseMessage response = await client.SendAsync(request, cancellationToken);
if (!response.IsSuccessStatusCode)
{
return user;
}
response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync();
UserDtos.UserDto? userDto =
JsonConvert.DeserializeObject(body, s_mJsonSerializerSettings);
user = UserMapper.FromDto(userDto) ?? new UserEntities.User();
return user;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return null;
}
///
/// Retrieves user information by ID.
///
/// The user list endpoint.
/// A JSON object when available.
public async Task GetUserInfoById(string endpoint)
{
if (!HasSignedRequestAuth())
{
return null;
}
try
{
HttpClient client = new();
HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary(), endpoint);
using HttpResponseMessage response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync();
// if the content creator doesn't exist, we get a 200 response, but the content isn't usable
// so let's not throw an exception, since "content creator no longer exists" is handled elsewhere
// which means we won't get loads of exceptions
if (body.Equals("[]"))
{
return null;
}
JObject jObject = JObject.Parse(body);
return jObject;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return null;
}
///
/// Retrieves active subscriptions.
///
/// The subscriptions endpoint.
/// Whether to include restricted subscriptions.
/// A username-to-userId map.
public async Task?> GetActiveSubscriptions(string endpoint, bool includeRestricted)
{
Dictionary getParams = new()
{
{ "offset", "0" }, { "limit", "50" }, { "type", "active" }, { "format", "infinite" }
};
return await GetAllSubscriptions(getParams, endpoint, includeRestricted);
}
///
/// Retrieves expired subscriptions.
///
/// The subscriptions endpoint.
/// Whether to include restricted subscriptions.
/// A username-to-userId map.
public async Task?> GetExpiredSubscriptions(string endpoint, bool includeRestricted)
{
Dictionary getParams = new()
{
{ "offset", "0" }, { "limit", "50" }, { "type", "expired" }, { "format", "infinite" }
};
Log.Debug("Calling GetExpiredSubscriptions");
return await GetAllSubscriptions(getParams, endpoint, includeRestricted);
}
///
/// Retrieves the user's lists.
///
/// The lists endpoint.
/// A list name to list ID map.
public async Task?> GetLists(string endpoint)
{
Log.Debug("Calling GetLists");
if (!HasSignedRequestAuth())
{
return null;
}
try
{
int offset = 0;
Dictionary getParams = new()
{
{ "offset", offset.ToString() },
{ "skip_users", "all" },
{ "limit", "50" },
{ "format", "infinite" }
};
Dictionary lists = new();
while (true)
{
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
if (body == null)
{
break;
}
ListDtos.UserListDto? userListDto = JsonConvert.DeserializeObject(body);
ListEntities.UserList userList = UserListsMapper.FromDto(userListDto);
foreach (ListEntities.UserListItem listItem in userList.List)
{
if (IsStringOnlyDigits(listItem.Id) && !lists.ContainsKey(listItem.Name))
{
lists.Add(listItem.Name, Convert.ToInt32(listItem.Id));
}
}
if (userList.HasMore)
{
offset += 50;
getParams["offset"] = Convert.ToString(offset);
}
else
{
break;
}
}
return lists;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return null;
}
///
/// Retrieves usernames in a specific list.
///
/// The list users endpoint.
/// The usernames in the list.
public async Task?> GetListUsers(string endpoint)
{
Log.Debug($"Calling GetListUsers - {endpoint}");
if (!HasSignedRequestAuth())
{
return null;
}
try
{
int offset = 0;
Dictionary getParams = new() { { "offset", offset.ToString() }, { "limit", "50" } };
List users = [];
while (true)
{
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
if (body == null)
{
break;
}
List? usersListDto =
JsonConvert.DeserializeObject>(body);
List usersList = UserListsMapper.FromDto(usersListDto);
if (usersList.Count <= 0)
{
break;
}
users.AddRange(usersList.Select(ul => ul.Username));
if (users.Count < 50)
{
break;
}
offset += 50;
getParams["offset"] = Convert.ToString(offset);
}
return users;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return null;
}
///
/// Retrieves media URLs for stories or highlights.
///
/// The media type to fetch.
/// The endpoint to query.
/// Optional username context.
/// The creator folder path.
///
/// A mediaId-to-URL map.
public async Task?> GetMedia(MediaType mediatype,
string endpoint,
string? username,
string folder,
CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetMedia - {username}");
if (!HasSignedRequestAuth())
{
return null;
}
try
{
cancellationToken.ThrowIfCancellationRequested();
Dictionary returnUrls = new();
const int limit = 5;
int offset = 0;
Dictionary getParams = new();
switch (mediatype)
{
case MediaType.Stories:
getParams = new Dictionary
{
{ "limit", Constants.ApiPageSize.ToString() },
{ "order", "publish_date_desc" },
{ "skip_users", "all" }
};
break;
case MediaType.Highlights:
getParams = new Dictionary
{
{ "limit", limit.ToString() }, { "offset", offset.ToString() }, { "skip_users", "all" }
};
break;
}
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
if (string.IsNullOrWhiteSpace(body))
{
Log.Warning("GetMedia returned empty response for {Endpoint}", endpoint);
return returnUrls;
}
if (mediatype == MediaType.Stories)
{
Log.Debug("Media Stories - " + endpoint);
List? storiesDto =
DeserializeJson>(body, s_mJsonSerializerSettings);
List stories = StoriesMapper.FromDto(storiesDto);
foreach (StoryEntities.Stories story in stories)
{
DateTime? storyCreatedAt = story.Media.Count > 0 ? story.Media[0].CreatedAt : null;
if (storyCreatedAt.HasValue)
{
await dbService.AddStory(folder, story.Id, "", "0", false, false, storyCreatedAt.Value);
}
else if (story.CreatedAt.HasValue)
{
await dbService.AddStory(folder, story.Id, "", "0", false, false, story.CreatedAt.Value);
}
else
{
await dbService.AddStory(folder, story.Id, "", "0", false, false, DateTime.Now);
}
if (story.Media.Count > 0)
{
foreach (StoryEntities.Medium medium in story.Media)
{
string? mediaUrl = medium.Files.Full?.Url;
if (string.IsNullOrEmpty(mediaUrl))
{
continue;
}
string? mediaType = ResolveMediaType(medium.Type);
if (mediaType == null)
{
continue;
}
await dbService.AddMedia(folder, medium.Id, story.Id, mediaUrl, null, null, null,
"Stories", mediaType, false, false, null);
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (medium.CanView)
{
returnUrls.TryAdd(medium.Id, mediaUrl);
}
}
}
}
}
else if (mediatype == MediaType.Highlights)
{
List highlightIds = [];
HighlightDtos.HighlightsDto? highlightsDto =
DeserializeJson(body, s_mJsonSerializerSettings);
HighlightEntities.Highlights highlights = HighlightsMapper.FromDto(highlightsDto);
if (highlights.HasMore)
{
offset += 5;
getParams["offset"] = offset.ToString();
while (true)
{
Log.Debug("Media Highlights - " + endpoint);
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
if (string.IsNullOrWhiteSpace(loopbody))
{
Log.Warning("Received empty body from API");
break;
}
HighlightDtos.HighlightsDto? newHighlightsDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
HighlightEntities.Highlights newHighlights = HighlightsMapper.FromDto(newHighlightsDto);
highlights.List.AddRange(newHighlights.List);
if (!newHighlights.HasMore)
{
break;
}
offset += 5;
getParams["offset"] = offset.ToString();
}
}
foreach (HighlightEntities.ListItem list in highlights.List)
{
if (!highlightIds.Contains(list.Id.ToString()))
{
highlightIds.Add(list.Id.ToString());
}
}
foreach (string highlightId in highlightIds)
{
Dictionary highlightHeaders =
GetDynamicHeaders("/api2/v2/stories/highlights/" + highlightId, "");
HttpClient highlightClient = GetHttpClient();
HttpRequestMessage highlightRequest = new(HttpMethod.Get,
$"https://onlyfans.com/api2/v2/stories/highlights/{highlightId}");
foreach (KeyValuePair keyValuePair in highlightHeaders)
{
highlightRequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
using HttpResponseMessage highlightResponse = await highlightClient.SendAsync(highlightRequest);
highlightResponse.EnsureSuccessStatusCode();
string highlightBody = await highlightResponse.Content.ReadAsStringAsync();
HighlightDtos.HighlightMediaDto? highlightMediaDto =
DeserializeJson(highlightBody, s_mJsonSerializerSettings);
HighlightEntities.HighlightMedia highlightMedia = HighlightsMapper.FromDto(highlightMediaDto);
foreach (HighlightEntities.Story item in highlightMedia.Stories)
{
DateTime? createdAt = item.Media is { Count: > 0 }
? item.Media[0].CreatedAt
: null;
if (createdAt.HasValue)
{
await dbService.AddStory(folder, item.Id, "", "0", false, false, createdAt.Value);
}
else if (item.CreatedAt.HasValue)
{
await dbService.AddStory(folder, item.Id, "", "0", false, false, item.CreatedAt.Value);
}
else
{
await dbService.AddStory(folder, item.Id, "", "0", false, false, DateTime.Now);
}
if (item.Media is not { Count: > 0 } || !item.Media[0].CanView)
{
continue;
}
string? storyUrl = item.Media[0].Files?.Full?.Url;
string storyUrlValue = storyUrl ?? string.Empty;
foreach (HighlightEntities.Medium medium in item.Media)
{
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
await dbService.AddMedia(folder, medium.Id, item.Id, storyUrlValue, null, null, null,
"Stories", mediaType, false, false, null);
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!returnUrls.ContainsKey(medium.Id) && !string.IsNullOrEmpty(storyUrl))
{
returnUrls.Add(medium.Id, storyUrl);
}
}
}
}
}
return returnUrls;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return null;
}
///
/// Retrieves paid posts and their media.
///
/// The paid posts endpoint.
/// The creator folder path.
/// The creator username.
/// A list to collect paid media IDs.
/// Status reporter.
///
/// A paid post collection.
public async Task GetPaidPosts(string endpoint, string folder,
string username,
List paidPostIds, IStatusReporter statusReporter, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetPaidPosts - {username}");
try
{
PurchasedEntities.PaidPostCollection paidPostCollection = new();
Dictionary getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() },
{ "skip_users", "all" },
{ "order", "publish_date_desc" },
{ "format", "infinite" },
{ "author", username }
};
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
PurchasedDtos.PurchasedDto? paidPostsDto =
DeserializeJson(body, s_mJsonSerializerSettings);
PurchasedEntities.Purchased paidPosts = PurchasedMapper.FromDto(paidPostsDto);
statusReporter.ReportStatus($"Getting Paid Posts - Found {paidPosts.List.Count}");
if (paidPosts.HasMore)
{
getParams["offset"] = paidPosts.List.Count.ToString();
while (true)
{
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
PurchasedDtos.PurchasedDto? newPaidPostsDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
PurchasedEntities.Purchased newPaidPosts = PurchasedMapper.FromDto(newPaidPostsDto);
paidPosts.List.AddRange(newPaidPosts.List);
statusReporter.ReportStatus($"Getting Paid Posts - Found {paidPosts.List.Count}");
if (!newPaidPosts.HasMore)
{
break;
}
getParams["offset"] =
Convert.ToString(Convert.ToInt32(getParams["offset"]) + Constants.ApiPageSize);
}
}
List paidPostList = paidPosts.List;
foreach (PurchasedEntities.ListItem purchase in paidPostList)
{
if (purchase.ResponseType != "post" || purchase.Media is not { Count: > 0 })
{
continue;
}
List previewIds = [];
if (purchase.Previews != null)
{
for (int i = 0; i < purchase.Previews.Count; i++)
{
if (purchase.Previews[i] is long previewId)
{
if (!previewIds.Contains(previewId))
{
previewIds.Add(previewId);
}
}
}
}
else if (purchase.Preview != null)
{
for (int i = 0; i < purchase.Preview.Count; i++)
{
if (purchase.Preview[i] is long previewId)
{
if (!previewIds.Contains(previewId))
{
previewIds.Add(previewId);
}
}
}
}
DateTime createdAt = purchase.CreatedAt ?? purchase.PostedAt ?? DateTime.Now;
bool isArchived = purchase.IsArchived ?? false;
await dbService.AddPost(folder, purchase.Id, purchase.Text ?? "",
purchase.Price ?? "0",
purchase is { Price: not null, IsOpened: true }, isArchived, createdAt);
paidPostCollection.PaidPostObjects.Add(purchase);
foreach (MessageEntities.Medium medium in purchase.Media)
{
if (!previewIds.Contains(medium.Id))
{
paidPostIds.Add(medium.Id);
}
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
string mediaType = ResolveMediaType(medium.Type) ?? "";
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = previewIds.Contains(medium.Id);
if (previewIds.Count > 0)
{
bool has = previewIds.Any(cus => cus.Equals(medium.Id));
if (!has && medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, purchase.Id, fullUrl,
null, null, null, "Posts",
mediaType, isPreview, false, null);
paidPostCollection.PaidPosts.Add(medium.Id, fullUrl);
paidPostCollection.PaidPostMedia.Add(medium);
}
}
else if (!has && medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, purchase.Id,
manifestDash, null, null, null, "Posts", mediaType, isPreview, false, null);
paidPostCollection.PaidPosts.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
paidPostCollection.PaidPostMedia.Add(medium);
}
}
}
else
{
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, purchase.Id, fullUrl,
null, null, null, "Posts",
mediaType, isPreview, false, null);
paidPostCollection.PaidPosts.Add(medium.Id, fullUrl);
paidPostCollection.PaidPostMedia.Add(medium);
}
}
else if (medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, purchase.Id,
manifestDash, null, null, null, "Posts", mediaType, isPreview, false, null);
paidPostCollection.PaidPosts.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
paidPostCollection.PaidPostMedia.Add(medium);
}
}
}
}
}
return paidPostCollection;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return new PurchasedEntities.PaidPostCollection();
}
///
/// Retrieves posts and their media.
///
/// The posts endpoint.
/// The creator folder path.
/// Paid post media IDs to skip.
/// Status reporter.
///
/// A post collection.
public async Task GetPosts(string endpoint, string folder, List paidPostIds,
IStatusReporter statusReporter, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetPosts - {endpoint}");
try
{
PostEntities.PostCollection postCollection = new();
Dictionary getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() },
{ "order", "publish_date_desc" },
{ "format", "infinite" },
{ "skip_users", "all" }
};
DownloadDateSelection downloadDateSelection = DownloadDateSelection.before;
DateTime? downloadAsOf = null;
if (configService.CurrentConfig is { DownloadOnlySpecificDates: true, CustomDate: not null })
{
downloadDateSelection = configService.CurrentConfig.DownloadDateSelection;
downloadAsOf = configService.CurrentConfig.CustomDate;
}
else if (configService.CurrentConfig.DownloadPostsIncrementally)
{
DateTime? mostRecentPostDate = await dbService.GetMostRecentPostDate(folder);
if (mostRecentPostDate.HasValue)
{
downloadDateSelection = DownloadDateSelection.after;
downloadAsOf = mostRecentPostDate.Value.AddMinutes(-5); // Back track a little for a margin of error
}
}
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
downloadAsOf);
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
PostDtos.PostDto? postsDto =
DeserializeJson(body, s_mJsonSerializerSettings);
PostEntities.Post posts = PostMapper.FromDto(postsDto);
statusReporter.ReportStatus($"Getting Posts - Found {posts.List.Count}");
if (posts.HasMore)
{
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
posts.TailMarker);
while (true)
{
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
PostDtos.PostDto? newPostsDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
PostEntities.Post newposts = PostMapper.FromDto(newPostsDto);
posts.List.AddRange(newposts.List);
statusReporter.ReportStatus($"Getting Posts - Found {posts.List.Count}");
if (!newposts.HasMore)
{
break;
}
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
newposts.TailMarker);
}
}
List postList = posts.List;
foreach (PostEntities.ListItem post in postList)
{
if (configService.CurrentConfig.SkipAds)
{
if (!string.IsNullOrEmpty(post.RawText) &&
(post.RawText.Contains("#ad") || post.RawText.Contains("/trial/") ||
post.RawText.Contains("#announcement")))
{
continue;
}
if (!string.IsNullOrEmpty(post.Text) &&
(post.Text.Contains("#ad") || post.Text.Contains("/trial/") ||
post.Text.Contains("#announcement")))
{
continue;
}
}
List postPreviewIds = [];
if (post.Preview is { Count: > 0 })
{
for (int i = 0; i < post.Preview.Count; i++)
{
if (post.Preview[i] is not long previewId)
{
continue;
}
if (!postPreviewIds.Contains(previewId))
{
postPreviewIds.Add(previewId);
}
}
}
await dbService.AddPost(folder, post.Id, !string.IsNullOrEmpty(post.RawText) ? post.RawText : "",
post.Price ?? "0", post is { Price: not null, IsOpened: true },
post.IsArchived, post.PostedAt);
postCollection.PostObjects.Add(post);
if (post.Media is not { Count: > 0 })
{
continue;
}
foreach (PostEntities.Medium medium in post.Media)
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
string? previewUrl = medium.Files?.Preview?.Url;
bool isPreview = postPreviewIds.Contains(medium.Id);
if (medium.CanView && medium.Files?.Drm == null)
{
bool has = paidPostIds.Any(cus => cus.Equals(medium.Id));
if (!has && !string.IsNullOrEmpty(fullUrl))
{
if (!postCollection.Posts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, post.Id, fullUrl, null, null, null,
"Posts", mediaType, isPreview, false, null);
postCollection.Posts.Add(medium.Id, fullUrl);
postCollection.PostMedia.Add(medium);
}
}
else if (!has && string.IsNullOrEmpty(fullUrl) && !string.IsNullOrEmpty(previewUrl))
{
if (!postCollection.Posts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, post.Id, previewUrl, null, null, null,
"Posts", mediaType, isPreview, false, null);
postCollection.Posts.Add(medium.Id, previewUrl);
postCollection.PostMedia.Add(medium);
}
}
}
else if (medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
out string cloudFrontSignature, out string cloudFrontKeyPairId))
{
bool has = paidPostIds.Any(cus => cus.Equals(medium.Id));
if (!has && !postCollection.Posts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, post.Id, manifestDash, null, null, null,
"Posts", mediaType, isPreview, false, null);
postCollection.Posts.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{post.Id}");
postCollection.PostMedia.Add(medium);
}
}
}
}
return postCollection;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return new PostEntities.PostCollection();
}
///
/// Retrieves a single post and its media.
///
/// The post endpoint.
/// The creator folder path.
///
/// A single post collection.
public async Task GetPost(string endpoint, string folder, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetPost - {endpoint}");
try
{
SinglePostCollection singlePostCollection = new();
Dictionary getParams = new() { { "skip_users", "all" } };
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
PostDtos.SinglePostDto? singlePostDto =
DeserializeJson(body, s_mJsonSerializerSettings);
PostEntities.SinglePost singlePost = PostMapper.FromDto(singlePostDto);
if (singlePostDto != null)
{
List postPreviewIds = [];
if (singlePost.Preview is { Count: > 0 })
{
for (int i = 0; i < singlePost.Preview.Count; i++)
{
if (singlePost.Preview[i] is not long previewId)
{
continue;
}
if (!postPreviewIds.Contains(previewId))
{
postPreviewIds.Add(previewId);
}
}
}
await dbService.AddPost(folder, singlePost.Id,
!string.IsNullOrEmpty(singlePost.Text) ? singlePost.Text : "",
singlePost.Price ?? "0",
singlePost is { Price: not null, IsOpened: true }, singlePost.IsArchived,
singlePost.PostedAt);
singlePostCollection.SinglePostObjects.Add(singlePost);
if (singlePost.Media == null || singlePost.Media.Count <= 0)
{
return singlePostCollection;
}
foreach (PostEntities.Medium medium in singlePost.Media)
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
bool isPreview = postPreviewIds.Contains(medium.Id);
string? fullUrl = medium.Files?.Full?.Url;
string? previewUrl = medium.Files?.Preview?.Url;
if (medium.CanView && medium.Files?.Drm == null)
{
switch (configService.CurrentConfig.DownloadVideoResolution)
{
case VideoResolution.source:
if (!string.IsNullOrEmpty(fullUrl))
{
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, singlePost.Id,
fullUrl, null, null, null, "Posts", mediaType, isPreview, false, null);
singlePostCollection.SinglePosts.Add(medium.Id, fullUrl);
singlePostCollection.SinglePostMedia.Add(medium);
}
}
break;
case VideoResolution._240:
string? video240 = medium.VideoSources?._240;
if (!string.IsNullOrEmpty(video240))
{
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, singlePost.Id, video240, null,
null, null, "Posts", mediaType, isPreview, false, null);
singlePostCollection.SinglePosts.Add(medium.Id, video240);
singlePostCollection.SinglePostMedia.Add(medium);
}
}
break;
case VideoResolution._720:
string? video720 = medium.VideoSources?._720;
if (!string.IsNullOrEmpty(video720))
{
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, singlePost.Id, video720, null,
null, null, "Posts", mediaType, isPreview, false, null);
singlePostCollection.SinglePosts.Add(medium.Id, video720);
singlePostCollection.SinglePostMedia.Add(medium);
}
}
break;
}
}
else if (medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
out string cloudFrontSignature, out string cloudFrontKeyPairId))
{
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, singlePost.Id, manifestDash, null, null,
null, "Posts", mediaType, isPreview, false, null);
singlePostCollection.SinglePosts.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{singlePost.Id}");
singlePostCollection.SinglePostMedia.Add(medium);
}
}
else if (!string.IsNullOrEmpty(previewUrl) && medium.Files?.Full == null)
{
if (!singlePostCollection.SinglePosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, singlePost.Id, previewUrl, null, null, null,
"Posts", mediaType, isPreview, false, null);
singlePostCollection.SinglePosts.Add(medium.Id, previewUrl);
singlePostCollection.SinglePostMedia.Add(medium);
}
}
}
}
return singlePostCollection;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return new SinglePostCollection();
}
///
/// Retrieves streams and their media.
///
/// The streams endpoint.
/// The creator folder path.
/// Paid post media IDs to skip.
/// Status reporter.
///
/// A streams collection.
public async Task GetStreams(string endpoint, string folder,
List paidPostIds,
IStatusReporter statusReporter, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetStreams - {endpoint}");
try
{
StreamEntities.StreamsCollection streamsCollection = new();
Dictionary getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() },
{ "order", "publish_date_desc" },
{ "format", "infinite" },
{ "skip_users", "all" }
};
DownloadDateSelection downloadDateSelection = DownloadDateSelection.before;
if (configService.CurrentConfig is { DownloadOnlySpecificDates: true, CustomDate: not null })
{
downloadDateSelection = configService.CurrentConfig.DownloadDateSelection;
}
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
configService.CurrentConfig.CustomDate);
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
StreamsDtos.StreamsDto? streamsDto =
DeserializeJson(body, s_mJsonSerializerSettings);
StreamEntities.Streams streams = StreamsMapper.FromDto(streamsDto);
statusReporter.ReportStatus($"Getting Streams - Found {streams.List.Count}");
if (streams.HasMore)
{
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
streams.TailMarker);
while (true)
{
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
StreamsDtos.StreamsDto? newStreamsDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
StreamEntities.Streams newstreams = StreamsMapper.FromDto(newStreamsDto);
streams.List.AddRange(newstreams.List);
statusReporter.ReportStatus($"Getting Streams - Found {streams.List.Count}");
if (!newstreams.HasMore)
{
break;
}
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
newstreams.TailMarker);
}
}
List streamList = streams.List;
foreach (StreamEntities.ListItem stream in streamList)
{
List streamPreviewIds = [];
if (stream.Preview is { Count: > 0 })
{
for (int i = 0; i < stream.Preview.Count; i++)
{
if (stream.Preview[i] is not long previewId)
{
continue;
}
if (!streamPreviewIds.Contains(previewId))
{
streamPreviewIds.Add(previewId);
}
}
}
await dbService.AddPost(folder, stream.Id, !string.IsNullOrEmpty(stream.Text) ? stream.Text : "",
stream.Price ?? "0", stream is { Price: not null, IsOpened: true },
stream.IsArchived, stream.PostedAt);
streamsCollection.StreamObjects.Add(stream);
if (stream.Media is not { Count: > 0 })
{
continue;
}
foreach (StreamEntities.Medium medium in stream.Media)
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = streamPreviewIds.Contains(medium.Id);
if (medium.CanView && medium.Files?.Drm == null)
{
bool has = paidPostIds.Any(cus => cus.Equals(medium.Id));
if (!has && !string.IsNullOrEmpty(fullUrl))
{
if (!streamsCollection.Streams.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, stream.Id, fullUrl, null, null, null,
"Posts", mediaType, isPreview, false, null);
streamsCollection.Streams.Add(medium.Id, fullUrl);
streamsCollection.StreamMedia.Add(medium);
}
}
}
else if (medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
out string cloudFrontSignature, out string cloudFrontKeyPairId))
{
bool has = paidPostIds.Any(cus => cus.Equals(medium.Id));
if (!has && !streamsCollection.Streams.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, stream.Id, manifestDash, null, null, null,
"Posts", mediaType, isPreview, false, null);
streamsCollection.Streams.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{stream.Id}");
streamsCollection.StreamMedia.Add(medium);
}
}
}
}
return streamsCollection;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return new StreamEntities.StreamsCollection();
}
///
/// Retrieves archived posts and their media.
///
/// The archived posts endpoint.
/// The creator folder path.
/// Status reporter.
///
/// An archived collection.
public async Task GetArchived(string endpoint, string folder,
IStatusReporter statusReporter, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetArchived - {endpoint}");
try
{
ArchivedEntities.ArchivedCollection archivedCollection = new();
Dictionary getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() },
{ "order", "publish_date_desc" },
{ "skip_users", "all" },
{ "format", "infinite" },
{ "label", "archived" },
{ "counters", "1" }
};
DownloadDateSelection downloadDateSelection = DownloadDateSelection.before;
if (configService.CurrentConfig is { DownloadOnlySpecificDates: true, CustomDate: not null })
{
downloadDateSelection = configService.CurrentConfig.DownloadDateSelection;
}
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
configService.CurrentConfig.CustomDate);
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
if (body == null)
{
throw new Exception("Failed to retrieve archived posts. Received null response.");
}
ArchivedDtos.ArchivedDto? archivedDto =
DeserializeJson(body, s_mJsonSerializerSettings);
ArchivedEntities.Archived archived = ArchivedMapper.FromDto(archivedDto);
statusReporter.ReportStatus($"Getting Archived Posts - Found {archived.List.Count}");
if (archived.HasMore)
{
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
archived.TailMarker);
while (true)
{
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
if (loopbody == null)
{
throw new Exception("Failed to retrieve archived posts. Received null response.");
}
ArchivedDtos.ArchivedDto? newarchivedDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
ArchivedEntities.Archived newarchived = ArchivedMapper.FromDto(newarchivedDto);
archived.List.AddRange(newarchived.List);
statusReporter.ReportStatus($"Getting Archived Posts - Found {archived.List.Count}");
if (!newarchived.HasMore)
{
break;
}
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
newarchived.TailMarker);
}
}
foreach (ArchivedEntities.ListItem archive in archived.List)
{
List previewids = new();
if (archive.Preview != null)
{
for (int i = 0; i < archive.Preview.Count; i++)
{
if (archive.Preview[i] is long previewId)
{
if (!previewids.Contains(previewId))
{
previewids.Add(previewId);
}
}
}
}
await dbService.AddPost(folder, archive.Id, archive.Text ?? "",
archive.Price ?? "0",
archive is { Price: not null, IsOpened: true }, archive.IsArchived, archive.PostedAt);
archivedCollection.ArchivedPostObjects.Add(archive);
if (archive.Media is not { Count: > 0 })
{
continue;
}
foreach (ArchivedEntities.Medium medium in archive.Media)
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = previewids.Contains(medium.Id);
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!archivedCollection.ArchivedPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, archive.Id, fullUrl, null, null, null,
"Posts", mediaType, isPreview, false, null);
archivedCollection.ArchivedPosts.Add(medium.Id, fullUrl);
archivedCollection.ArchivedPostMedia.Add(medium);
}
}
else if (medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
out string cloudFrontSignature, out string cloudFrontKeyPairId))
{
if (!archivedCollection.ArchivedPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, archive.Id, manifestDash, null, null, null,
"Posts", mediaType, isPreview, false, null);
archivedCollection.ArchivedPosts.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{archive.Id}");
archivedCollection.ArchivedPostMedia.Add(medium);
}
}
}
}
return archivedCollection;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return new ArchivedEntities.ArchivedCollection();
}
///
/// Retrieves messages and their media.
///
/// The messages endpoint.
/// The creator folder path.
/// Status reporter.
///
/// A message collection.
public async Task GetMessages(string endpoint, string folder,
IStatusReporter statusReporter, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetMessages - {endpoint}");
try
{
MessageEntities.MessageCollection messageCollection = new();
Dictionary getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() }, { "order", "desc" }, { "skip_users", "all" }
};
int currentUserId = GetCurrentUserIdOrDefault();
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
MessageDtos.MessagesDto? messagesDto =
DeserializeJson(body, s_mJsonSerializerSettings);
MessageEntities.Messages messages = MessagesMapper.FromDto(messagesDto);
statusReporter.ReportStatus($"Getting Messages - Found {messages.List.Count}");
if (messages.HasMore)
{
getParams["id"] = messages.List[^1].Id.ToString();
while (true)
{
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
MessageDtos.MessagesDto? newMessagesDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
MessageEntities.Messages newMessages = MessagesMapper.FromDto(newMessagesDto);
messages.List.AddRange(newMessages.List);
statusReporter.ReportStatus($"Getting Messages - Found {messages.List.Count}");
if (!newMessages.HasMore)
{
break;
}
getParams["id"] = newMessages.List[^1].Id.ToString();
}
}
foreach (MessageEntities.ListItem list in messages.List)
{
if (configService.CurrentConfig.SkipAds)
{
if (!string.IsNullOrEmpty(list.Text) &&
(list.Text.Contains("#ad") || list.Text.Contains("/trial/")))
{
continue;
}
}
List messagePreviewIds = [];
if (list.Previews is { Count: > 0 })
{
for (int i = 0; i < list.Previews.Count; i++)
{
if (list.Previews[i] is not long previewId)
{
continue;
}
if (!messagePreviewIds.Contains(previewId))
{
messagePreviewIds.Add(previewId);
}
}
}
if (configService.CurrentConfig.IgnoreOwnMessages && list.FromUser?.Id == currentUserId)
{
continue;
}
DateTime createdAt = list.CreatedAt ?? DateTime.Now;
await dbService.AddMessage(folder, list.Id, list.Text ?? "", list.Price ?? "0",
list.CanPurchaseReason == "opened" ||
(list.CanPurchaseReason == "opened" && ((bool?)null ?? false)), false,
createdAt,
list.FromUser?.Id ?? int.MinValue);
messageCollection.MessageObjects.Add(list);
if (list.CanPurchaseReason != "opened" && list.Media is { Count: > 0 })
{
foreach (MessageEntities.Medium medium in list.Media ?? [])
{
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = messagePreviewIds.Contains(medium.Id);
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!messageCollection.Messages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, list.Id, fullUrl, null, null, null,
"Messages", mediaType, isPreview, false, null);
messageCollection.Messages.Add(medium.Id, fullUrl);
messageCollection.MessageMedia.Add(medium);
}
}
else if (medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!messageCollection.Messages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, list.Id, manifestDash, null, null, null,
"Messages", mediaType, isPreview, false, null);
messageCollection.Messages.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{list.Id}");
messageCollection.MessageMedia.Add(medium);
}
}
}
}
else if (messagePreviewIds.Count > 0)
{
foreach (MessageEntities.Medium medium in list.Media ?? new List())
{
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = messagePreviewIds.Contains(medium.Id);
if (medium.CanView && !string.IsNullOrEmpty(fullUrl) && isPreview)
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!messageCollection.Messages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, list.Id, fullUrl, null, null, null,
"Messages", mediaType, isPreview, false, null);
messageCollection.Messages.Add(medium.Id, fullUrl);
messageCollection.MessageMedia.Add(medium);
}
}
else if (medium.CanView && isPreview &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!messageCollection.Messages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, list.Id, manifestDash, null, null, null,
"Messages", mediaType, isPreview, false, null);
messageCollection.Messages.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{list.Id}");
messageCollection.MessageMedia.Add(medium);
}
}
}
}
}
return messageCollection;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return new MessageEntities.MessageCollection();
}
///
/// Retrieves a single paid message and its media.
///
/// The paid message endpoint.
/// The creator folder path.
///
/// A single paid message collection.
public async Task GetPaidMessage(string endpoint, string folder, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetPaidMessage - {endpoint}");
try
{
PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection = new();
Dictionary getParams =
new() { { "limit", Constants.ApiPageSize.ToString() }, { "order", "desc" } };
int currentUserId = GetCurrentUserIdOrDefault();
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
MessageDtos.SingleMessageDto? messageDto =
DeserializeJson(body, s_mJsonSerializerSettings);
MessageEntities.SingleMessage message = MessagesMapper.FromDto(messageDto);
if (configService.CurrentConfig.IgnoreOwnMessages && message.FromUser?.Id == currentUserId)
{
return singlePaidMessageCollection;
}
DateTime createdAt = message.CreatedAt ?? DateTime.Now;
await dbService.AddMessage(folder, message.Id, message.Text ?? "",
message.Price?.ToString() ?? "0", true, false,
createdAt,
message.FromUser?.Id ?? int.MinValue);
singlePaidMessageCollection.SingleMessageObjects.Add(message);
List messagePreviewIds = [];
if (message.Previews is { Count: > 0 })
{
for (int i = 0; i < message.Previews.Count; i++)
{
if (message.Previews[i] is not long previewId)
{
continue;
}
if (!messagePreviewIds.Contains(previewId))
{
messagePreviewIds.Add(previewId);
}
}
}
if (message.Media is not { Count: > 0 })
{
return singlePaidMessageCollection;
}
foreach (MessageEntities.Medium medium in message.Media)
{
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = messagePreviewIds.Contains(medium.Id);
if (!isPreview && medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!singlePaidMessageCollection.SingleMessages.TryAdd(medium.Id, fullUrl))
{
continue;
}
await dbService.AddMedia(folder, medium.Id, message.Id, fullUrl, null, null, null,
"Messages", mediaType, isPreview, false, null);
singlePaidMessageCollection.SingleMessageMedia.Add(medium);
}
else if (isPreview && medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!singlePaidMessageCollection.PreviewSingleMessages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, message.Id, fullUrl, null, null, null,
"Messages", mediaType, isPreview, false, null);
singlePaidMessageCollection.PreviewSingleMessages.Add(medium.Id, fullUrl);
singlePaidMessageCollection.PreviewSingleMessageMedia.Add(medium);
}
}
else if (!isPreview && medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash, out string cloudFrontPolicy,
out string cloudFrontSignature, out string cloudFrontKeyPairId))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!singlePaidMessageCollection.SingleMessages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, message.Id, manifestDash, null, null, null,
"Messages", mediaType, isPreview, false, null);
singlePaidMessageCollection.SingleMessages.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{message.Id}");
singlePaidMessageCollection.SingleMessageMedia.Add(medium);
}
}
else if (isPreview && medium.CanView &&
TryGetDrmInfo(medium.Files, out string previewManifestDash,
out string previewCloudFrontPolicy, out string previewCloudFrontSignature,
out string previewCloudFrontKeyPairId))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!singlePaidMessageCollection.PreviewSingleMessages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, message.Id, previewManifestDash, null, null,
null, "Messages", mediaType, isPreview, false, null);
singlePaidMessageCollection.PreviewSingleMessages.Add(medium.Id,
$"{previewManifestDash},{previewCloudFrontPolicy},{previewCloudFrontSignature},{previewCloudFrontKeyPairId},{medium.Id},{message.Id}");
singlePaidMessageCollection.PreviewSingleMessageMedia.Add(medium);
}
}
}
return singlePaidMessageCollection;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return new PurchasedEntities.SinglePaidMessageCollection();
}
///
/// Retrieves paid messages and their media.
///
/// The paid messages endpoint.
/// The creator folder path.
/// The creator username.
/// Status reporter.
///
/// A paid message collection.
public async Task GetPaidMessages(string endpoint, string folder,
string username,
IStatusReporter statusReporter, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetPaidMessages - {username}");
try
{
PurchasedEntities.PaidMessageCollection paidMessageCollection = new();
Dictionary getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() },
{ "order", "publish_date_desc" },
{ "format", "infinite" },
{ "author", username },
{ "skip_users", "all" }
};
int currentUserId = GetCurrentUserIdOrDefault();
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
PurchasedDtos.PurchasedDto? paidMessagesDto =
DeserializeJson(body, s_mJsonSerializerSettings);
PurchasedEntities.Purchased paidMessages = PurchasedMapper.FromDto(paidMessagesDto);
statusReporter.ReportStatus($"Getting Paid Messages - Found {paidMessages.List.Count}");
if (paidMessages.HasMore)
{
getParams["offset"] = paidMessages.List.Count.ToString();
while (true)
{
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
PurchasedEntities.Purchased newpaidMessages;
Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
HttpClient loopclient = GetHttpClient();
HttpRequestMessage looprequest =
new(HttpMethod.Get, $"{Constants.ApiUrl}{endpoint}{loopqueryParams}");
foreach (KeyValuePair keyValuePair in loopheaders)
{
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
using (HttpResponseMessage loopresponse = await loopclient.SendAsync(looprequest))
{
loopresponse.EnsureSuccessStatusCode();
string loopbody = await loopresponse.Content.ReadAsStringAsync();
PurchasedDtos.PurchasedDto? newPaidMessagesDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
newpaidMessages = PurchasedMapper.FromDto(newPaidMessagesDto);
}
paidMessages.List.AddRange(newpaidMessages.List);
statusReporter.ReportStatus($"Getting Paid Messages - Found {paidMessages.List.Count}");
if (!newpaidMessages.HasMore)
{
break;
}
getParams["offset"] =
Convert.ToString(Convert.ToInt32(getParams["offset"]) + Constants.ApiPageSize);
}
}
List paidMessageList =
paidMessages.List;
if (paidMessageList.Count > 0)
{
foreach (PurchasedEntities.ListItem purchase in paidMessageList
.Where(p => p.ResponseType == "message")
.OrderByDescending(p => p.PostedAt ?? p.CreatedAt))
{
long fromUserId = purchase.FromUser?.Id ?? long.MinValue;
if (configService.CurrentConfig.IgnoreOwnMessages && fromUserId == currentUserId)
{
continue;
}
DateTime createdAt = purchase.PostedAt ?? purchase.CreatedAt ?? DateTime.Now;
await dbService.AddMessage(folder, purchase.Id,
purchase.Text ?? "",
purchase.Price ?? "0", true, false, createdAt,
fromUserId);
paidMessageCollection.PaidMessageObjects.Add(purchase);
if (purchase.Media is not { Count: > 0 })
{
continue;
}
List previewIds = [];
if (purchase.Previews != null)
{
for (int i = 0; i < purchase.Previews.Count; i++)
{
if (purchase.Previews[i] is not long previewId)
{
continue;
}
if (!previewIds.Contains(previewId))
{
previewIds.Add(previewId);
}
}
}
else if (purchase.Preview != null)
{
for (int i = 0; i < purchase.Preview.Count; i++)
{
if (purchase.Preview[i] is long previewId)
{
if (!previewIds.Contains(previewId))
{
previewIds.Add(previewId);
}
}
}
}
foreach (MessageEntities.Medium medium in purchase.Media)
{
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = previewIds.Contains(medium.Id);
if (previewIds.Count > 0)
{
bool has = previewIds.Any(cus => cus.Equals(medium.Id));
if (!has && medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, purchase.Id,
fullUrl, null, null, null, "Messages", mediaType, isPreview, false,
null);
paidMessageCollection.PaidMessages.Add(medium.Id, fullUrl);
paidMessageCollection.PaidMessageMedia.Add(medium);
}
}
else if (!has && medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, purchase.Id,
manifestDash, null, null, null, "Messages", mediaType, isPreview, false,
null);
paidMessageCollection.PaidMessages.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
paidMessageCollection.PaidMessageMedia.Add(medium);
}
}
}
else
{
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, purchase.Id,
fullUrl, null, null, null, "Messages", mediaType, isPreview, false,
null);
paidMessageCollection.PaidMessages.Add(medium.Id, fullUrl);
paidMessageCollection.PaidMessageMedia.Add(medium);
}
}
else if (medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id))
{
await dbService.AddMedia(folder, medium.Id, purchase.Id,
manifestDash, null, null, null, "Messages", mediaType, isPreview, false,
null);
paidMessageCollection.PaidMessages.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
paidMessageCollection.PaidMessageMedia.Add(medium);
}
}
}
}
}
}
return paidMessageCollection;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return new PurchasedEntities.PaidMessageCollection();
}
///
/// Retrieves users that appear in the Purchased tab.
///
/// The purchased tab endpoint.
/// Known users map.
///
/// A username-to-userId map.
public async Task> GetPurchasedTabUsers(string endpoint, Dictionary users, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetPurchasedTabUsers - {endpoint}");
try
{
Dictionary purchasedTabUsers = new();
Dictionary getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() },
{ "order", "publish_date_desc" },
{ "format", "infinite" },
{ "skip_users", "all" }
};
cancellationToken.ThrowIfCancellationRequested();
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
if (body == null)
{
throw new Exception("Failed to get purchased tab users. null body returned.");
}
PurchasedDtos.PurchasedDto? purchasedDto =
DeserializeJson(body, s_mJsonSerializerSettings);
PurchasedEntities.Purchased purchased = PurchasedMapper.FromDto(purchasedDto);
if (purchased.HasMore)
{
getParams["offset"] = purchased.List.Count.ToString();
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
PurchasedEntities.Purchased newPurchased;
Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
HttpClient loopclient = GetHttpClient();
HttpRequestMessage looprequest =
new(HttpMethod.Get, $"{Constants.ApiUrl}{endpoint}{loopqueryParams}");
foreach (KeyValuePair keyValuePair in loopheaders)
{
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
using (HttpResponseMessage loopresponse = await loopclient.SendAsync(looprequest, cancellationToken))
{
loopresponse.EnsureSuccessStatusCode();
string loopbody = await loopresponse.Content.ReadAsStringAsync();
PurchasedDtos.PurchasedDto? newPurchasedDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
newPurchased = PurchasedMapper.FromDto(newPurchasedDto);
}
purchased.List.AddRange(newPurchased.List);
if (!newPurchased.HasMore)
{
break;
}
getParams["offset"] =
Convert.ToString(Convert.ToInt32(getParams["offset"]) + Constants.ApiPageSize);
}
}
if (purchased.List.Count > 0)
{
foreach (PurchasedEntities.ListItem purchase in
purchased.List.OrderByDescending(p => p.PostedAt ?? p.CreatedAt))
{
cancellationToken.ThrowIfCancellationRequested();
long fromUserId = purchase.FromUser?.Id ?? 0;
long authorId = purchase.Author?.Id ?? 0;
if (fromUserId != 0)
{
if (users.Values.Contains(fromUserId))
{
string? matchedUsername = users.FirstOrDefault(x => x.Value == fromUserId).Key;
if (!string.IsNullOrEmpty(matchedUsername))
{
purchasedTabUsers.TryAdd(matchedUsername, fromUserId);
}
else if (!purchasedTabUsers.ContainsKey($"Deleted User - {fromUserId}"))
{
purchasedTabUsers.Add($"Deleted User - {fromUserId}", fromUserId);
}
}
else
{
JObject? user = await GetUserInfoById($"/users/list?x[]={fromUserId}");
string? fetchedUsername = user?[fromUserId.ToString()]?["username"]?.ToString();
if (string.IsNullOrEmpty(fetchedUsername))
{
if (!configService.CurrentConfig.BypassContentForCreatorsWhoNoLongerExist &&
!purchasedTabUsers.ContainsKey($"Deleted User - {fromUserId}"))
{
purchasedTabUsers.Add($"Deleted User - {fromUserId}", fromUserId);
}
Log.Debug("Content creator not longer exists - {0}", fromUserId);
}
else
{
purchasedTabUsers.TryAdd(fetchedUsername, fromUserId);
}
}
}
else if (authorId != 0)
{
if (users.ContainsValue(authorId))
{
string? matchedUsername = users.FirstOrDefault(x => x.Value == authorId).Key;
if (!string.IsNullOrEmpty(matchedUsername))
{
if (!purchasedTabUsers.ContainsKey(matchedUsername) &&
users.ContainsKey(matchedUsername))
{
purchasedTabUsers.Add(matchedUsername, authorId);
}
}
else if (!purchasedTabUsers.ContainsKey($"Deleted User - {authorId}"))
{
purchasedTabUsers.Add($"Deleted User - {authorId}", authorId);
}
}
else
{
JObject? user = await GetUserInfoById($"/users/list?x[]={authorId}");
string? fetchedUsername = user?[authorId.ToString()]?["username"]?.ToString();
if (string.IsNullOrEmpty(fetchedUsername))
{
if (!configService.CurrentConfig.BypassContentForCreatorsWhoNoLongerExist &&
!purchasedTabUsers.ContainsKey($"Deleted User - {authorId}"))
{
purchasedTabUsers.Add($"Deleted User - {authorId}", authorId);
}
Log.Debug("Content creator not longer exists - {0}", authorId);
}
else if (!purchasedTabUsers.ContainsKey(fetchedUsername) &&
users.ContainsKey(fetchedUsername))
{
purchasedTabUsers.Add(fetchedUsername, authorId);
}
}
}
}
}
return purchasedTabUsers;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return new Dictionary();
}
///
/// Retrieves Purchased tab content grouped by user.
///
/// The purchased tab endpoint.
/// The base download folder.
/// Known users map.
///
/// A list of purchased tab collections.
public async Task> GetPurchasedTab(string endpoint, string folder,
Dictionary users, CancellationToken cancellationToken = default)
{
Log.Debug($"Calling GetPurchasedTab - {endpoint}");
try
{
Dictionary> userPurchases = new();
List purchasedTabCollections = [];
Dictionary getParams = new()
{
{ "limit", Constants.ApiPageSize.ToString() },
{ "order", "publish_date_desc" },
{ "format", "infinite" },
{ "skip_users", "all" }
};
cancellationToken.ThrowIfCancellationRequested();
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient());
PurchasedDtos.PurchasedDto? purchasedDto =
DeserializeJson(body, s_mJsonSerializerSettings);
PurchasedEntities.Purchased purchased = PurchasedMapper.FromDto(purchasedDto);
if (purchased.HasMore)
{
getParams["offset"] = purchased.List.Count.ToString();
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
PurchasedEntities.Purchased newPurchased;
Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams);
HttpClient loopclient = GetHttpClient();
HttpRequestMessage looprequest =
new(HttpMethod.Get, $"{Constants.ApiUrl}{endpoint}{loopqueryParams}");
foreach (KeyValuePair keyValuePair in loopheaders)
{
looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
using (HttpResponseMessage loopresponse = await loopclient.SendAsync(looprequest, cancellationToken))
{
loopresponse.EnsureSuccessStatusCode();
string loopbody = await loopresponse.Content.ReadAsStringAsync();
PurchasedDtos.PurchasedDto? newPurchasedDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
newPurchased = PurchasedMapper.FromDto(newPurchasedDto);
}
purchased.List.AddRange(newPurchased.List);
if (!newPurchased.HasMore)
{
break;
}
getParams["offset"] =
Convert.ToString(Convert.ToInt32(getParams["offset"]) + Constants.ApiPageSize);
}
}
if (purchased.List.Count > 0)
{
foreach (PurchasedEntities.ListItem purchase in
purchased.List.OrderByDescending(p => p.PostedAt ?? p.CreatedAt))
{
cancellationToken.ThrowIfCancellationRequested();
if (purchase.FromUser != null)
{
if (!userPurchases.ContainsKey(purchase.FromUser.Id))
{
userPurchases.Add(purchase.FromUser.Id, new List());
}
userPurchases[purchase.FromUser.Id].Add(purchase);
}
else if (purchase.Author != null)
{
if (!userPurchases.ContainsKey(purchase.Author.Id))
{
userPurchases.Add(purchase.Author.Id, new List());
}
userPurchases[purchase.Author.Id].Add(purchase);
}
}
}
foreach (KeyValuePair> user in userPurchases)
{
cancellationToken.ThrowIfCancellationRequested();
PurchasedEntities.PurchasedTabCollection purchasedTabCollection = new();
JObject? userObject = await GetUserInfoById($"/users/list?x[]={user.Key}");
purchasedTabCollection.UserId = user.Key;
string? fetchedUsername = userObject?[user.Key.ToString()]?["username"]?.ToString();
purchasedTabCollection.Username = !string.IsNullOrEmpty(fetchedUsername)
? fetchedUsername
: $"Deleted User - {user.Key}";
string path = Path.Combine(folder, purchasedTabCollection.Username);
if (Path.Exists(path))
{
foreach (PurchasedEntities.ListItem purchase in user.Value)
{
if (purchase.Media == null)
{
Log.Warning(
"PurchasedTab purchase media null, setting empty list | userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt}",
user.Key, purchasedTabCollection.Username, purchase.Id, purchase.ResponseType,
purchase.CreatedAt, purchase.PostedAt);
purchase.Media = new List();
}
switch (purchase.ResponseType)
{
case "post":
List previewids = new();
if (purchase.Previews != null)
{
for (int i = 0; i < purchase.Previews.Count; i++)
{
if (purchase.Previews[i] is long previewId)
{
if (!previewids.Contains(previewId))
{
previewids.Add(previewId);
}
}
}
}
else if (purchase.Preview != null)
{
for (int i = 0; i < purchase.Preview.Count; i++)
{
if (purchase.Preview[i] is long previewId)
{
if (!previewids.Contains(previewId))
{
previewids.Add(previewId);
}
}
}
}
DateTime createdAt = purchase.CreatedAt ?? purchase.PostedAt ?? DateTime.Now;
bool isArchived = purchase.IsArchived ?? false;
await dbService.AddPost(path, purchase.Id,
purchase.Text ?? "",
purchase.Price ?? "0",
purchase is { Price: not null, IsOpened: true },
isArchived,
createdAt);
purchasedTabCollection.PaidPosts.PaidPostObjects.Add(purchase);
foreach (MessageEntities.Medium medium in purchase.Media)
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = previewids.Contains(medium.Id);
if (previewids.Count > 0)
{
bool has = previewids.Any(cus => cus.Equals(medium.Id));
if (!has && medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(path, medium.Id, purchase.Id, fullUrl, null,
null, null, "Posts", mediaType, isPreview, false, null);
purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id, fullUrl);
purchasedTabCollection.PaidPosts.PaidPostMedia.Add(medium);
}
}
else if (!has && medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(path, medium.Id, purchase.Id, manifestDash,
null, null, null, "Posts", mediaType, isPreview, false, null);
purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
purchasedTabCollection.PaidPosts.PaidPostMedia.Add(medium);
}
}
}
else
{
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(path, medium.Id, purchase.Id, fullUrl, null,
null, null, "Posts", mediaType, isPreview, false, null);
purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id, fullUrl);
purchasedTabCollection.PaidPosts.PaidPostMedia.Add(medium);
}
}
else if (medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id))
{
await dbService.AddMedia(path, medium.Id, purchase.Id, manifestDash,
null, null, null, "Posts", mediaType, isPreview, false, null);
purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
purchasedTabCollection.PaidPosts.PaidPostMedia.Add(medium);
}
}
}
}
break;
case "message":
DateTime messageCreatedAt = purchase.PostedAt ?? purchase.CreatedAt ?? DateTime.Now;
long fromUserId = purchase.FromUser?.Id ?? long.MinValue;
await dbService.AddMessage(path, purchase.Id,
purchase.Text ?? "",
purchase.Price ?? "0", true, false,
messageCreatedAt, fromUserId);
purchasedTabCollection.PaidMessages.PaidMessageObjects.Add(purchase);
if (purchase.Media is { Count: > 0 })
{
List paidMessagePreviewIds = [];
if (purchase.Previews != null)
{
for (int i = 0; i < purchase.Previews.Count; i++)
{
if (purchase.Previews[i] is long previewId)
{
if (!paidMessagePreviewIds.Contains(previewId))
{
paidMessagePreviewIds.Add(previewId);
}
}
}
}
else if (purchase.Preview != null)
{
for (int i = 0; i < purchase.Preview.Count; i++)
{
if (purchase.Preview[i] is long previewId)
{
if (!paidMessagePreviewIds.Contains(previewId))
{
paidMessagePreviewIds.Add(previewId);
}
}
}
}
foreach (MessageEntities.Medium medium in purchase.Media)
{
if (paidMessagePreviewIds.Count > 0)
{
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = paidMessagePreviewIds.Contains(medium.Id);
bool has = paidMessagePreviewIds.Any(cus => cus.Equals(medium.Id));
if (!has && medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey(
medium.Id))
{
await dbService.AddMedia(path, medium.Id, purchase.Id,
fullUrl, null, null, null, "Messages", mediaType,
isPreview, false, null);
purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id,
fullUrl);
purchasedTabCollection.PaidMessages.PaidMessageMedia.Add(
medium);
}
}
else if (!has && medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey(
medium.Id))
{
await dbService.AddMedia(path, medium.Id, purchase.Id,
manifestDash, null, null, null, "Messages", mediaType,
isPreview, false, null);
purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
purchasedTabCollection.PaidMessages.PaidMessageMedia.Add(
medium);
}
}
}
else
{
string mediaType = ResolveMediaType(medium.Type) ?? string.Empty;
string? fullUrl = medium.Files?.Full?.Url;
bool isPreview = paidMessagePreviewIds.Contains(medium.Id);
if (medium.CanView && !string.IsNullOrEmpty(fullUrl))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey(
medium.Id))
{
await dbService.AddMedia(path, medium.Id, purchase.Id,
fullUrl, null, null, null, "Messages", mediaType,
isPreview, false, null);
purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id,
fullUrl);
purchasedTabCollection.PaidMessages.PaidMessageMedia.Add(
medium);
}
}
else if (medium.CanView &&
TryGetDrmInfo(medium.Files, out string manifestDash,
out string cloudFrontPolicy, out string cloudFrontSignature,
out string cloudFrontKeyPairId))
{
if (!IsMediaTypeDownloadEnabled(medium.Type))
{
continue;
}
if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey(
medium.Id))
{
await dbService.AddMedia(path, medium.Id, purchase.Id,
manifestDash, null, null, null, "Messages", mediaType,
isPreview, false, null);
purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id,
$"{manifestDash},{cloudFrontPolicy},{cloudFrontSignature},{cloudFrontKeyPairId},{medium.Id},{purchase.Id}");
purchasedTabCollection.PaidMessages.PaidMessageMedia.Add(
medium);
}
}
}
}
}
break;
}
}
purchasedTabCollections.Add(purchasedTabCollection);
}
}
return purchasedTabCollections;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return [];
}
///
/// Retrieves DRM metadata (PSSH, Last-Modified, and duration) from an MPD manifest
///
/// The MPD URL.
/// CloudFront policy token.
/// CloudFront signature token.
/// CloudFront key pair ID.
/// Tuple with PSSH, Last-Modified, and duration seconds.
public async Task<(string pssh, DateTime lastModified, double? durationSeconds)> GetDrmMpdInfo(
string mpdUrl, string policy, string signature, string kvp)
{
Log.Debug("Calling GetDrmMpdInfo");
Log.Debug("mpdUrl: {MpdUrl}", mpdUrl);
Log.Debug("policy: {Policy}", policy);
Log.Debug("signature: {Signature}", signature);
Log.Debug("kvp: {Kvp}", kvp);
try
{
Auth? currentAuth = authService.CurrentAuth;
if (currentAuth?.UserAgent == null || currentAuth.Cookie == null)
{
throw new Exception("Auth service is missing required fields");
}
HttpClient client = new();
HttpRequestMessage request = new(HttpMethod.Get, mpdUrl);
request.Headers.Add("user-agent", currentAuth.UserAgent);
request.Headers.Add("Accept", "*/*");
request.Headers.Add("Cookie",
$"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {currentAuth.Cookie};");
using HttpResponseMessage response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
DateTime lastModified = response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now;
if (response.Content.Headers.LastModified == null
&& response.Headers.TryGetValues("Last-Modified", out IEnumerable? lastModifiedValues))
{
string? lastModifiedRaw = lastModifiedValues.FirstOrDefault();
if (!string.IsNullOrWhiteSpace(lastModifiedRaw)
&& DateTimeOffset.TryParse(lastModifiedRaw, out DateTimeOffset parsedLastModified))
{
lastModified = parsedLastModified.LocalDateTime;
}
}
string body = await response.Content.ReadAsStringAsync();
XDocument xmlDoc = XDocument.Parse(body);
XNamespace cenc = "urn:mpeg:cenc:2013";
List psshElements = xmlDoc.Descendants(cenc + "pssh").ToList();
string pssh = psshElements.Skip(1).FirstOrDefault()?.Value
?? psshElements.FirstOrDefault()?.Value
?? string.Empty;
string? durationText = xmlDoc.Root?.Attribute("mediaPresentationDuration")?.Value
?? xmlDoc.Root?.Elements().FirstOrDefault(e => e.Name.LocalName == "Period")
?.Attribute("duration")?.Value;
double? durationSeconds = ParseDurationSeconds(durationText);
return (pssh, lastModified, durationSeconds);
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return (string.Empty, DateTime.Now, null);
}
///
/// Retrieves a decryption key via the OFDL fallback service.
///
/// The DRM request headers.
/// The license URL.
/// The PSSH payload.
/// The decryption key string.
public async Task GetDecryptionKeyOfdl(Dictionary drmHeaders, string licenceUrl,
string pssh)
{
Log.Debug("Calling GetDecryptionKeyOfdl");
try
{
HttpClient client = new();
int attempt = 0;
OfdlRequest ofdlRequest = new()
{
Pssh = pssh, LicenseUrl = licenceUrl, Headers = JsonConvert.SerializeObject(drmHeaders)
};
string json = JsonConvert.SerializeObject(ofdlRequest);
Log.Debug("Posting to ofdl.tools: {Json}", json);
while (attempt < MaxAttempts)
{
attempt++;
HttpRequestMessage request = new(HttpMethod.Post, "https://ofdl.tools/WV")
{
Content = new StringContent(json, Encoding.UTF8, "application/json")
};
using HttpResponseMessage response = await client.SendAsync(request);
if (!response.IsSuccessStatusCode)
{
continue;
}
string body = await response.Content.ReadAsStringAsync();
if (!body.TrimStart().StartsWith('{'))
{
return body;
}
Log.Debug($"Received JSON object instead of string. Retrying... Attempt {attempt} of {MaxAttempts}");
await Task.Delay(DelayBetweenAttempts);
}
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return string.Empty;
}
///
/// Retrieves a decryption key using the local CDM integration.
///
/// The DRM request headers.
/// The license URL.
/// The PSSH payload.
/// The decryption key string.
public async Task GetDecryptionKeyCdm(Dictionary drmHeaders, string licenceUrl,
string pssh)
{
Log.Debug("Calling GetDecryptionKeyCDM");
try
{
byte[] resp1 = await PostData(licenceUrl, drmHeaders, [0x08, 0x04]);
string certDataB64 = Convert.ToBase64String(resp1);
CDMApi cdm = new();
byte[]? challenge = cdm.GetChallenge(pssh, certDataB64);
if (challenge == null)
{
throw new Exception("Failed to get challenge from CDM");
}
byte[] resp2 = await PostData(licenceUrl, drmHeaders, challenge);
string licenseB64 = Convert.ToBase64String(resp2);
Log.Debug("resp1: {Resp1}", resp1);
Log.Debug("certDataB64: {CertDataB64}", certDataB64);
Log.Debug("challenge: {Challenge}", challenge);
Log.Debug("resp2: {Resp2}", resp2);
Log.Debug("licenseB64: {LicenseB64}", licenseB64);
cdm.ProvideLicense(licenseB64);
List keys = cdm.GetKeys();
if (keys.Count > 0)
{
Log.Debug("GetDecryptionKeyCDM Key: {ContentKey}", keys[0]);
return keys[0].ToString();
}
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return string.Empty;
}
private async Task BuildHeaderAndExecuteRequests(Dictionary getParams, string endpoint,
HttpClient client)
{
Log.Debug("Calling BuildHeaderAndExecuteRequests");
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
using HttpResponseMessage response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync();
Log.Debug(body);
return body;
}
private Task BuildHttpRequestMessage(Dictionary getParams,
string endpoint)
{
Log.Debug("Calling BuildHttpRequestMessage");
string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
Dictionary headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.ApiUrl}{endpoint}{queryParams}");
Log.Debug($"Full request URL: {Constants.ApiUrl}{endpoint}{queryParams}");
foreach (KeyValuePair keyValuePair in headers)
{
request.Headers.Add(keyValuePair.Key, keyValuePair.Value);
}
return Task.FromResult(request);
}
private static double? ParseDurationSeconds(string? iso8601Duration)
{
if (string.IsNullOrWhiteSpace(iso8601Duration))
{
return null;
}
try
{
TimeSpan duration = XmlConvert.ToTimeSpan(iso8601Duration);
return duration.TotalSeconds > 0 ? duration.TotalSeconds : null;
}
catch (FormatException)
{
return null;
}
}
private static double ConvertToUnixTimestampWithMicrosecondPrecision(DateTime date)
{
DateTime origin = new(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
TimeSpan diff = date.ToUniversalTime() - origin;
return
diff.TotalSeconds; // This gives the number of seconds. If you need milliseconds, use diff.TotalMilliseconds
}
private static bool IsStringOnlyDigits(string input) => input.All(char.IsDigit);
private HttpClient GetHttpClient()
{
HttpClient client = new();
if (configService.CurrentConfig.Timeout is > 0)
{
client.Timeout = TimeSpan.FromSeconds(configService.CurrentConfig.Timeout.Value);
}
return client;
}
private static T? DeserializeJson(string? body, JsonSerializerSettings? settings = null)
{
if (string.IsNullOrWhiteSpace(body))
{
return default;
}
return settings == null
? JsonConvert.DeserializeObject(body)
: JsonConvert.DeserializeObject(body, settings);
}
private bool IsMediaTypeDownloadEnabled(string? type)
{
if (string.IsNullOrWhiteSpace(type))
{
return true;
}
return type switch
{
"photo" => configService.CurrentConfig.DownloadImages,
"video" => configService.CurrentConfig.DownloadVideos,
"gif" => configService.CurrentConfig.DownloadVideos,
"audio" => configService.CurrentConfig.DownloadAudios,
_ => true
};
}
private static string? ResolveMediaType(string? type) =>
type switch
{
"photo" => "Images",
"video" => "Videos",
"gif" => "Videos",
"audio" => "Audios",
_ => null
};
private static bool TryGetDrmInfo(Files? files, out string manifestDash, out string cloudFrontPolicy,
out string cloudFrontSignature, out string cloudFrontKeyPairId)
{
manifestDash = string.Empty;
cloudFrontPolicy = string.Empty;
cloudFrontSignature = string.Empty;
cloudFrontKeyPairId = string.Empty;
string? dash = files?.Drm?.Manifest?.Dash;
Dash? signatureDash = files?.Drm?.Signature?.Dash;
if (string.IsNullOrEmpty(dash) || signatureDash == null)
{
return false;
}
if (string.IsNullOrEmpty(signatureDash.CloudFrontPolicy) ||
string.IsNullOrEmpty(signatureDash.CloudFrontSignature) ||
string.IsNullOrEmpty(signatureDash.CloudFrontKeyPairId))
{
return false;
}
manifestDash = dash;
cloudFrontPolicy = signatureDash.CloudFrontPolicy;
cloudFrontSignature = signatureDash.CloudFrontSignature;
cloudFrontKeyPairId = signatureDash.CloudFrontKeyPairId;
return true;
}
private int GetCurrentUserIdOrDefault()
{
if (authService.CurrentAuth?.UserId == null)
{
return int.MinValue;
}
return int.TryParse(authService.CurrentAuth.UserId, out int userId) ? userId : int.MinValue;
}
///
/// this one is used during initialization only
/// if the config option is not available, then no modifications will be done on the getParams
///
///
///
///
private static void UpdateGetParamsForDateSelection(DownloadDateSelection downloadDateSelection,
ref Dictionary getParams, DateTime? dt)
{
//if (config.DownloadOnlySpecificDates && dt.HasValue)
//{
if (dt.HasValue)
{
UpdateGetParamsForDateSelection(
downloadDateSelection,
ref getParams,
ConvertToUnixTimestampWithMicrosecondPrecision(dt.Value)
.ToString("0.000000", CultureInfo.InvariantCulture)
);
}
//}
}
private static void UpdateGetParamsForDateSelection(DownloadDateSelection downloadDateSelection,
ref Dictionary getParams, string? unixTimeStampInMicrosec)
{
if (string.IsNullOrWhiteSpace(unixTimeStampInMicrosec))
{
return;
}
switch (downloadDateSelection)
{
case DownloadDateSelection.before:
getParams["beforePublishTime"] = unixTimeStampInMicrosec;
break;
case DownloadDateSelection.after:
getParams["order"] = "publish_date_asc";
getParams["afterPublishTime"] = unixTimeStampInMicrosec;
break;
}
}
private async Task?> GetAllSubscriptions(Dictionary getParams,
string endpoint, bool includeRestricted)
{
if (!HasSignedRequestAuth())
{
return null;
}
try
{
Dictionary users = new();
Log.Debug("Calling GetAllSubscrptions");
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
SubscriptionsDtos.SubscriptionsDto? subscriptionsDto =
DeserializeJson(body, s_mJsonSerializerSettings);
SubscriptionEntities.Subscriptions subscriptions = SubscriptionsMapper.FromDto(subscriptionsDto);
if (subscriptions.HasMore)
{
getParams["offset"] = subscriptions.List.Count.ToString();
while (true)
{
SubscriptionEntities.Subscriptions newSubscriptions;
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
{
SubscriptionsDtos.SubscriptionsDto? newSubscriptionsDto =
DeserializeJson(loopbody, s_mJsonSerializerSettings);
newSubscriptions = SubscriptionsMapper.FromDto(newSubscriptionsDto);
}
else
{
break;
}
subscriptions.List.AddRange(newSubscriptions.List);
if (!newSubscriptions.HasMore)
{
break;
}
getParams["offset"] = subscriptions.List.Count.ToString();
}
}
foreach (SubscriptionEntities.ListItem subscription in subscriptions.List)
{
if ((!(subscription.IsRestricted ?? false) ||
((subscription.IsRestricted ?? false) && includeRestricted))
&& !users.ContainsKey(subscription.Username))
{
users.Add(subscription.Username, subscription.Id);
}
}
return users;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return null;
}
private static string? GetDynamicRules()
{
Log.Debug("Calling GetDynamicRules");
try
{
HttpClient client = new();
HttpRequestMessage request = new(HttpMethod.Get,
"https://git.ofdl.tools/sim0n00ps/dynamic-rules/raw/branch/main/rules.json");
using HttpResponseMessage response = client.Send(request);
if (!response.IsSuccessStatusCode)
{
Log.Debug("GetDynamicRules did not return a Success Status Code");
return null;
}
string body = response.Content.ReadAsStringAsync().Result;
Log.Debug("GetDynamicRules Response: ");
Log.Debug(body);
return body;
}
catch (Exception ex)
{
ExceptionLoggerHelper.LogException(ex);
}
return null;
}
}