using System.Globalization; using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Xml.Linq; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using OF_DL.Models; using OF_DL.Models.Post; using OF_DL.Models.Purchased; using OF_DL.Models.Stories; using OF_DL.Models.Streams; using OF_DL.Enumerations; using ArchivedDtos = OF_DL.Models.Dtos.Archived; using MessageDtos = OF_DL.Models.Dtos.Messages; using ListDtos = OF_DL.Models.Dtos.Lists; using HighlightDtos = OF_DL.Models.Dtos.Highlights; using ArchivedEntities = OF_DL.Models.Entities.Archived; using HighlightEntities = OF_DL.Models.Entities.Highlights; using ListEntities = OF_DL.Models.Entities.Lists; using MessageEntities = OF_DL.Models.Entities.Messages; using OF_DL.Models.Mappers; using OF_DL.Widevine; using Serilog; using Spectre.Console; using static OF_DL.Utils.HttpUtil; using Constants = OF_DL.Helpers.Constants; namespace OF_DL.Services; public class APIService(IAuthService authService, IConfigService configService, IDBService dbService) : IAPIService { private const int MaxAttempts = 30; private const int DelayBetweenAttempts = 3000; private static readonly JsonSerializerSettings m_JsonSerializerSettings; private static DateTime? cachedDynamicRulesExpiration; private static DynamicRules? cachedDynamicRules; static APIService() => m_JsonSerializerSettings = new JsonSerializerSettings { MissingMemberHandling = MissingMemberHandling.Ignore }; public Dictionary GetDynamicHeaders(string path, string queryParams) { Log.Debug("Calling GetDynamicHeaders"); Log.Debug($"Path: {path}"); Log.Debug($"Query Params: {queryParams}"); DynamicRules? root; //Check if we have a cached version of the dynamic rules if (cachedDynamicRules != null && cachedDynamicRulesExpiration.HasValue && DateTime.UtcNow < cachedDynamicRulesExpiration) { Log.Debug("Using cached dynamic rules"); root = cachedDynamicRules; } else { //Get rules from GitHub and fallback to local file string? dynamicRulesJSON = GetDynamicRules(); if (!string.IsNullOrEmpty(dynamicRulesJSON)) { Log.Debug("Using dynamic rules from GitHub"); root = JsonConvert.DeserializeObject(dynamicRulesJSON); // Cache the GitHub response for 15 minutes cachedDynamicRules = root; cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(15); } else { Log.Debug("Using dynamic rules from local file"); root = JsonConvert.DeserializeObject(File.ReadAllText("rules.json")); // Cache the dynamic rules from local file to prevent unnecessary disk // operations and frequent call to GitHub. Since the GitHub dynamic rules // are preferred to the local file, the cache time is shorter than when dynamic rules // are successfully retrieved from GitHub. cachedDynamicRules = root; cachedDynamicRulesExpiration = DateTime.UtcNow.AddMinutes(5); } } DateTimeOffset dto = DateTime.UtcNow; long timestamp = dto.ToUnixTimeMilliseconds(); string input = $"{root!.StaticParam}\n{timestamp}\n{path + queryParams}\n{authService.CurrentAuth.USER_ID}"; byte[] inputBytes = Encoding.UTF8.GetBytes(input); byte[] hashBytes = SHA1.HashData(inputBytes); string hashString = BitConverter.ToString(hashBytes).Replace("-", "").ToLower(); int checksum = root.ChecksumIndexes.Aggregate(0, (current, number) => current + hashString[number]) + root.ChecksumConstant!.Value; string sign = $"{root.Prefix}:{hashString}:{checksum.ToString("X").ToLower()}:{root.Suffix}"; Dictionary headers = new() { { "accept", "application/json, text/plain" }, { "app-token", root.AppToken! }, { "cookie", authService.CurrentAuth!.COOKIE! }, { "sign", sign }, { "time", timestamp.ToString() }, { "user-id", authService.CurrentAuth!.USER_ID! }, { "user-agent", authService.CurrentAuth!.USER_AGENT! }, { "x-bc", authService.CurrentAuth!.X_BC! } }; return headers; } public async Task GetUserInfo(string endpoint) { Log.Debug($"Calling GetUserInfo: {endpoint}"); try { User? user = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "publish_date_asc" } }; HttpClient client = new(); HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint); using HttpResponseMessage response = await client.SendAsync(request); if (!response.IsSuccessStatusCode) { return user; } response.EnsureSuccessStatusCode(); string body = await response.Content.ReadAsStringAsync(); user = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); return user; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetUserInfoById(string endpoint) { try { HttpClient client = new(); HttpRequestMessage request = await BuildHttpRequestMessage(new Dictionary(), endpoint); using HttpResponseMessage response = await client.SendAsync(request); response.EnsureSuccessStatusCode(); string body = await response.Content.ReadAsStringAsync(); //if the content creator doesnt exist, we get a 200 response, but the content isnt usable //so let's not throw an exception, since "content creator no longer exists" is handled elsewhere //which means we wont get loads of exceptions if (body.Equals("[]")) { return null; } JObject jObject = JObject.Parse(body); return jObject; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task?> GetActiveSubscriptions(string endpoint, bool includeRestricted) { Dictionary getParams = new() { { "offset", "0" }, { "limit", "50" }, { "type", "active" }, { "format", "infinite" } }; return await GetAllSubscriptions(getParams, endpoint, includeRestricted); } public async Task?> GetExpiredSubscriptions(string endpoint, bool includeRestricted) { Dictionary getParams = new() { { "offset", "0" }, { "limit", "50" }, { "type", "expired" }, { "format", "infinite" } }; Log.Debug("Calling GetExpiredSubscriptions"); return await GetAllSubscriptions(getParams, endpoint, includeRestricted); } public async Task> GetLists(string endpoint) { Log.Debug("Calling GetLists"); try { int offset = 0; Dictionary getParams = new() { { "offset", offset.ToString() }, { "skip_users", "all" }, { "limit", "50" }, { "format", "infinite" } }; Dictionary lists = new(); while (true) { string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); if (body == null) { break; } ListDtos.UserListDto? userListDto = JsonConvert.DeserializeObject(body); ListEntities.UserList userList = UserListsMapper.FromDto(userListDto); foreach (ListEntities.UserListItem listItem in userList.List) { if (IsStringOnlyDigits(listItem.Id) && !lists.ContainsKey(listItem.Name)) { lists.Add(listItem.Name, Convert.ToInt32(listItem.Id)); } } if (userList.HasMore) { offset += 50; getParams["offset"] = Convert.ToString(offset); } else { break; } } return lists; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task?> GetListUsers(string endpoint) { Log.Debug($"Calling GetListUsers - {endpoint}"); try { int offset = 0; Dictionary getParams = new() { { "offset", offset.ToString() }, { "limit", "50" } }; List users = new(); while (true) { string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); if (body == null) { break; } List? usersListDto = JsonConvert.DeserializeObject>(body); List usersList = UserListsMapper.FromDto(usersListDto); if (usersList.Count <= 0) { break; } foreach (ListEntities.UsersList ul in usersList) { users.Add(ul.Username); } if (users.Count < 50) { break; } offset += 50; getParams["offset"] = Convert.ToString(offset); } return users; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task> GetMedia(MediaType mediatype, string endpoint, string? username, string folder, List paid_post_ids) { Log.Debug($"Calling GetMedia - {username}"); try { Dictionary return_urls = new(); int post_limit = 50; int limit = 5; int offset = 0; Dictionary getParams = new(); switch (mediatype) { case MediaType.Stories: getParams = new Dictionary { { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, { "skip_users", "all" } }; break; case MediaType.Highlights: getParams = new Dictionary { { "limit", limit.ToString() }, { "offset", offset.ToString() }, { "skip_users", "all" } }; break; } string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); if (mediatype == MediaType.Stories) { Log.Debug("Media Stories - " + endpoint); List stories = JsonConvert.DeserializeObject>(body, m_JsonSerializerSettings) ?? new List(); foreach (Stories story in stories) { if (story.media[0].createdAt.HasValue) { await dbService.AddStory(folder, story.id, string.Empty, "0", false, false, story.media[0].createdAt.Value); } else if (story.createdAt.HasValue) { await dbService.AddStory(folder, story.id, string.Empty, "0", false, false, story.createdAt.Value); } else { await dbService.AddStory(folder, story.id, string.Empty, "0", false, false, DateTime.Now); } if (story.media != null && story.media.Count > 0) { foreach (Stories.Medium medium in story.media) { await dbService.AddMedia(folder, medium.id, story.id, medium.files.full.url, null, null, null, "Stories", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, false, false, null); if (medium.type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (medium.canView) { if (!return_urls.ContainsKey(medium.id)) { return_urls.Add(medium.id, medium.files.full.url); } } } } } } else if (mediatype == MediaType.Highlights) { List highlight_ids = new(); HighlightDtos.HighlightsDto? highlightsDto = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); HighlightEntities.Highlights highlights = HighlightsMapper.FromDto(highlightsDto); if (highlights.HasMore) { offset += 5; getParams["offset"] = offset.ToString(); while (true) { Log.Debug("Media Highlights - " + endpoint); string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); HighlightDtos.HighlightsDto? newHighlightsDto = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); HighlightEntities.Highlights newHighlights = HighlightsMapper.FromDto(newHighlightsDto); highlights.List.AddRange(newHighlights.List); if (!newHighlights.HasMore) { break; } offset += 5; getParams["offset"] = offset.ToString(); } } foreach (HighlightEntities.ListItem list in highlights.List) { if (!highlight_ids.Contains(list.Id.ToString())) { highlight_ids.Add(list.Id.ToString()); } } foreach (string highlight_id in highlight_ids) { Dictionary highlight_headers = GetDynamicHeaders("/api2/v2/stories/highlights/" + highlight_id, string.Empty); HttpClient highlight_client = GetHttpClient(); HttpRequestMessage highlight_request = new(HttpMethod.Get, $"https://onlyfans.com/api2/v2/stories/highlights/{highlight_id}"); foreach (KeyValuePair keyValuePair in highlight_headers) { highlight_request.Headers.Add(keyValuePair.Key, keyValuePair.Value); } using HttpResponseMessage highlightResponse = await highlight_client.SendAsync(highlight_request); highlightResponse.EnsureSuccessStatusCode(); string highlightBody = await highlightResponse.Content.ReadAsStringAsync(); HighlightDtos.HighlightMediaDto? highlightMediaDto = JsonConvert.DeserializeObject(highlightBody, m_JsonSerializerSettings); HighlightEntities.HighlightMedia highlightMedia = HighlightsMapper.FromDto(highlightMediaDto); foreach (HighlightEntities.Story item in highlightMedia.Stories) { if (item.Media != null && item.Media.Count > 0 && item.Media[0].CreatedAt.HasValue) { await dbService.AddStory(folder, item.Id, string.Empty, "0", false, false, item.Media[0].CreatedAt.Value); } else if (item.CreatedAt.HasValue) { await dbService.AddStory(folder, item.Id, string.Empty, "0", false, false, item.CreatedAt.Value); } else { await dbService.AddStory(folder, item.Id, string.Empty, "0", false, false, DateTime.Now); } if (item.Media != null && item.Media.Count > 0 && item.Media[0].CanView) { foreach (HighlightEntities.Medium medium in item.Media) { string storyUrl = item.Media[0].Files?.Full?.Url ?? string.Empty; await dbService.AddMedia(folder, medium.Id, item.Id, storyUrl, null, null, null, "Stories", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, false, false, null); if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!return_urls.ContainsKey(medium.Id) && !string.IsNullOrEmpty(storyUrl)) { return_urls.Add(medium.Id, storyUrl); } } } } } } return return_urls; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetPaidPosts(string endpoint, string folder, string username, List paid_post_ids, StatusContext ctx) { Log.Debug($"Calling GetPaidPosts - {username}"); try { Purchased paidPosts = new(); PaidPostCollection paidPostCollection = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "skip_users", "all" }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "author", username } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); paidPosts = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); ctx.Status($"[red]Getting Paid Posts\n[/] [red]Found {paidPosts.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (paidPosts != null && paidPosts.hasMore) { getParams["offset"] = paidPosts.list.Count.ToString(); while (true) { Purchased newPaidPosts = new(); string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); newPaidPosts = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); paidPosts.list.AddRange(newPaidPosts.list); ctx.Status($"[red]Getting Paid Posts\n[/] [red]Found {paidPosts.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (!newPaidPosts.hasMore) { break; } getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); } } foreach (Purchased.List purchase in paidPosts.list) { if (purchase.responseType == "post" && purchase.media != null && purchase.media.Count > 0) { List previewids = new(); if (purchase.previews != null) { for (int i = 0; i < purchase.previews.Count; i++) { if (purchase.previews[i] is long previewId) { if (!previewids.Contains(previewId)) { previewids.Add(previewId); } } } } else if (purchase.preview != null) { for (int i = 0; i < purchase.preview.Count; i++) { if (purchase.preview[i] is long previewId) { if (!previewids.Contains(previewId)) { previewids.Add(previewId); } } } } await dbService.AddPost(folder, purchase.id, purchase.text != null ? purchase.text : string.Empty, purchase.price != null ? purchase.price : "0", purchase.price != null && purchase.isOpened ? true : false, purchase.isArchived.HasValue ? purchase.isArchived.Value : false, purchase.createdAt != null ? purchase.createdAt.Value : purchase.postedAt.Value); paidPostCollection.PaidPostObjects.Add(purchase); foreach (MessageDtos.MediumDto medium in purchase.media) { MessageEntities.Medium mappedMedium = MessagesMapper.MapMedium(medium); if (!previewids.Contains(medium.Id)) { paid_post_ids.Add(medium.Id); } if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (previewids.Count > 0) { bool has = previewids.Any(cus => cus.Equals(medium.Id)); if (!has && medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, purchase.id, medium.Files.Full.Url, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); paidPostCollection.PaidPosts.Add(medium.Id, medium.Files.Full.Url); paidPostCollection.PaidPostMedia.Add(mappedMedium); } } else if (!has && medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, purchase.id, medium.Files.Drm.Manifest.Dash, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); paidPostCollection.PaidPosts.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{purchase.id}"); paidPostCollection.PaidPostMedia.Add(mappedMedium); } } } else { if (medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, purchase.id, medium.Files.Full.Url, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); paidPostCollection.PaidPosts.Add(medium.Id, medium.Files.Full.Url); paidPostCollection.PaidPostMedia.Add(mappedMedium); } } else if (medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (!paidPostCollection.PaidPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, purchase.id, medium.Files.Drm.Manifest.Dash, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); paidPostCollection.PaidPosts.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{purchase.id}"); paidPostCollection.PaidPostMedia.Add(mappedMedium); } } } } } } return paidPostCollection; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetPosts(string endpoint, string folder, List paid_post_ids, StatusContext ctx) { Log.Debug($"Calling GetPosts - {endpoint}"); try { Post posts = new(); PostCollection postCollection = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "skip_users", "all" } }; DownloadDateSelection downloadDateSelection = DownloadDateSelection.before; DateTime? downloadAsOf = null; if (configService.CurrentConfig.DownloadOnlySpecificDates && configService.CurrentConfig.CustomDate.HasValue) { downloadDateSelection = configService.CurrentConfig.DownloadDateSelection; downloadAsOf = configService.CurrentConfig.CustomDate; } else if (configService.CurrentConfig.DownloadPostsIncrementally) { DateTime? mostRecentPostDate = await dbService.GetMostRecentPostDate(folder); if (mostRecentPostDate.HasValue) { downloadDateSelection = DownloadDateSelection.after; downloadAsOf = mostRecentPostDate.Value.AddMinutes(-5); // Back track a little for a margin of error } } UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, downloadAsOf); string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); posts = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); ctx.Status( $"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (posts != null && posts.hasMore) { UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, posts.tailMarker); while (true) { Post newposts = new(); string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); newposts = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); posts.list.AddRange(newposts.list); ctx.Status( $"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (!newposts.hasMore) { break; } UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, newposts.tailMarker); } } foreach (Post.List post in posts.list) { if (configService.CurrentConfig.SkipAds) { if (post.rawText != null && (post.rawText.Contains("#ad") || post.rawText.Contains("/trial/") || post.rawText.Contains("#announcement"))) { continue; } if (post.text != null && (post.text.Contains("#ad") || post.text.Contains("/trial/") || post.text.Contains("#announcement"))) { continue; } } List postPreviewIds = new(); if (post.preview != null && post.preview.Count > 0) { for (int i = 0; i < post.preview.Count; i++) { if (post.preview[i] is long previewId) { if (!postPreviewIds.Contains(previewId)) { postPreviewIds.Add(previewId); } } } } await dbService.AddPost(folder, post.id, post.rawText != null ? post.rawText : string.Empty, post.price != null ? post.price : "0", post.price != null && post.isOpened ? true : false, post.isArchived, post.postedAt); postCollection.PostObjects.Add(post); if (post.media != null && post.media.Count > 0) { foreach (Post.Medium medium in post.media) { if (medium.type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (medium.canView && medium.files?.drm == null) { bool has = paid_post_ids.Any(cus => cus.Equals(medium.id)); if (medium.files!.full != null && !string.IsNullOrEmpty(medium.files!.full.url)) { if (!has) { if (!postCollection.Posts.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, post.id, medium.files!.full.url, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, postPreviewIds.Contains(medium.id) ? true : false, false, null); postCollection.Posts.Add(medium.id, medium.files!.full.url); postCollection.PostMedia.Add(medium); } } } else if (medium.files.preview != null && medium.files!.full == null) { if (!has) { if (!postCollection.Posts.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, post.id, medium.files.preview.url, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, postPreviewIds.Contains(medium.id) ? true : false, false, null); postCollection.Posts.Add(medium.id, medium.files.preview.url); postCollection.PostMedia.Add(medium); } } } } else if (medium.canView && medium.files != null && medium.files.drm != null) { bool has = paid_post_ids.Any(cus => cus.Equals(medium.id)); if (!has && medium.files != null && medium.files.drm != null) { if (!postCollection.Posts.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, post.id, medium.files.drm.manifest.dash, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, postPreviewIds.Contains(medium.id) ? true : false, false, null); postCollection.Posts.Add(medium.id, $"{medium.files.drm.manifest.dash},{medium.files.drm.signature.dash.CloudFrontPolicy},{medium.files.drm.signature.dash.CloudFrontSignature},{medium.files.drm.signature.dash.CloudFrontKeyPairId},{medium.id},{post.id}"); postCollection.PostMedia.Add(medium); } } } } } } return postCollection; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetPost(string endpoint, string folder) { Log.Debug($"Calling GetPost - {endpoint}"); try { SinglePost singlePost = new(); SinglePostCollection singlePostCollection = new(); Dictionary getParams = new() { { "skip_users", "all" } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); singlePost = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); if (singlePost != null) { List postPreviewIds = new(); if (singlePost.preview != null && singlePost.preview.Count > 0) { for (int i = 0; i < singlePost.preview.Count; i++) { if (singlePost.preview[i] is long previewId) { if (!postPreviewIds.Contains(previewId)) { postPreviewIds.Add(previewId); } } } } await dbService.AddPost(folder, singlePost.id, singlePost.text != null ? singlePost.text : string.Empty, singlePost.price != null ? singlePost.price : "0", singlePost.price != null && singlePost.isOpened ? true : false, singlePost.isArchived, singlePost.postedAt); singlePostCollection.SinglePostObjects.Add(singlePost); if (singlePost.media != null && singlePost.media.Count > 0) { foreach (SinglePost.Medium medium in singlePost.media) { if (medium.type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (medium.canView && medium.files?.drm == null) { switch (configService.CurrentConfig.DownloadVideoResolution) { case VideoResolution.source: if (medium.files!.full != null && !string.IsNullOrEmpty(medium.files!.full.url)) { if (!singlePostCollection.SinglePosts.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, singlePost.id, medium.files!.full.url, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, postPreviewIds.Contains(medium.id) ? true : false, false, null); singlePostCollection.SinglePosts.Add(medium.id, medium.files!.full.url); singlePostCollection.SinglePostMedia.Add(medium); } } break; case VideoResolution._240: if (medium.videoSources != null) { if (!string.IsNullOrEmpty(medium.videoSources._240)) { if (!singlePostCollection.SinglePosts.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, singlePost.id, medium.videoSources._240, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, postPreviewIds.Contains(medium.id) ? true : false, false, null); singlePostCollection.SinglePosts.Add(medium.id, medium.videoSources._240); singlePostCollection.SinglePostMedia.Add(medium); } } } break; case VideoResolution._720: if (medium.videoSources != null) { if (!string.IsNullOrEmpty(medium.videoSources._720)) { if (!singlePostCollection.SinglePosts.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, singlePost.id, medium.videoSources._720, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, postPreviewIds.Contains(medium.id) ? true : false, false, null); singlePostCollection.SinglePosts.Add(medium.id, medium.videoSources._720); singlePostCollection.SinglePostMedia.Add(medium); } } } break; } } else if (medium.canView && medium.files != null && medium.files.drm != null) { if (medium.files != null && medium.files.drm != null) { if (!singlePostCollection.SinglePosts.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, singlePost.id, medium.files.drm.manifest.dash, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, postPreviewIds.Contains(medium.id) ? true : false, false, null); singlePostCollection.SinglePosts.Add(medium.id, $"{medium.files.drm.manifest.dash},{medium.files.drm.signature.dash.CloudFrontPolicy},{medium.files.drm.signature.dash.CloudFrontSignature},{medium.files.drm.signature.dash.CloudFrontKeyPairId},{medium.id},{singlePost.id}"); singlePostCollection.SinglePostMedia.Add(medium); } } } else if (medium.files.preview != null && medium.files!.full == null) { if (!singlePostCollection.SinglePosts.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, singlePost.id, medium.files.preview.url, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, postPreviewIds.Contains(medium.id) ? true : false, false, null); singlePostCollection.SinglePosts.Add(medium.id, medium.files.preview.url); singlePostCollection.SinglePostMedia.Add(medium); } } } } } return singlePostCollection; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetStreams(string endpoint, string folder, List paid_post_ids, StatusContext ctx) { Log.Debug($"Calling GetStreams - {endpoint}"); try { Streams streams = new(); StreamsCollection streamsCollection = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "skip_users", "all" } }; DownloadDateSelection downloadDateSelection = DownloadDateSelection.before; if (configService.CurrentConfig.DownloadOnlySpecificDates && configService.CurrentConfig.CustomDate.HasValue) { downloadDateSelection = configService.CurrentConfig.DownloadDateSelection; } UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, configService.CurrentConfig.CustomDate); string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); streams = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (streams != null && streams.hasMore) { UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, streams.tailMarker); while (true) { Streams newstreams = new(); string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); newstreams = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); streams.list.AddRange(newstreams.list); ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (!newstreams.hasMore) { break; } UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, newstreams.tailMarker); } } foreach (Streams.List stream in streams.list) { List streamPreviewIds = new(); if (stream.preview != null && stream.preview.Count > 0) { for (int i = 0; i < stream.preview.Count; i++) { if (stream.preview[i] is long previewId) { if (!streamPreviewIds.Contains(previewId)) { streamPreviewIds.Add(previewId); } } } } await dbService.AddPost(folder, stream.id, stream.text != null ? stream.text : string.Empty, stream.price != null ? stream.price : "0", stream.price != null && stream.isOpened ? true : false, stream.isArchived, stream.postedAt); streamsCollection.StreamObjects.Add(stream); if (stream.media != null && stream.media.Count > 0) { foreach (Streams.Medium medium in stream.media) { if (medium.type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (medium.canView && medium.files?.drm == null) { bool has = paid_post_ids.Any(cus => cus.Equals(medium.id)); if (!has && medium.canView && medium.files != null && medium.files.full != null && !string.IsNullOrEmpty(medium.files.full.url)) { if (!streamsCollection.Streams.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, stream.id, medium.files.full.url, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, streamPreviewIds.Contains(medium.id) ? true : false, false, null); streamsCollection.Streams.Add(medium.id, medium.files.full.url); streamsCollection.StreamMedia.Add(medium); } } } else if (medium.canView && medium.files != null && medium.files.drm != null) { bool has = paid_post_ids.Any(cus => cus.Equals(medium.id)); if (!has && medium.files != null && medium.files.drm != null) { if (!streamsCollection.Streams.ContainsKey(medium.id)) { await dbService.AddMedia(folder, medium.id, stream.id, medium.files.drm.manifest.dash, null, null, null, "Posts", medium.type == "photo" ? "Images" : medium.type == "video" || medium.type == "gif" ? "Videos" : medium.type == "audio" ? "Audios" : null, streamPreviewIds.Contains(medium.id) ? true : false, false, null); streamsCollection.Streams.Add(medium.id, $"{medium.files.drm.manifest.dash},{medium.files.drm.signature.dash.CloudFrontPolicy},{medium.files.drm.signature.dash.CloudFrontSignature},{medium.files.drm.signature.dash.CloudFrontKeyPairId},{medium.id},{stream.id}"); streamsCollection.StreamMedia.Add(medium); } } } } } } return streamsCollection; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetArchived(string endpoint, string folder, StatusContext ctx) { Log.Debug($"Calling GetArchived - {endpoint}"); try { ArchivedEntities.Archived archived = new(); ArchivedEntities.ArchivedCollection archivedCollection = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, { "skip_users", "all" }, { "format", "infinite" }, { "label", "archived" }, { "counters", "1" } }; DownloadDateSelection downloadDateSelection = DownloadDateSelection.before; if (configService.CurrentConfig.DownloadOnlySpecificDates && configService.CurrentConfig.CustomDate.HasValue) { downloadDateSelection = configService.CurrentConfig.DownloadDateSelection; } UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, configService.CurrentConfig.CustomDate); string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); ArchivedDtos.ArchivedDto archivedDto = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); archived = ArchivedMapper.FromDto(archivedDto); ctx.Status($"[red]Getting Archived Posts\n[/] [red]Found {archived.List.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (archived != null && archived.HasMore) { UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, archived.TailMarker); while (true) { ArchivedEntities.Archived newarchived = new(); string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); ArchivedDtos.ArchivedDto newarchivedDto = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); newarchived = ArchivedMapper.FromDto(newarchivedDto); archived.List.AddRange(newarchived.List); ctx.Status($"[red]Getting Archived Posts\n[/] [red]Found {archived.List.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (!newarchived.HasMore) { break; } UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, newarchived.TailMarker); } } foreach (ArchivedEntities.ListItem archive in archived.List) { List previewids = new(); if (archive.Preview != null) { for (int i = 0; i < archive.Preview.Count; i++) { if (archive.Preview[i] is long previewId) { if (!previewids.Contains(previewId)) { previewids.Add(previewId); } } } } await dbService.AddPost(folder, archive.Id, archive.Text != null ? archive.Text : string.Empty, archive.Price != null ? archive.Price : "0", archive.Price != null && archive.IsOpened ? true : false, archive.IsArchived, archive.PostedAt); archivedCollection.ArchivedPostObjects.Add(archive); if (archive.Media != null && archive.Media.Count > 0) { foreach (ArchivedEntities.Medium medium in archive.Media) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (!archivedCollection.ArchivedPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, archive.Id, medium.Files.Full.Url, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id) ? true : false, false, null); archivedCollection.ArchivedPosts.Add(medium.Id, medium.Files.Full.Url); archivedCollection.ArchivedPostMedia.Add(medium); } } else if (medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (!archivedCollection.ArchivedPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, archive.Id, medium.Files.Drm.Manifest.Dash, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id) ? true : false, false, null); archivedCollection.ArchivedPosts.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{archive.Id}"); archivedCollection.ArchivedPostMedia.Add(medium); } } } } } return archivedCollection; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetMessages(string endpoint, string folder, StatusContext ctx) { Log.Debug($"Calling GetMessages - {endpoint}"); try { MessageEntities.Messages messages = new(); MessageEntities.MessageCollection messageCollection = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "desc" }, { "skip_users", "all" } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); MessageDtos.MessagesDto? messagesDto = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); messages = MessagesMapper.FromDto(messagesDto); ctx.Status($"[red]Getting Messages\n[/] [red]Found {messages.List.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (messages.HasMore) { getParams["id"] = messages.List[^1].Id.ToString(); while (true) { MessageEntities.Messages newMessages = new(); string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); MessageDtos.MessagesDto? newMessagesDto = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); newMessages = MessagesMapper.FromDto(newMessagesDto); messages.List.AddRange(newMessages.List); ctx.Status($"[red]Getting Messages\n[/] [red]Found {messages.List.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (!newMessages.HasMore) { break; } getParams["id"] = newMessages.List[newMessages.List.Count - 1].Id.ToString(); } } foreach (MessageEntities.ListItem list in messages.List) { if (configService.CurrentConfig.SkipAds) { if (!string.IsNullOrEmpty(list.Text) && (list.Text.Contains("#ad") || list.Text.Contains("/trial/"))) { continue; } } List messagePreviewIds = new(); if (list.Previews != null && list.Previews.Count > 0) { for (int i = 0; i < list.Previews.Count; i++) { if (list.Previews[i] is long previewId) { if (!messagePreviewIds.Contains(previewId)) { messagePreviewIds.Add(previewId); } } } } if (!configService.CurrentConfig.IgnoreOwnMessages || list.FromUser?.Id != Convert.ToInt32(authService.CurrentAuth.USER_ID)) { await dbService.AddMessage(folder, list.Id, list.Text ?? string.Empty, list.Price ?? "0", list.CanPurchaseReason == "opened" ? true : list.CanPurchaseReason != "opened" ? false : (bool?)null ?? false, false, list.CreatedAt.HasValue ? list.CreatedAt.Value : DateTime.Now, list.FromUser?.Id ?? int.MinValue); messageCollection.MessageObjects.Add(list); if (list.CanPurchaseReason != "opened" && list.Media != null && list.Media.Count > 0) { foreach (MessageEntities.Medium medium in list.Media) { if (medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!messageCollection.Messages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, list.Id, medium.Files.Full.Url, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, messagePreviewIds.Contains(medium.Id), false, null); messageCollection.Messages.Add(medium.Id, medium.Files.Full.Url); messageCollection.MessageMedia.Add(medium); } } else if (medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!messageCollection.Messages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, list.Id, medium.Files.Drm.Manifest.Dash, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, messagePreviewIds.Contains(medium.Id), false, null); messageCollection.Messages.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{list.Id}"); messageCollection.MessageMedia.Add(medium); } } } } else if (messagePreviewIds.Count > 0) { foreach (MessageEntities.Medium medium in list.Media) { if (medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url) && messagePreviewIds.Contains(medium.Id)) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!messageCollection.Messages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, list.Id, medium.Files.Full.Url, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, messagePreviewIds.Contains(medium.Id), false, null); messageCollection.Messages.Add(medium.Id, medium.Files.Full.Url); messageCollection.MessageMedia.Add(medium); } } else if (medium.CanView && medium.Files != null && medium.Files.Drm != null && messagePreviewIds.Contains(medium.Id)) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!messageCollection.Messages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, list.Id, medium.Files.Drm.Manifest.Dash, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, messagePreviewIds.Contains(medium.Id), false, null); messageCollection.Messages.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{list.Id}"); messageCollection.MessageMedia.Add(medium); } } } } } } return messageCollection; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetPaidMessage(string endpoint, string folder) { Log.Debug($"Calling GetPaidMessage - {endpoint}"); try { MessageEntities.SingleMessage message = new(); SinglePaidMessageCollection singlePaidMessageCollection = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "desc" } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); MessageDtos.SingleMessageDto? messageDto = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); message = MessagesMapper.FromDto(messageDto); if (!configService.CurrentConfig.IgnoreOwnMessages || message.FromUser?.Id != Convert.ToInt32(authService.CurrentAuth.USER_ID)) { await dbService.AddMessage(folder, message.Id, message.Text ?? string.Empty, message.Price != null ? message.Price.ToString() : "0", true, false, message.CreatedAt.HasValue ? message.CreatedAt.Value : DateTime.Now, message.FromUser?.Id ?? int.MinValue); singlePaidMessageCollection.SingleMessageObjects.Add(message); List messagePreviewIds = new(); if (message.Previews != null && message.Previews.Count > 0) { for (int i = 0; i < message.Previews.Count; i++) { if (message.Previews[i] is long previewId) { if (!messagePreviewIds.Contains(previewId)) { messagePreviewIds.Add(previewId); } } } } if (message.Media != null && message.Media.Count > 0) { foreach (MessageEntities.Medium medium in message.Media) { if (!messagePreviewIds.Contains(medium.Id) && medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!singlePaidMessageCollection.SingleMessages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, message.Id, medium.Files.Full.Url, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, messagePreviewIds.Contains(medium.Id), false, null); singlePaidMessageCollection.SingleMessages.Add(medium.Id, medium.Files.Full.Url); singlePaidMessageCollection.SingleMessageMedia.Add(medium); } } else if (messagePreviewIds.Contains(medium.Id) && medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!singlePaidMessageCollection.PreviewSingleMessages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, message.Id, medium.Files.Full.Url, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, messagePreviewIds.Contains(medium.Id), false, null); singlePaidMessageCollection.PreviewSingleMessages.Add(medium.Id, medium.Files.Full.Url); singlePaidMessageCollection.PreviewSingleMessageMedia.Add(medium); } } else if (!messagePreviewIds.Contains(medium.Id) && medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!singlePaidMessageCollection.SingleMessages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, message.Id, medium.Files.Drm.Manifest.Dash, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, messagePreviewIds.Contains(medium.Id), false, null); singlePaidMessageCollection.SingleMessages.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{message.Id}"); singlePaidMessageCollection.SingleMessageMedia.Add(medium); } } else if (messagePreviewIds.Contains(medium.Id) && medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!singlePaidMessageCollection.PreviewSingleMessages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, message.Id, medium.Files.Drm.Manifest.Dash, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, messagePreviewIds.Contains(medium.Id), false, null); singlePaidMessageCollection.PreviewSingleMessages.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{message.Id}"); singlePaidMessageCollection.PreviewSingleMessageMedia.Add(medium); } } } } } return singlePaidMessageCollection; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetPaidMessages(string endpoint, string folder, string username, StatusContext ctx) { Log.Debug($"Calling GetPaidMessages - {username}"); try { Purchased paidMessages = new(); PaidMessageCollection paidMessageCollection = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "author", username }, { "skip_users", "all" } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); paidMessages = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (paidMessages != null && paidMessages.hasMore) { getParams["offset"] = paidMessages.list.Count.ToString(); while (true) { string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); Purchased newpaidMessages = new(); Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams); HttpClient loopclient = GetHttpClient(); HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}"); foreach (KeyValuePair keyValuePair in loopheaders) { looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); } using (HttpResponseMessage loopresponse = await loopclient.SendAsync(looprequest)) { loopresponse.EnsureSuccessStatusCode(); string loopbody = await loopresponse.Content.ReadAsStringAsync(); newpaidMessages = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); } paidMessages.list.AddRange(newpaidMessages.list); ctx.Status($"[red]Getting Paid Messages\n[/] [red]Found {paidMessages.list.Count}[/]"); ctx.Spinner(Spinner.Known.Dots); ctx.SpinnerStyle(Style.Parse("blue")); if (!newpaidMessages.hasMore) { break; } getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); } } if (paidMessages.list != null && paidMessages.list.Count > 0) { foreach (Purchased.List purchase in paidMessages.list.Where(p => p.responseType == "message") .OrderByDescending(p => p.postedAt ?? p.createdAt)) { if (!configService.CurrentConfig.IgnoreOwnMessages || purchase.fromUser.id != Convert.ToInt32(authService.CurrentAuth.USER_ID)) { if (purchase.postedAt != null) { await dbService.AddMessage(folder, purchase.id, purchase.text != null ? purchase.text : string.Empty, purchase.price != null ? purchase.price : "0", true, false, purchase.postedAt.Value, purchase.fromUser.id); } else { await dbService.AddMessage(folder, purchase.id, purchase.text != null ? purchase.text : string.Empty, purchase.price != null ? purchase.price : "0", true, false, purchase.createdAt.Value, purchase.fromUser.id); } paidMessageCollection.PaidMessageObjects.Add(purchase); if (purchase.media != null && purchase.media.Count > 0) { List previewids = new(); if (purchase.previews != null) { for (int i = 0; i < purchase.previews.Count; i++) { if (purchase.previews[i] is long previewId) { if (!previewids.Contains(previewId)) { previewids.Add(previewId); } } } } else if (purchase.preview != null) { for (int i = 0; i < purchase.preview.Count; i++) { if (purchase.preview[i] is long previewId) { if (!previewids.Contains(previewId)) { previewids.Add(previewId); } } } } foreach (MessageDtos.MediumDto medium in purchase.media) { if (previewids.Count > 0) { MessageEntities.Medium mappedMedium = MessagesMapper.MapMedium(medium); bool has = previewids.Any(cus => cus.Equals(medium.Id)); if (!has && medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, purchase.id, medium.Files.Full.Url, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); paidMessageCollection.PaidMessages.Add(medium.Id, medium.Files.Full.Url); paidMessageCollection.PaidMessageMedia.Add(mappedMedium); } } else if (!has && medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, purchase.id, medium.Files.Drm.Manifest.Dash, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); paidMessageCollection.PaidMessages.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{purchase.id}"); paidMessageCollection.PaidMessageMedia.Add(mappedMedium); } } } else { MessageEntities.Medium mappedMedium = MessagesMapper.MapMedium(medium); if (medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, purchase.id, medium.Files.Full.Url, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); paidMessageCollection.PaidMessages.Add(medium.Id, medium.Files.Full.Url); paidMessageCollection.PaidMessageMedia.Add(mappedMedium); } } else if (medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!paidMessageCollection.PaidMessages.ContainsKey(medium.Id)) { await dbService.AddMedia(folder, medium.Id, purchase.id, medium.Files.Drm.Manifest.Dash, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); paidMessageCollection.PaidMessages.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{purchase.id}"); paidMessageCollection.PaidMessageMedia.Add(mappedMedium); } } } } } } } } return paidMessageCollection; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task> GetPurchasedTabUsers(string endpoint, Dictionary users) { Log.Debug($"Calling GetPurchasedTabUsers - {endpoint}"); try { Dictionary purchasedTabUsers = new(); Purchased purchased = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "skip_users", "all" } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); purchased = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); if (purchased != null && purchased.hasMore) { getParams["offset"] = purchased.list.Count.ToString(); while (true) { string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); Purchased newPurchased = new(); Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams); HttpClient loopclient = GetHttpClient(); HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}"); foreach (KeyValuePair keyValuePair in loopheaders) { looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); } using (HttpResponseMessage loopresponse = await loopclient.SendAsync(looprequest)) { loopresponse.EnsureSuccessStatusCode(); string loopbody = await loopresponse.Content.ReadAsStringAsync(); newPurchased = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); } purchased.list.AddRange(newPurchased.list); if (!newPurchased.hasMore) { break; } getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); } } if (purchased.list != null && purchased.list.Count > 0) { foreach (Purchased.List purchase in purchased.list.OrderByDescending(p => p.postedAt ?? p.createdAt)) { if (purchase.fromUser != null) { if (users.Values.Contains(purchase.fromUser.id)) { if (!string.IsNullOrEmpty(users.FirstOrDefault(x => x.Value == purchase.fromUser.id).Key)) { if (!purchasedTabUsers.ContainsKey(users .FirstOrDefault(x => x.Value == purchase.fromUser.id).Key)) { purchasedTabUsers.Add( users.FirstOrDefault(x => x.Value == purchase.fromUser.id).Key, purchase.fromUser.id); } } else { if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}")) { purchasedTabUsers.Add($"Deleted User - {purchase.fromUser.id}", purchase.fromUser.id); } } } else { JObject user = await GetUserInfoById($"/users/list?x[]={purchase.fromUser.id}"); if (user is null) { if (!configService.CurrentConfig.BypassContentForCreatorsWhoNoLongerExist) { if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}")) { purchasedTabUsers.Add($"Deleted User - {purchase.fromUser.id}", purchase.fromUser.id); } } Log.Debug("Content creator not longer exists - {0}", purchase.fromUser.id); } else if (!string.IsNullOrEmpty(user[purchase.fromUser.id.ToString()]["username"] .ToString())) { if (!purchasedTabUsers.ContainsKey(user[purchase.fromUser.id.ToString()]["username"] .ToString())) { purchasedTabUsers.Add(user[purchase.fromUser.id.ToString()]["username"].ToString(), purchase.fromUser.id); } } else { if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.fromUser.id}")) { purchasedTabUsers.Add($"Deleted User - {purchase.fromUser.id}", purchase.fromUser.id); } } } } else if (purchase.author != null) { if (users.Values.Contains(purchase.author.id)) { if (!string.IsNullOrEmpty(users.FirstOrDefault(x => x.Value == purchase.author.id).Key)) { if (!purchasedTabUsers.ContainsKey(users .FirstOrDefault(x => x.Value == purchase.author.id).Key) && users.ContainsKey(users.FirstOrDefault(x => x.Value == purchase.author.id).Key)) { purchasedTabUsers.Add(users.FirstOrDefault(x => x.Value == purchase.author.id).Key, purchase.author.id); } } else { if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}")) { purchasedTabUsers.Add($"Deleted User - {purchase.author.id}", purchase.author.id); } } } else { JObject user = await GetUserInfoById($"/users/list?x[]={purchase.author.id}"); if (user is null) { if (!configService.CurrentConfig.BypassContentForCreatorsWhoNoLongerExist) { if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}")) { purchasedTabUsers.Add($"Deleted User - {purchase.author.id}", purchase.author.id); } } Log.Debug("Content creator not longer exists - {0}", purchase.author.id); } else if (!string.IsNullOrEmpty(user[purchase.author.id.ToString()]["username"].ToString())) { if (!purchasedTabUsers.ContainsKey(user[purchase.author.id.ToString()]["username"] .ToString()) && users.ContainsKey(user[purchase.author.id.ToString()]["username"].ToString())) { purchasedTabUsers.Add(user[purchase.author.id.ToString()]["username"].ToString(), purchase.author.id); } } else { if (!purchasedTabUsers.ContainsKey($"Deleted User - {purchase.author.id}")) { purchasedTabUsers.Add($"Deleted User - {purchase.author.id}", purchase.author.id); } } } } } } return purchasedTabUsers; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task> GetPurchasedTab(string endpoint, string folder, Dictionary users) { Log.Debug($"Calling GetPurchasedTab - {endpoint}"); try { Dictionary> userPurchases = new(); List purchasedTabCollections = new(); Purchased purchased = new(); int post_limit = 50; Dictionary getParams = new() { { "limit", post_limit.ToString() }, { "order", "publish_date_desc" }, { "format", "infinite" }, { "skip_users", "all" } }; string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient()); purchased = JsonConvert.DeserializeObject(body, m_JsonSerializerSettings); if (purchased != null && purchased.hasMore) { getParams["offset"] = purchased.list.Count.ToString(); while (true) { string loopqueryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); Purchased newPurchased = new(); Dictionary loopheaders = GetDynamicHeaders("/api2/v2" + endpoint, loopqueryParams); HttpClient loopclient = GetHttpClient(); HttpRequestMessage looprequest = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{loopqueryParams}"); foreach (KeyValuePair keyValuePair in loopheaders) { looprequest.Headers.Add(keyValuePair.Key, keyValuePair.Value); } using (HttpResponseMessage loopresponse = await loopclient.SendAsync(looprequest)) { loopresponse.EnsureSuccessStatusCode(); string loopbody = await loopresponse.Content.ReadAsStringAsync(); newPurchased = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); } purchased.list.AddRange(newPurchased.list); if (!newPurchased.hasMore) { break; } getParams["offset"] = Convert.ToString(Convert.ToInt32(getParams["offset"]) + post_limit); } } if (purchased.list != null && purchased.list.Count > 0) { foreach (Purchased.List purchase in purchased.list.OrderByDescending(p => p.postedAt ?? p.createdAt)) { if (purchase.fromUser != null) { if (!userPurchases.ContainsKey(purchase.fromUser.id)) { userPurchases.Add(purchase.fromUser.id, new List()); } userPurchases[purchase.fromUser.id].Add(purchase); } else if (purchase.author != null) { if (!userPurchases.ContainsKey(purchase.author.id)) { userPurchases.Add(purchase.author.id, new List()); } userPurchases[purchase.author.id].Add(purchase); } } } foreach (KeyValuePair> user in userPurchases) { PurchasedTabCollection purchasedTabCollection = new(); JObject userObject = await GetUserInfoById($"/users/list?x[]={user.Key}"); purchasedTabCollection.UserId = user.Key; purchasedTabCollection.Username = userObject is not null && !string.IsNullOrEmpty(userObject[user.Key.ToString()]["username"].ToString()) ? userObject[user.Key.ToString()]["username"].ToString() : $"Deleted User - {user.Key}"; string path = Path.Combine(folder, purchasedTabCollection.Username); if (Path.Exists(path)) { foreach (Purchased.List purchase in user.Value) { if (purchase.media == null) { Log.Warning( "PurchasedTab purchase media null, setting empty list | userId={UserId} username={Username} purchaseId={PurchaseId} responseType={ResponseType} createdAt={CreatedAt} postedAt={PostedAt}", user.Key, purchasedTabCollection.Username, purchase.id, purchase.responseType, purchase.createdAt, purchase.postedAt); purchase.media = new List(); } switch (purchase.responseType) { case "post": List previewids = new(); if (purchase.previews != null) { for (int i = 0; i < purchase.previews.Count; i++) { if (purchase.previews[i] is long previewId) { if (!previewids.Contains(previewId)) { previewids.Add(previewId); } } } } else if (purchase.preview != null) { for (int i = 0; i < purchase.preview.Count; i++) { if (purchase.preview[i] is long previewId) { if (!previewids.Contains(previewId)) { previewids.Add(previewId); } } } } await dbService.AddPost(path, purchase.id, purchase.text != null ? purchase.text : string.Empty, purchase.price != null ? purchase.price : "0", purchase.price != null && purchase.isOpened ? true : false, purchase.isArchived.HasValue ? purchase.isArchived.Value : false, purchase.createdAt != null ? purchase.createdAt.Value : purchase.postedAt.Value); purchasedTabCollection.PaidPosts.PaidPostObjects.Add(purchase); foreach (MessageDtos.MediumDto medium in purchase.media) { MessageEntities.Medium mappedMedium = MessagesMapper.MapMedium(medium); if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (previewids.Count > 0) { bool has = previewids.Any(cus => cus.Equals(medium.Id)); if (!has && medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(path, medium.Id, purchase.id, medium.Files.Full.Url, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id, medium.Files.Full.Url); purchasedTabCollection.PaidPosts.PaidPostMedia.Add(mappedMedium); } } else if (!has && medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(path, medium.Id, purchase.id, medium.Files.Drm.Manifest.Dash, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{purchase.id}"); purchasedTabCollection.PaidPosts.PaidPostMedia.Add(mappedMedium); } } } else { if (medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(path, medium.Id, purchase.id, medium.Files.Full.Url, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id, medium.Files.Full.Url); purchasedTabCollection.PaidPosts.PaidPostMedia.Add(mappedMedium); } } else if (medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (!purchasedTabCollection.PaidPosts.PaidPosts.ContainsKey(medium.Id)) { await dbService.AddMedia(path, medium.Id, purchase.id, medium.Files.Drm.Manifest.Dash, null, null, null, "Posts", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, previewids.Contains(medium.Id), false, null); purchasedTabCollection.PaidPosts.PaidPosts.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{purchase.id}"); purchasedTabCollection.PaidPosts.PaidPostMedia.Add(mappedMedium); } } } } break; case "message": if (purchase.postedAt != null) { await dbService.AddMessage(path, purchase.id, purchase.text != null ? purchase.text : string.Empty, purchase.price != null ? purchase.price : "0", true, false, purchase.postedAt.Value, purchase.fromUser.id); } else { await dbService.AddMessage(path, purchase.id, purchase.text != null ? purchase.text : string.Empty, purchase.price != null ? purchase.price : "0", true, false, purchase.createdAt.Value, purchase.fromUser.id); } purchasedTabCollection.PaidMessages.PaidMessageObjects.Add(purchase); if (purchase.media != null && purchase.media.Count > 0) { List paidMessagePreviewids = new(); if (purchase.previews != null) { for (int i = 0; i < purchase.previews.Count; i++) { if (purchase.previews[i] is long previewId) { if (!paidMessagePreviewids.Contains(previewId)) { paidMessagePreviewids.Add(previewId); } } } } else if (purchase.preview != null) { for (int i = 0; i < purchase.preview.Count; i++) { if (purchase.preview[i] is long previewId) { if (!paidMessagePreviewids.Contains(previewId)) { paidMessagePreviewids.Add(previewId); } } } } foreach (MessageDtos.MediumDto medium in purchase.media) { if (paidMessagePreviewids.Count > 0) { MessageEntities.Medium mappedMedium = MessagesMapper.MapMedium(medium); bool has = paidMessagePreviewids.Any(cus => cus.Equals(medium.Id)); if (!has && medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey( medium.Id)) { await dbService.AddMedia(path, medium.Id, purchase.id, medium.Files.Full.Url, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, paidMessagePreviewids.Contains(medium.Id), false, null); purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id, medium.Files.Full.Url); purchasedTabCollection.PaidMessages.PaidMessageMedia.Add( mappedMedium); } } else if (!has && medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey( medium.Id)) { await dbService.AddMedia(path, medium.Id, purchase.id, medium.Files.Drm.Manifest.Dash, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, paidMessagePreviewids.Contains(medium.Id), false, null); purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{purchase.id}"); purchasedTabCollection.PaidMessages.PaidMessageMedia.Add( mappedMedium); } } } else { MessageEntities.Medium mappedMedium = MessagesMapper.MapMedium(medium); if (medium.CanView && medium.Files != null && medium.Files.Full != null && !string.IsNullOrEmpty(medium.Files.Full.Url)) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey( medium.Id)) { await dbService.AddMedia(path, medium.Id, purchase.id, medium.Files.Full.Url, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, paidMessagePreviewids.Contains(medium.Id), false, null); purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id, medium.Files.Full.Url); purchasedTabCollection.PaidMessages.PaidMessageMedia.Add( mappedMedium); } } else if (medium.CanView && medium.Files != null && medium.Files.Drm != null) { if (medium.Type == "photo" && !configService.CurrentConfig.DownloadImages) { continue; } if (medium.Type == "video" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "gif" && !configService.CurrentConfig.DownloadVideos) { continue; } if (medium.Type == "audio" && !configService.CurrentConfig.DownloadAudios) { continue; } if (!purchasedTabCollection.PaidMessages.PaidMessages.ContainsKey( medium.Id)) { await dbService.AddMedia(path, medium.Id, purchase.id, medium.Files.Drm.Manifest.Dash, null, null, null, "Messages", medium.Type == "photo" ? "Images" : medium.Type == "video" || medium.Type == "gif" ? "Videos" : medium.Type == "audio" ? "Audios" : null, paidMessagePreviewids.Contains(medium.Id), false, null); purchasedTabCollection.PaidMessages.PaidMessages.Add(medium.Id, $"{medium.Files.Drm.Manifest.Dash},{medium.Files.Drm.Signature.Dash.CloudFrontPolicy},{medium.Files.Drm.Signature.Dash.CloudFrontSignature},{medium.Files.Drm.Signature.Dash.CloudFrontKeyPairId},{medium.Id},{purchase.id}"); purchasedTabCollection.PaidMessages.PaidMessageMedia.Add( mappedMedium); } } } } } break; } } purchasedTabCollections.Add(purchasedTabCollection); } } return purchasedTabCollections; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp) { try { string pssh = null; HttpClient client = new(); HttpRequestMessage request = new(HttpMethod.Get, mpdUrl); request.Headers.Add("user-agent", authService.CurrentAuth.USER_AGENT); request.Headers.Add("Accept", "*/*"); request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {authService.CurrentAuth.COOKIE};"); using (HttpResponseMessage response = await client.SendAsync(request)) { response.EnsureSuccessStatusCode(); string body = await response.Content.ReadAsStringAsync(); XNamespace ns = "urn:mpeg:dash:schema:mpd:2011"; XNamespace cenc = "urn:mpeg:cenc:2013"; XDocument xmlDoc = XDocument.Parse(body); IEnumerable psshElements = xmlDoc.Descendants(cenc + "pssh"); pssh = psshElements.ElementAt(1).Value; } return pssh; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetDRMMPDLastModified(string mpdUrl, string policy, string signature, string kvp) { Log.Debug("Calling GetDRMMPDLastModified"); Log.Debug($"mpdUrl: {mpdUrl}"); Log.Debug($"policy: {policy}"); Log.Debug($"signature: {signature}"); Log.Debug($"kvp: {kvp}"); try { DateTime lastmodified; HttpClient client = new(); HttpRequestMessage request = new(HttpMethod.Get, mpdUrl); request.Headers.Add("user-agent", authService.CurrentAuth.USER_AGENT); request.Headers.Add("Accept", "*/*"); request.Headers.Add("Cookie", $"CloudFront-Policy={policy}; CloudFront-Signature={signature}; CloudFront-Key-Pair-Id={kvp}; {authService.CurrentAuth.COOKIE};"); using (HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead)) { response.EnsureSuccessStatusCode(); lastmodified = response.Content.Headers.LastModified?.LocalDateTime ?? DateTime.Now; Log.Debug($"Last modified: {lastmodified}"); } return lastmodified; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return DateTime.Now; } public async Task GetDecryptionKeyCDRMProject(Dictionary drmHeaders, string licenceURL, string pssh) { Log.Debug("Calling GetDecryptionKey"); int attempt = 0; try { string dcValue = string.Empty; HttpClient client = new(); CDRMProjectRequest cdrmProjectRequest = new() { PSSH = pssh, LicenseURL = licenceURL, Headers = JsonConvert.SerializeObject(drmHeaders), Cookies = "", Data = "" }; string json = JsonConvert.SerializeObject(cdrmProjectRequest); Log.Debug($"Posting to CDRM Project: {json}"); while (attempt < MaxAttempts) { attempt++; HttpRequestMessage request = new(HttpMethod.Post, "https://cdrm-project.com/api/decrypt") { Content = new StringContent(json, Encoding.UTF8, "application/json") }; using HttpResponseMessage response = await client.SendAsync(request); Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}"); response.EnsureSuccessStatusCode(); string body = await response.Content.ReadAsStringAsync(); JsonDocument doc = JsonDocument.Parse(body); if (doc.RootElement.TryGetProperty("status", out JsonElement status)) { if (status.ToString().Trim().Equals("success", StringComparison.OrdinalIgnoreCase)) { dcValue = doc.RootElement.GetProperty("message").GetString().Trim(); return dcValue; } Log.Debug($"CDRM response status not successful. Retrying... Attempt {attempt} of {MaxAttempts}"); if (attempt < MaxAttempts) { await Task.Delay(DelayBetweenAttempts); } } else { Log.Debug($"Status not in CDRM response. Retrying... Attempt {attempt} of {MaxAttempts}"); if (attempt < MaxAttempts) { await Task.Delay(DelayBetweenAttempts); } } } throw new Exception("Maximum retry attempts reached. Unable to get a valid decryption key."); } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetDecryptionKeyOFDL(Dictionary drmHeaders, string licenceURL, string pssh) { Log.Debug("Calling GetDecryptionOFDL"); try { HttpClient client = new(); int attempt = 0; OFDLRequest ofdlRequest = new() { PSSH = pssh, LicenseURL = licenceURL, Headers = JsonConvert.SerializeObject(drmHeaders) }; string json = JsonConvert.SerializeObject(ofdlRequest); Log.Debug($"Posting to ofdl.tools: {json}"); while (attempt < MaxAttempts) { attempt++; HttpRequestMessage request = new(HttpMethod.Post, "https://ofdl.tools/WV") { Content = new StringContent(json, Encoding.UTF8, "application/json") }; using HttpResponseMessage response = await client.SendAsync(request); if (!response.IsSuccessStatusCode) { continue; } string body = await response.Content.ReadAsStringAsync(); if (!body.TrimStart().StartsWith('{')) { return body; } Log.Debug($"Received JSON object instead of string. Retrying... Attempt {attempt} of {MaxAttempts}"); await Task.Delay(DelayBetweenAttempts); } } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public async Task GetDecryptionKeyCDM(Dictionary drmHeaders, string licenceURL, string pssh) { Log.Debug("Calling GetDecryptionKeyCDM"); try { byte[] resp1 = await PostData(licenceURL, drmHeaders, new byte[] { 0x08, 0x04 }); string certDataB64 = Convert.ToBase64String(resp1); CDMApi cdm = new(); byte[] challenge = cdm.GetChallenge(pssh, certDataB64); byte[] resp2 = await PostData(licenceURL, drmHeaders, challenge); string licenseB64 = Convert.ToBase64String(resp2); Log.Debug($"resp1: {resp1}"); Log.Debug($"certDataB64: {certDataB64}"); Log.Debug($"challenge: {challenge}"); Log.Debug($"resp2: {resp2}"); Log.Debug($"licenseB64: {licenseB64}"); cdm.ProvideLicense(licenseB64); List keys = cdm.GetKeys(); if (keys.Count > 0) { Log.Debug($"GetDecryptionKeyCDM Key: {keys[0]}"); return keys[0].ToString(); } } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } private async Task BuildHeaderAndExecuteRequests(Dictionary getParams, string endpoint, HttpClient client) { Log.Debug("Calling BuildHeaderAndExecuteRequests"); HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint); using HttpResponseMessage response = await client.SendAsync(request); response.EnsureSuccessStatusCode(); string body = await response.Content.ReadAsStringAsync(); Log.Debug(body); return body; } private async Task BuildHttpRequestMessage(Dictionary getParams, string endpoint) { Log.Debug("Calling BuildHttpRequestMessage"); string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}")); Dictionary headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams); HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{queryParams}"); Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}"); foreach (KeyValuePair keyValuePair in headers) { request.Headers.Add(keyValuePair.Key, keyValuePair.Value); } return request; } private static double ConvertToUnixTimestampWithMicrosecondPrecision(DateTime date) { DateTime origin = new(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); TimeSpan diff = date.ToUniversalTime() - origin; return diff.TotalSeconds; // This gives the number of seconds. If you need milliseconds, use diff.TotalMilliseconds } public static bool IsStringOnlyDigits(string input) => input.All(char.IsDigit); private HttpClient GetHttpClient() { HttpClient client = new(); if (configService.CurrentConfig?.Timeout != null && configService.CurrentConfig.Timeout > 0) { client.Timeout = TimeSpan.FromSeconds(configService.CurrentConfig.Timeout.Value); } return client; } /// /// this one is used during initialization only /// if the config option is not available then no modificatiotns will be done on the getParams /// /// /// /// private static void UpdateGetParamsForDateSelection(DownloadDateSelection downloadDateSelection, ref Dictionary getParams, DateTime? dt) { //if (config.DownloadOnlySpecificDates && dt.HasValue) //{ if (dt.HasValue) { UpdateGetParamsForDateSelection( downloadDateSelection, ref getParams, ConvertToUnixTimestampWithMicrosecondPrecision(dt.Value) .ToString("0.000000", CultureInfo.InvariantCulture) ); } //} } private static void UpdateGetParamsForDateSelection(DownloadDateSelection downloadDateSelection, ref Dictionary getParams, string unixTimeStampInMicrosec) { switch (downloadDateSelection) { case DownloadDateSelection.before: getParams["beforePublishTime"] = unixTimeStampInMicrosec; break; case DownloadDateSelection.after: getParams["order"] = "publish_date_asc"; getParams["afterPublishTime"] = unixTimeStampInMicrosec; break; } } public async Task?> GetAllSubscriptions(Dictionary getParams, string endpoint, bool includeRestricted) { try { Dictionary users = new(); Subscriptions subscriptions = new(); Log.Debug("Calling GetAllSubscrptions"); string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); subscriptions = JsonConvert.DeserializeObject(body); if (subscriptions != null && subscriptions.hasMore) { getParams["offset"] = subscriptions.list.Count.ToString(); while (true) { Subscriptions newSubscriptions = new(); string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient()); if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]")) { newSubscriptions = JsonConvert.DeserializeObject(loopbody, m_JsonSerializerSettings); } else { break; } subscriptions.list.AddRange(newSubscriptions.list); if (!newSubscriptions.hasMore) { break; } getParams["offset"] = subscriptions.list.Count.ToString(); } } foreach (Subscriptions.List subscription in subscriptions.list) { if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted)) && !users.ContainsKey(subscription.username)) { users.Add(subscription.username, subscription.id); } } return users; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } public static string? GetDynamicRules() { Log.Debug("Calling GetDynamicRules"); try { HttpClient client = new(); HttpRequestMessage request = new(HttpMethod.Get, "https://git.ofdl.tools/sim0n00ps/dynamic-rules/raw/branch/main/rules.json"); using HttpResponseMessage response = client.Send(request); if (!response.IsSuccessStatusCode) { Log.Debug("GetDynamicRules did not return a Success Status Code"); return null; } string body = response.Content.ReadAsStringAsync().Result; Log.Debug("GetDynamicRules Response: "); Log.Debug(body); return body; } catch (Exception ex) { Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); if (ex.InnerException != null) { Console.WriteLine("\nInner Exception:"); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace); } } return null; } }