using Newtonsoft.Json.Linq; using OF_DL.Enumerations; using OF_DL.Models.Config; using OF_DL.Models.Downloads; using Serilog; using PostEntities = OF_DL.Models.Entities.Posts; using PurchasedEntities = OF_DL.Models.Entities.Purchased; using UserEntities = OF_DL.Models.Entities.Users; namespace OF_DL.Services; public class DownloadOrchestrationService( IAPIService apiService, IConfigService configService, IDownloadService downloadService, IDBService dbService) : IDownloadOrchestrationService { public List PaidPostIds { get; } = new(); public async Task GetAvailableUsersAsync() { UserListResult result = new(); Config config = configService.CurrentConfig; Dictionary? activeSubs = await apiService.GetActiveSubscriptions("/subscriptions/subscribes", config.IncludeRestrictedSubscriptions); if (activeSubs != null) { Log.Debug("Subscriptions: "); foreach (KeyValuePair activeSub in activeSubs) { if (!result.Users.ContainsKey(activeSub.Key)) { result.Users.Add(activeSub.Key, activeSub.Value); Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}"); } } } else { Log.Error("Couldn't get active subscriptions. Received null response."); } if (config.IncludeExpiredSubscriptions) { Log.Debug("Inactive Subscriptions: "); Dictionary? expiredSubs = await apiService.GetExpiredSubscriptions("/subscriptions/subscribes", config.IncludeRestrictedSubscriptions); if (expiredSubs != null) { foreach (KeyValuePair expiredSub in expiredSubs.Where(expiredSub => !result.Users.ContainsKey(expiredSub.Key))) { result.Users.Add(expiredSub.Key, expiredSub.Value); Log.Debug("Name: {ExpiredSubKey} ID: {ExpiredSubValue}", expiredSub.Key, expiredSub.Value); } } else { Log.Error("Couldn't get expired subscriptions. Received null response."); } } result.Lists = await apiService.GetLists("/lists") ?? new Dictionary(); // Remove users from the list if they are in the ignored list if (!string.IsNullOrEmpty(config.IgnoredUsersListName)) { if (!result.Lists.TryGetValue(config.IgnoredUsersListName, out long ignoredUsersListId)) { result.IgnoredListError = $"Ignored users list '{config.IgnoredUsersListName}' not found"; Log.Error(result.IgnoredListError); } else { List ignoredUsernames = await apiService.GetListUsers($"/lists/{ignoredUsersListId}/users") ?? []; result.Users = result.Users.Where(x => !ignoredUsernames.Contains(x.Key)) .ToDictionary(x => x.Key, x => x.Value); } } await dbService.CreateUsersDB(result.Users); return result; } public async Task> GetUsersForListAsync( string listName, Dictionary allUsers, Dictionary lists) { long listId = lists[listName]; List listUsernames = await apiService.GetListUsers($"/lists/{listId}/users") ?? []; return allUsers.Where(x => listUsernames.Contains(x.Key)).Distinct() .ToDictionary(x => x.Key, x => x.Value); } public string ResolveDownloadPath(string username) => !string.IsNullOrEmpty(configService.CurrentConfig.DownloadPath) ? Path.Combine(configService.CurrentConfig.DownloadPath, username) : $"__user_data__/sites/OnlyFans/{username}"; public async Task PrepareUserFolderAsync(string username, long userId, string path) { await dbService.CheckUsername(new KeyValuePair(username, userId), path); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); Log.Debug($"Created folder for {username}"); } await dbService.CreateDB(path); } public async Task DownloadCreatorContentAsync( string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, IDownloadEventHandler eventHandler) { Config config = configService.CurrentConfig; CreatorDownloadResult counts = new(); eventHandler.OnUserStarting(username); Log.Debug($"Scraping Data for {username}"); await PrepareUserFolderAsync(username, userId, path); if (config.DownloadAvatarHeaderPhoto) { UserEntities.User? userInfo = await apiService.GetUserInfo($"/users/{username}"); if (userInfo != null) { await downloadService.DownloadAvatarHeader(userInfo.Avatar, userInfo.Header, path, username); } } if (config.DownloadPaidPosts) { counts.PaidPostCount = await DownloadContentTypeAsync("Paid Posts", async statusReporter => await apiService.GetPaidPosts("/posts/paid/post", path, username, PaidPostIds, statusReporter), posts => posts.PaidPosts.Count, posts => posts.PaidPostObjects.Count, posts => posts.PaidPosts.Values.ToList(), async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter), eventHandler); } if (config.DownloadPosts) { eventHandler.OnMessage( "Getting Posts (this may take a long time, depending on the number of Posts the creator has)"); Log.Debug($"Calling DownloadFreePosts - {username}"); counts.PostCount = await DownloadContentTypeAsync("Posts", async statusReporter => await apiService.GetPosts($"/users/{userId}/posts", path, PaidPostIds, statusReporter), posts => posts.Posts.Count, posts => posts.PostObjects.Count, posts => posts.Posts.Values.ToList(), async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter), eventHandler); } if (config.DownloadArchived) { counts.ArchivedCount = await DownloadContentTypeAsync("Archived Posts", async statusReporter => await apiService.GetArchived($"/users/{userId}/posts", path, statusReporter), archived => archived.ArchivedPosts.Count, archived => archived.ArchivedPostObjects.Count, archived => archived.ArchivedPosts.Values.ToList(), async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter), eventHandler); } if (config.DownloadStreams) { counts.StreamsCount = await DownloadContentTypeAsync("Streams", async statusReporter => await apiService.GetStreams($"/users/{userId}/posts/streams", path, PaidPostIds, statusReporter), streams => streams.Streams.Count, streams => streams.StreamObjects.Count, streams => streams.Streams.Values.ToList(), async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter), eventHandler); } if (config.DownloadStories) { eventHandler.OnMessage("Getting Stories"); Dictionary? tempStories = await apiService.GetMedia(MediaType.Stories, $"/users/{userId}/stories", null, path, PaidPostIds); if (tempStories != null && tempStories.Count > 0) { eventHandler.OnContentFound("Stories", tempStories.Count, tempStories.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize(tempStories.Values.ToList()) : tempStories.Count; DownloadResult result = await eventHandler.WithProgressAsync( $"Downloading {tempStories.Count} Stories", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadStories(username, userId, path, PaidPostIds.ToHashSet(), reporter)); eventHandler.OnDownloadComplete("Stories", result); counts.StoriesCount = result.TotalCount; } else { eventHandler.OnNoContentFound("Stories"); } } if (config.DownloadHighlights) { eventHandler.OnMessage("Getting Highlights"); Dictionary? tempHighlights = await apiService.GetMedia(MediaType.Highlights, $"/users/{userId}/stories/highlights", null, path, PaidPostIds); if (tempHighlights != null && tempHighlights.Count > 0) { eventHandler.OnContentFound("Highlights", tempHighlights.Count, tempHighlights.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize(tempHighlights.Values.ToList()) : tempHighlights.Count; DownloadResult result = await eventHandler.WithProgressAsync( $"Downloading {tempHighlights.Count} Highlights", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadHighlights(username, userId, path, PaidPostIds.ToHashSet(), reporter)); eventHandler.OnDownloadComplete("Highlights", result); counts.HighlightsCount = result.TotalCount; } else { eventHandler.OnNoContentFound("Highlights"); } } if (config.DownloadMessages) { counts.MessagesCount = await DownloadContentTypeAsync("Messages", async statusReporter => await apiService.GetMessages($"/chats/{userId}/messages", path, statusReporter), messages => messages.Messages.Count, messages => messages.MessageObjects.Count, messages => messages.Messages.Values.ToList(), async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter), eventHandler); } if (config.DownloadPaidMessages) { counts.PaidMessagesCount = await DownloadContentTypeAsync("Paid Messages", async statusReporter => await apiService.GetPaidMessages("/posts/paid/chat", path, username, statusReporter), paidMessages => paidMessages.PaidMessages.Count, paidMessages => paidMessages.PaidMessageObjects.Count, paidMessages => paidMessages.PaidMessages.Values.ToList(), async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter), eventHandler); } eventHandler.OnUserComplete(username, counts); return counts; } public async Task DownloadSinglePostAsync( string username, long postId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, IDownloadEventHandler eventHandler) { Log.Debug($"Calling DownloadSinglePost - {postId}"); eventHandler.OnMessage("Getting Post"); PostEntities.SinglePostCollection post = await apiService.GetPost($"/posts/{postId}", path); if (post.SinglePosts.Count == 0) { eventHandler.OnMessage("Couldn't find post"); Log.Debug("Couldn't find post"); return; } Config config = configService.CurrentConfig; long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize(post.SinglePosts.Values.ToList()) : post.SinglePosts.Count; DownloadResult result = await eventHandler.WithProgressAsync( "Downloading Post", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadSinglePost(username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, post, reporter)); if (result.NewDownloads > 0) { eventHandler.OnMessage($"Post {postId} downloaded"); Log.Debug($"Post {postId} downloaded"); } else { eventHandler.OnMessage($"Post {postId} already downloaded"); Log.Debug($"Post {postId} already downloaded"); } } public async Task DownloadPurchasedTabAsync( Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, IDownloadEventHandler eventHandler) { Config config = configService.CurrentConfig; Dictionary purchasedTabUsers = await apiService.GetPurchasedTabUsers("/posts/paid/all", users); eventHandler.OnMessage("Checking folders for Users in Purchased Tab"); foreach (KeyValuePair user in purchasedTabUsers) { string path = ResolveDownloadPath(user.Key); Log.Debug($"Download path: {path}"); await dbService.CheckUsername(user, path); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); Log.Debug($"Created folder for {user.Key}"); } await apiService.GetUserInfo($"/users/{user.Key}"); await dbService.CreateDB(path); } string basePath = !string.IsNullOrEmpty(config.DownloadPath) ? config.DownloadPath : "__user_data__/sites/OnlyFans/"; Log.Debug($"Download path: {basePath}"); List purchasedTabCollections = await apiService.GetPurchasedTab("/posts/paid/all", basePath, users); foreach (PurchasedEntities.PurchasedTabCollection purchasedTabCollection in purchasedTabCollections) { eventHandler.OnUserStarting(purchasedTabCollection.Username); string path = ResolveDownloadPath(purchasedTabCollection.Username); Log.Debug($"Download path: {path}"); int paidPostCount = 0; int paidMessagesCount = 0; // Download paid posts if (purchasedTabCollection.PaidPosts.PaidPosts.Count > 0) { eventHandler.OnContentFound("Paid Posts", purchasedTabCollection.PaidPosts.PaidPosts.Count, purchasedTabCollection.PaidPosts.PaidPostObjects.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize( purchasedTabCollection.PaidPosts.PaidPosts.Values.ToList()) : purchasedTabCollection.PaidPosts.PaidPosts.Count; DownloadResult postResult = await eventHandler.WithProgressAsync( $"Downloading {purchasedTabCollection.PaidPosts.PaidPosts.Count} Paid Posts", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadPaidPostsPurchasedTab( purchasedTabCollection.Username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, purchasedTabCollection.PaidPosts, reporter)); eventHandler.OnDownloadComplete("Paid Posts", postResult); paidPostCount = postResult.TotalCount; } else { eventHandler.OnNoContentFound("Paid Posts"); } // Download paid messages if (purchasedTabCollection.PaidMessages.PaidMessages.Count > 0) { eventHandler.OnContentFound("Paid Messages", purchasedTabCollection.PaidMessages.PaidMessages.Count, purchasedTabCollection.PaidMessages.PaidMessageObjects.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize( purchasedTabCollection.PaidMessages.PaidMessages.Values.ToList()) : purchasedTabCollection.PaidMessages.PaidMessages.Count; DownloadResult msgResult = await eventHandler.WithProgressAsync( $"Downloading {purchasedTabCollection.PaidMessages.PaidMessages.Count} Paid Messages", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadPaidMessagesPurchasedTab( purchasedTabCollection.Username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, purchasedTabCollection.PaidMessages, reporter)); eventHandler.OnDownloadComplete("Paid Messages", msgResult); paidMessagesCount = msgResult.TotalCount; } else { eventHandler.OnNoContentFound("Paid Messages"); } eventHandler.OnPurchasedTabUserComplete(purchasedTabCollection.Username, paidPostCount, paidMessagesCount); } } public async Task DownloadSinglePaidMessageAsync( string username, long messageId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, IDownloadEventHandler eventHandler) { Log.Debug($"Calling DownloadSinglePaidMessage - {username}"); eventHandler.OnMessage("Getting Paid Message"); PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection = await apiService.GetPaidMessage($"/messages/{messageId}", path); if (singlePaidMessageCollection.SingleMessages.Count == 0) { eventHandler.OnNoContentFound("Paid Messages"); return; } Config config = configService.CurrentConfig; // Handle preview messages if (singlePaidMessageCollection.PreviewSingleMessages.Count > 0) { eventHandler.OnContentFound("Preview Paid Messages", singlePaidMessageCollection.PreviewSingleMessages.Count, singlePaidMessageCollection.SingleMessageObjects.Count); long previewSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize( singlePaidMessageCollection.PreviewSingleMessages.Values.ToList()) : singlePaidMessageCollection.PreviewSingleMessages.Count; DownloadResult previewResult = await eventHandler.WithProgressAsync( $"Downloading {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Paid Messages", previewSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadSinglePaidMessage(username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, singlePaidMessageCollection, reporter)); eventHandler.OnDownloadComplete("Paid Messages", previewResult); } else if (singlePaidMessageCollection.SingleMessages.Count > 0) { // Only actual paid messages, no preview eventHandler.OnContentFound("Paid Messages", singlePaidMessageCollection.SingleMessages.Count, singlePaidMessageCollection.SingleMessageObjects.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize( singlePaidMessageCollection.SingleMessages.Values.ToList()) : singlePaidMessageCollection.SingleMessages.Count; DownloadResult result = await eventHandler.WithProgressAsync( $"Downloading {singlePaidMessageCollection.SingleMessages.Count} Paid Messages", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadSinglePaidMessage(username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, singlePaidMessageCollection, reporter)); eventHandler.OnDownloadComplete("Paid Messages", result); } else { eventHandler.OnNoContentFound("Paid Messages"); } } public async Task ResolveUsernameAsync(long userId) { JObject? user = await apiService.GetUserInfoById($"/users/list?x[]={userId}"); if (user == null) { return $"Deleted User - {userId}"; } string? username = user[userId.ToString()]?["username"]?.ToString(); return !string.IsNullOrEmpty(username) ? username : $"Deleted User - {userId}"; } /// /// Generic helper for the common pattern: fetch with status -> check count -> download with progress. /// private async Task DownloadContentTypeAsync( string contentType, Func> fetchData, Func getMediaCount, Func getObjectCount, Func?> getUrls, Func> downloadData, IDownloadEventHandler eventHandler) { T data = await eventHandler.WithStatusAsync($"Getting {contentType}", async statusReporter => await fetchData(statusReporter)); int mediaCount = getMediaCount(data); if (mediaCount <= 0) { eventHandler.OnNoContentFound(contentType); Log.Debug($"Found 0 {contentType}"); return 0; } int objectCount = getObjectCount(data); eventHandler.OnContentFound(contentType, mediaCount, objectCount); Log.Debug($"Found {mediaCount} Media from {objectCount} {contentType}"); Config config = configService.CurrentConfig; List? urls = getUrls(data); long totalSize = config.ShowScrapeSize && urls != null ? await downloadService.CalculateTotalFileSize(urls) : mediaCount; DownloadResult result = await eventHandler.WithProgressAsync( $"Downloading {mediaCount} {contentType}", totalSize, config.ShowScrapeSize, async reporter => await downloadData(data, reporter)); eventHandler.OnDownloadComplete(contentType, result); Log.Debug( $"{contentType} Already Downloaded: {result.ExistingDownloads} New {contentType} Downloaded: {result.NewDownloads}"); return result.TotalCount; } }