using Newtonsoft.Json.Linq; using OF_DL.Enumerations; using OF_DL.Models.Config; using OF_DL.Models.Downloads; using Serilog; using PostEntities = OF_DL.Models.Entities.Posts; using PurchasedEntities = OF_DL.Models.Entities.Purchased; using UserEntities = OF_DL.Models.Entities.Users; namespace OF_DL.Services; public class DownloadOrchestrationService( IApiService apiService, IConfigService configService, IDownloadService downloadService, IDbService dbService) : IDownloadOrchestrationService { /// /// Gets the list of paid post media IDs to avoid duplicates. /// public List PaidPostIds { get; } = []; /// /// Retrieves the available users and lists based on the current configuration. /// /// A result containing users, lists, and any errors. public async Task GetAvailableUsersAsync() { UserListResult result = new(); Config config = configService.CurrentConfig; Dictionary? activeSubs = await apiService.GetActiveSubscriptions("/subscriptions/subscribes", config.IncludeRestrictedSubscriptions); if (activeSubs != null) { Log.Debug("Subscriptions: "); foreach (KeyValuePair activeSub in activeSubs) { if (!result.Users.ContainsKey(activeSub.Key)) { result.Users.Add(activeSub.Key, activeSub.Value); Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}"); } } } else { Log.Error("Couldn't get active subscriptions. Received null response."); } if (config.IncludeExpiredSubscriptions) { Log.Debug("Inactive Subscriptions: "); Dictionary? expiredSubs = await apiService.GetExpiredSubscriptions("/subscriptions/subscribes", config.IncludeRestrictedSubscriptions); if (expiredSubs != null) { foreach (KeyValuePair expiredSub in expiredSubs.Where(expiredSub => !result.Users.ContainsKey(expiredSub.Key))) { result.Users.Add(expiredSub.Key, expiredSub.Value); Log.Debug("Name: {ExpiredSubKey} ID: {ExpiredSubValue}", expiredSub.Key, expiredSub.Value); } } else { Log.Error("Couldn't get expired subscriptions. Received null response."); } } result.Lists = await apiService.GetLists("/lists") ?? new Dictionary(); // Remove users from the list if they are in the ignored list if (!string.IsNullOrEmpty(config.IgnoredUsersListName)) { if (!result.Lists.TryGetValue(config.IgnoredUsersListName, out long ignoredUsersListId)) { result.IgnoredListError = $"Ignored users list '{config.IgnoredUsersListName}' not found"; Log.Error(result.IgnoredListError); } else { List ignoredUsernames = await apiService.GetListUsers($"/lists/{ignoredUsersListId}/users") ?? []; result.Users = result.Users.Where(x => !ignoredUsernames.Contains(x.Key)) .ToDictionary(x => x.Key, x => x.Value); } } await dbService.CreateUsersDb(result.Users); return result; } /// /// Retrieves the user's lists only. /// /// A dictionary of list names to list IDs. public async Task> GetUserListsAsync() => await apiService.GetLists("/lists") ?? new Dictionary(); /// /// Resolves the users that belong to a specific list. /// /// The list name. /// All available users. /// Known lists keyed by name. /// The users that belong to the list. public async Task> GetUsersForListAsync( string listName, Dictionary allUsers, Dictionary lists) { long listId = lists[listName]; List listUsernames = await apiService.GetListUsers($"/lists/{listId}/users") ?? []; return allUsers.Where(x => listUsernames.Contains(x.Key)) .ToDictionary(x => x.Key, x => x.Value); } /// /// Resolves the download path for a username based on configuration. /// /// The creator username. /// The resolved download path. public string ResolveDownloadPath(string username) => !string.IsNullOrEmpty(configService.CurrentConfig.DownloadPath) ? Path.Combine(configService.CurrentConfig.DownloadPath, username) : $"__user_data__/sites/OnlyFans/{username}"; /// /// Ensures the user folder and metadata database exist. /// /// The creator username. /// The creator user ID. /// The creator folder path. public async Task PrepareUserFolderAsync(string username, long userId, string path) { await dbService.CheckUsername(new KeyValuePair(username, userId), path); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); Log.Debug($"Created folder for {username}"); } await dbService.CreateDb(path); } /// /// Downloads all configured content types for a creator. /// /// The creator username. /// The creator user ID. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// Download event handler. /// Counts of downloaded items per content type. public async Task DownloadCreatorContentAsync( string username, long userId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, IDownloadEventHandler eventHandler) { Config config = configService.CurrentConfig; CreatorDownloadResult counts = new(); eventHandler.OnUserStarting(username); Log.Debug($"Scraping Data for {username}"); eventHandler.CancellationToken.ThrowIfCancellationRequested(); await PrepareUserFolderAsync(username, userId, path); if (config.DownloadAvatarHeaderPhoto) { eventHandler.CancellationToken.ThrowIfCancellationRequested(); UserEntities.User? userInfo = await apiService.GetUserInfo($"/users/{username}", eventHandler.CancellationToken); if (userInfo != null) { await downloadService.DownloadAvatarHeader(userInfo.Avatar, userInfo.Header, path, username); } } if (config.DownloadPaidPosts) { counts.PaidPostCount = await DownloadContentTypeAsync("Paid Posts", async statusReporter => await apiService.GetPaidPosts("/posts/paid/post", path, username, PaidPostIds, statusReporter), posts => posts.PaidPosts.Count, posts => posts.PaidPostObjects.Count, posts => posts.PaidPosts.Values.ToList(), async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter), eventHandler); } if (config.DownloadPosts) { eventHandler.OnMessage( "Getting Posts (this may take a long time, depending on the number of Posts the creator has)"); Log.Debug("Calling DownloadFreePosts - {Username}", username); counts.PostCount = await DownloadContentTypeAsync("Posts", async statusReporter => await apiService.GetPosts($"/users/{userId}/posts", path, PaidPostIds, statusReporter), posts => posts.Posts.Count, posts => posts.PostObjects.Count, posts => posts.Posts.Values.ToList(), async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter), eventHandler); } if (config.DownloadArchived) { counts.ArchivedCount = await DownloadContentTypeAsync("Archived Posts", async statusReporter => await apiService.GetArchived($"/users/{userId}/posts", path, statusReporter), archived => archived.ArchivedPosts.Count, archived => archived.ArchivedPostObjects.Count, archived => archived.ArchivedPosts.Values.ToList(), async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter), eventHandler); } if (config.DownloadStreams) { counts.StreamsCount = await DownloadContentTypeAsync("Streams", async statusReporter => await apiService.GetStreams($"/users/{userId}/posts/streams", path, PaidPostIds, statusReporter), streams => streams.Streams.Count, streams => streams.StreamObjects.Count, streams => streams.Streams.Values.ToList(), async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter), eventHandler); } if (config.DownloadStories) { eventHandler.CancellationToken.ThrowIfCancellationRequested(); eventHandler.OnMessage("Getting Stories"); Dictionary? tempStories = await apiService.GetMedia(MediaType.Stories, $"/users/{userId}/stories", null, path, eventHandler.CancellationToken); if (tempStories is { Count: > 0 }) { eventHandler.OnContentFound("Stories", tempStories.Count, tempStories.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize(tempStories.Values.ToList()) : tempStories.Count; DownloadResult result = await eventHandler.WithProgressAsync( $"Downloading {tempStories.Count} Stories", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadStories(username, userId, path, PaidPostIds.ToHashSet(), reporter)); eventHandler.OnDownloadComplete("Stories", result); counts.StoriesCount = result.TotalCount; } else { eventHandler.OnNoContentFound("Stories"); } } if (config.DownloadHighlights) { eventHandler.CancellationToken.ThrowIfCancellationRequested(); eventHandler.OnMessage("Getting Highlights"); Dictionary? tempHighlights = await apiService.GetMedia(MediaType.Highlights, $"/users/{userId}/stories/highlights", null, path, eventHandler.CancellationToken); if (tempHighlights is { Count: > 0 }) { eventHandler.OnContentFound("Highlights", tempHighlights.Count, tempHighlights.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize(tempHighlights.Values.ToList()) : tempHighlights.Count; DownloadResult result = await eventHandler.WithProgressAsync( $"Downloading {tempHighlights.Count} Highlights", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadHighlights(username, userId, path, PaidPostIds.ToHashSet(), reporter)); eventHandler.OnDownloadComplete("Highlights", result); counts.HighlightsCount = result.TotalCount; } else { eventHandler.OnNoContentFound("Highlights"); } } if (config.DownloadMessages) { counts.MessagesCount = await DownloadContentTypeAsync("Messages", async statusReporter => await apiService.GetMessages($"/chats/{userId}/messages", path, statusReporter), messages => messages.Messages.Count, messages => messages.MessageObjects.Count, messages => messages.Messages.Values.ToList(), async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users, clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter), eventHandler); } if (config.DownloadPaidMessages) { counts.PaidMessagesCount = await DownloadContentTypeAsync("Paid Messages", async statusReporter => await apiService.GetPaidMessages("/posts/paid/chat", path, username, statusReporter), paidMessages => paidMessages.PaidMessages.Count, paidMessages => paidMessages.PaidMessageObjects.Count, paidMessages => paidMessages.PaidMessages.Values.ToList(), async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter), eventHandler); } eventHandler.OnUserComplete(username, counts); return counts; } /// /// Downloads a single post by ID for a creator. /// /// The creator username. /// The post ID. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// Download event handler. public async Task DownloadSinglePostAsync( string username, long postId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, IDownloadEventHandler eventHandler) { Log.Debug($"Calling DownloadSinglePost - {postId}"); eventHandler.OnMessage("Getting Post"); PostEntities.SinglePostCollection post = await apiService.GetPost($"/posts/{postId}", path); if (post.SinglePosts.Count == 0) { eventHandler.OnMessage("Couldn't find post"); Log.Debug("Couldn't find post"); return; } Config config = configService.CurrentConfig; long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize(post.SinglePosts.Values.ToList()) : post.SinglePosts.Count; DownloadResult result = await eventHandler.WithProgressAsync( "Downloading Post", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadSinglePost(username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, post, reporter)); if (result.NewDownloads > 0) { eventHandler.OnMessage($"Post {postId} downloaded"); Log.Debug($"Post {postId} downloaded"); } else { eventHandler.OnMessage($"Post {postId} already downloaded"); Log.Debug($"Post {postId} already downloaded"); } } /// /// Downloads content from the Purchased tab across creators. /// /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// Download event handler. public async Task DownloadPurchasedTabAsync( Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, IDownloadEventHandler eventHandler) { Config config = configService.CurrentConfig; eventHandler.OnMessage("Fetching purchased tab users..."); eventHandler.CancellationToken.ThrowIfCancellationRequested(); Dictionary purchasedTabUsers = await apiService.GetPurchasedTabUsers("/posts/paid/all", users, eventHandler.CancellationToken); eventHandler.OnMessage("Checking folders for users in Purchased Tab"); eventHandler.CancellationToken.ThrowIfCancellationRequested(); foreach (KeyValuePair user in purchasedTabUsers) { eventHandler.CancellationToken.ThrowIfCancellationRequested(); string path = ResolveDownloadPath(user.Key); Log.Debug($"Download path: {path}"); await dbService.CheckUsername(user, path); if (!Directory.Exists(path)) { Directory.CreateDirectory(path); Log.Debug($"Created folder for {user.Key}"); } await apiService.GetUserInfo($"/users/{user.Key}", eventHandler.CancellationToken); await dbService.CreateDb(path); } string basePath = !string.IsNullOrEmpty(config.DownloadPath) ? config.DownloadPath : "__user_data__/sites/OnlyFans/"; Log.Debug($"Download path: {basePath}"); eventHandler.OnMessage("Fetching purchased tab content..."); eventHandler.CancellationToken.ThrowIfCancellationRequested(); List purchasedTabCollections = await apiService.GetPurchasedTab("/posts/paid/all", basePath, users, eventHandler.CancellationToken); foreach (PurchasedEntities.PurchasedTabCollection purchasedTabCollection in purchasedTabCollections) { eventHandler.CancellationToken.ThrowIfCancellationRequested(); eventHandler.OnUserStarting(purchasedTabCollection.Username); string path = ResolveDownloadPath(purchasedTabCollection.Username); Log.Debug($"Download path: {path}"); int paidPostCount = 0; int paidMessagesCount = 0; // Download paid posts if (purchasedTabCollection.PaidPosts.PaidPosts.Count > 0) { eventHandler.OnContentFound("Paid Posts", purchasedTabCollection.PaidPosts.PaidPosts.Count, purchasedTabCollection.PaidPosts.PaidPostObjects.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize( purchasedTabCollection.PaidPosts.PaidPosts.Values.ToList()) : purchasedTabCollection.PaidPosts.PaidPosts.Count; DownloadResult postResult = await eventHandler.WithProgressAsync( $"Downloading {purchasedTabCollection.PaidPosts.PaidPosts.Count} Media from {purchasedTabCollection.PaidPosts.PaidPostObjects.Count} Paid Posts", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadPaidPostsPurchasedTab( purchasedTabCollection.Username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, purchasedTabCollection.PaidPosts, reporter)); eventHandler.OnDownloadComplete("Paid Posts", postResult); paidPostCount = postResult.TotalCount; } else { eventHandler.OnNoContentFound("Paid Posts"); } // Download paid messages if (purchasedTabCollection.PaidMessages.PaidMessages.Count > 0) { eventHandler.OnContentFound("Paid Messages", purchasedTabCollection.PaidMessages.PaidMessages.Count, purchasedTabCollection.PaidMessages.PaidMessageObjects.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize( purchasedTabCollection.PaidMessages.PaidMessages.Values.ToList()) : purchasedTabCollection.PaidMessages.PaidMessages.Count; DownloadResult msgResult = await eventHandler.WithProgressAsync( $"Downloading {purchasedTabCollection.PaidMessages.PaidMessages.Count} Media from {purchasedTabCollection.PaidMessages.PaidMessageObjects.Count} Paid Messages", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadPaidMessagesPurchasedTab( purchasedTabCollection.Username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, purchasedTabCollection.PaidMessages, reporter)); eventHandler.OnDownloadComplete("Paid Messages", msgResult); paidMessagesCount = msgResult.TotalCount; } else { eventHandler.OnNoContentFound("Paid Messages"); } eventHandler.OnPurchasedTabUserComplete(purchasedTabCollection.Username, paidPostCount, paidMessagesCount); } } /// /// Downloads a single paid message by ID. /// /// The creator username. /// The message ID. /// The creator folder path. /// Known users map. /// Whether the CDM client ID blob is missing. /// Whether the CDM private key is missing. /// Download event handler. public async Task DownloadSinglePaidMessageAsync( string username, long messageId, string path, Dictionary users, bool clientIdBlobMissing, bool devicePrivateKeyMissing, IDownloadEventHandler eventHandler) { Log.Debug($"Calling DownloadSinglePaidMessage - {username}"); eventHandler.OnMessage("Getting Paid Message"); PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection = await apiService.GetPaidMessage($"/messages/{messageId}", path); if (singlePaidMessageCollection.SingleMessages.Count == 0 && singlePaidMessageCollection.PreviewSingleMessages.Count == 0) { eventHandler.OnNoContentFound("Paid Messages"); return; } Config config = configService.CurrentConfig; int messageCount = singlePaidMessageCollection.SingleMessageObjects.Count; string messageLabel = messageCount == 1 ? "Paid Message" : "Paid Messages"; int previewCount = singlePaidMessageCollection.PreviewSingleMessages.Count; int paidCount = singlePaidMessageCollection.SingleMessages.Count; int totalCount = previewCount + paidCount; // Handle mixed paid + preview message media. if (previewCount > 0 && paidCount > 0) { eventHandler.OnContentFound("Paid Messages", totalCount, singlePaidMessageCollection.SingleMessageObjects.Count); List allMessageUrls = []; allMessageUrls.AddRange(singlePaidMessageCollection.PreviewSingleMessages.Values); allMessageUrls.AddRange(singlePaidMessageCollection.SingleMessages.Values); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize(allMessageUrls) : totalCount; DownloadResult result = await eventHandler.WithProgressAsync( $"Downloading {totalCount} Media from {messageCount} {messageLabel} ({paidCount} Paid + {previewCount} Preview)", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadSinglePaidMessage(username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, singlePaidMessageCollection, reporter)); eventHandler.OnDownloadComplete("Paid Messages", result); } // Handle preview-only message media. else if (previewCount > 0) { eventHandler.OnContentFound("Preview Paid Messages", previewCount, singlePaidMessageCollection.SingleMessageObjects.Count); long previewSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize( singlePaidMessageCollection.PreviewSingleMessages.Values.ToList()) : previewCount; DownloadResult previewResult = await eventHandler.WithProgressAsync( $"Downloading {previewCount} Preview Media from {messageCount} {messageLabel}", previewSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadSinglePaidMessage(username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, singlePaidMessageCollection, reporter)); eventHandler.OnDownloadComplete("Paid Messages", previewResult); } else if (paidCount > 0) { // Only actual paid messages, no preview eventHandler.OnContentFound("Paid Messages", paidCount, singlePaidMessageCollection.SingleMessageObjects.Count); long totalSize = config.ShowScrapeSize ? await downloadService.CalculateTotalFileSize( singlePaidMessageCollection.SingleMessages.Values.ToList()) : paidCount; DownloadResult result = await eventHandler.WithProgressAsync( $"Downloading {paidCount} Paid Media from {messageCount} {messageLabel}", totalSize, config.ShowScrapeSize, async reporter => await downloadService.DownloadSinglePaidMessage(username, path, users, clientIdBlobMissing, devicePrivateKeyMissing, singlePaidMessageCollection, reporter)); eventHandler.OnDownloadComplete("Paid Messages", result); } else { eventHandler.OnNoContentFound("Paid Messages"); } } /// /// Resolves a username for a user ID, including deleted users. /// /// The user ID. /// The resolved username or a deleted user placeholder. public async Task ResolveUsernameAsync(long userId) { JObject? user = await apiService.GetUserInfoById($"/users/list?x[]={userId}"); if (user == null) { return $"Deleted User - {userId}"; } string? username = user[userId.ToString()]?["username"]?.ToString(); return !string.IsNullOrEmpty(username) ? username : $"Deleted User - {userId}"; } /// /// Generic helper for the common pattern: fetch with status -> check count -> download with progress. /// private async Task DownloadContentTypeAsync( string contentType, Func> fetchData, Func getMediaCount, Func getObjectCount, Func?> getUrls, Func> downloadData, IDownloadEventHandler eventHandler) { T data = await eventHandler.WithStatusAsync($"Getting {contentType}", async statusReporter => await fetchData(statusReporter)); int mediaCount = getMediaCount(data); if (mediaCount <= 0) { eventHandler.OnNoContentFound(contentType); Log.Debug($"Found 0 {contentType}"); return 0; } int objectCount = getObjectCount(data); eventHandler.OnContentFound(contentType, mediaCount, objectCount); Log.Debug("Found {MediaCount} Media from {ObjectCount} {ContentType}", mediaCount, objectCount, contentType); Config config = configService.CurrentConfig; List? urls = getUrls(data); long totalSize = config.ShowScrapeSize && urls != null ? await downloadService.CalculateTotalFileSize(urls) : mediaCount; DownloadResult result = await eventHandler.WithProgressAsync( $"Downloading {mediaCount} Media from {objectCount} {contentType}", totalSize, config.ShowScrapeSize, async reporter => await downloadData(data, reporter)); eventHandler.OnDownloadComplete(contentType, result); Log.Debug( $"{contentType} Media Already Downloaded: {result.ExistingDownloads} New {contentType} Media Downloaded: {result.NewDownloads}"); return result.TotalCount; } }