OF-DL/OF DL.Core/Services/DownloadOrchestrationService.cs

using Newtonsoft.Json.Linq;
using OF_DL.Enumerations;
using OF_DL.Models.Config;
using OF_DL.Models.Downloads;
using Serilog;
using PostEntities = OF_DL.Models.Entities.Posts;
using PurchasedEntities = OF_DL.Models.Entities.Purchased;
using UserEntities = OF_DL.Models.Entities.Users;

namespace OF_DL.Services;

public class DownloadOrchestrationService(
    IAPIService apiService,
    IConfigService configService,
    IDownloadService downloadService,
    IDBService dbService) : IDownloadOrchestrationService
{
    public List<long> PaidPostIds { get; } = new();

    public async Task<UserListResult> GetAvailableUsersAsync()
    {
        UserListResult result = new();
        Config config = configService.CurrentConfig;

        Dictionary<string, long>? activeSubs =
            await apiService.GetActiveSubscriptions("/subscriptions/subscribes",
                config.IncludeRestrictedSubscriptions);

        if (activeSubs != null)
        {
            Log.Debug("Subscriptions: ");
            foreach (KeyValuePair<string, long> activeSub in activeSubs)
            {
                if (!result.Users.ContainsKey(activeSub.Key))
                {
                    result.Users.Add(activeSub.Key, activeSub.Value);
                    Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}");
                }
            }
        }
        else
        {
            Log.Error("Couldn't get active subscriptions. Received null response.");
        }


        if (config.IncludeExpiredSubscriptions)
        {
            Log.Debug("Inactive Subscriptions: ");
            Dictionary<string, long>? expiredSubs =
                await apiService.GetExpiredSubscriptions("/subscriptions/subscribes",
                    config.IncludeRestrictedSubscriptions);

            if (expiredSubs != null)
            {
                foreach (KeyValuePair<string, long> expiredSub in expiredSubs.Where(expiredSub =>
                             !result.Users.ContainsKey(expiredSub.Key)))
                {
                    result.Users.Add(expiredSub.Key, expiredSub.Value);
                    Log.Debug("Name: {ExpiredSubKey} ID: {ExpiredSubValue}", expiredSub.Key, expiredSub.Value);
                }
            }
            else
            {
                Log.Error("Couldn't get expired subscriptions. Received null response.");
            }
        }

        result.Lists = await apiService.GetLists("/lists") ?? new Dictionary<string, long>();

        // Remove users from the list if they are in the ignored list
        if (!string.IsNullOrEmpty(config.IgnoredUsersListName))
        {
            if (!result.Lists.TryGetValue(config.IgnoredUsersListName, out long ignoredUsersListId))
            {
                result.IgnoredListError = $"Ignored users list '{config.IgnoredUsersListName}' not found";
                Log.Error(result.IgnoredListError);
            }
            else
            {
                List<string> ignoredUsernames =
                    await apiService.GetListUsers($"/lists/{ignoredUsersListId}/users") ?? [];
                result.Users = result.Users.Where(x => !ignoredUsernames.Contains(x.Key))
                    .ToDictionary(x => x.Key, x => x.Value);
            }
        }

        await dbService.CreateUsersDB(result.Users);
        return result;
    }

    public async Task<Dictionary<string, long>> GetUsersForListAsync(
        string listName, Dictionary<string, long> allUsers, Dictionary<string, long> lists)
    {
        long listId = lists[listName];
        List<string> listUsernames = await apiService.GetListUsers($"/lists/{listId}/users") ?? [];
        return allUsers.Where(x => listUsernames.Contains(x.Key))
            .ToDictionary(x => x.Key, x => x.Value);
    }

    public string ResolveDownloadPath(string username) =>
        !string.IsNullOrEmpty(configService.CurrentConfig.DownloadPath)
            ? Path.Combine(configService.CurrentConfig.DownloadPath, username)
            : $"__user_data__/sites/OnlyFans/{username}";

    public async Task PrepareUserFolderAsync(string username, long userId, string path)
    {
        await dbService.CheckUsername(new KeyValuePair<string, long>(username, userId), path);

        if (!Directory.Exists(path))
        {
            Directory.CreateDirectory(path);
            Log.Debug($"Created folder for {username}");
        }

        await dbService.CreateDB(path);
    }

    public async Task<CreatorDownloadResult> DownloadCreatorContentAsync(
        string username, long userId, string path,
        Dictionary<string, long> users,
        bool clientIdBlobMissing, bool devicePrivateKeyMissing,
        IDownloadEventHandler eventHandler)
    {
        Config config = configService.CurrentConfig;
        CreatorDownloadResult counts = new();

        eventHandler.OnUserStarting(username);
        Log.Debug($"Scraping Data for {username}");

        await PrepareUserFolderAsync(username, userId, path);

        if (config.DownloadAvatarHeaderPhoto)
        {
            UserEntities.User? userInfo = await apiService.GetUserInfo($"/users/{username}");
            if (userInfo != null)
            {
                await downloadService.DownloadAvatarHeader(userInfo.Avatar, userInfo.Header, path, username);
            }
        }

        if (config.DownloadPaidPosts)
        {
            counts.PaidPostCount = await DownloadContentTypeAsync("Paid Posts",
                async statusReporter =>
                    await apiService.GetPaidPosts("/posts/paid/post", path, username, PaidPostIds, statusReporter),
                posts => posts.PaidPosts.Count,
                posts => posts.PaidPostObjects.Count,
                posts => posts.PaidPosts.Values.ToList(),
                async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users,
                    clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
                eventHandler);
        }

        if (config.DownloadPosts)
        {
            eventHandler.OnMessage(
                "Getting Posts (this may take a long time, depending on the number of Posts the creator has)");
            Log.Debug($"Calling DownloadFreePosts - {username}");

            counts.PostCount = await DownloadContentTypeAsync("Posts",
                async statusReporter =>
                    await apiService.GetPosts($"/users/{userId}/posts", path, PaidPostIds, statusReporter),
                posts => posts.Posts.Count,
                posts => posts.PostObjects.Count,
                posts => posts.Posts.Values.ToList(),
                async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users,
                    clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
                eventHandler);
        }

        if (config.DownloadArchived)
        {
            counts.ArchivedCount = await DownloadContentTypeAsync("Archived Posts",
                async statusReporter =>
                    await apiService.GetArchived($"/users/{userId}/posts", path, statusReporter),
                archived => archived.ArchivedPosts.Count,
                archived => archived.ArchivedPostObjects.Count,
                archived => archived.ArchivedPosts.Values.ToList(),
                async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users,
                    clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter),
                eventHandler);
        }

        if (config.DownloadStreams)
        {
            counts.StreamsCount = await DownloadContentTypeAsync("Streams",
                async statusReporter =>
                    await apiService.GetStreams($"/users/{userId}/posts/streams", path, PaidPostIds, statusReporter),
                streams => streams.Streams.Count,
                streams => streams.StreamObjects.Count,
                streams => streams.Streams.Values.ToList(),
                async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users,
                    clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter),
                eventHandler);
        }

        if (config.DownloadStories)
        {
            eventHandler.OnMessage("Getting Stories");
            Dictionary<long, string>? tempStories = await apiService.GetMedia(MediaType.Stories,
                $"/users/{userId}/stories", null, path, PaidPostIds);

            if (tempStories != null && tempStories.Count > 0)
            {
                eventHandler.OnContentFound("Stories", tempStories.Count, tempStories.Count);

                long totalSize = config.ShowScrapeSize
                    ? await downloadService.CalculateTotalFileSize(tempStories.Values.ToList())
                    : tempStories.Count;

                DownloadResult result = await eventHandler.WithProgressAsync(
                    $"Downloading {tempStories.Count} Stories", totalSize, config.ShowScrapeSize,
                    async reporter => await downloadService.DownloadStories(username, userId, path,
                        PaidPostIds.ToHashSet(), reporter));

                eventHandler.OnDownloadComplete("Stories", result);
                counts.StoriesCount = result.TotalCount;
            }
            else
            {
                eventHandler.OnNoContentFound("Stories");
            }
        }

        if (config.DownloadHighlights)
        {
            eventHandler.OnMessage("Getting Highlights");
            Dictionary<long, string>? tempHighlights = await apiService.GetMedia(MediaType.Highlights,
                $"/users/{userId}/stories/highlights", null, path, PaidPostIds);

            if (tempHighlights != null && tempHighlights.Count > 0)
            {
                eventHandler.OnContentFound("Highlights", tempHighlights.Count, tempHighlights.Count);

                long totalSize = config.ShowScrapeSize
                    ? await downloadService.CalculateTotalFileSize(tempHighlights.Values.ToList())
                    : tempHighlights.Count;

                DownloadResult result = await eventHandler.WithProgressAsync(
                    $"Downloading {tempHighlights.Count} Highlights", totalSize, config.ShowScrapeSize,
                    async reporter => await downloadService.DownloadHighlights(username, userId, path,
                        PaidPostIds.ToHashSet(), reporter));

                eventHandler.OnDownloadComplete("Highlights", result);
                counts.HighlightsCount = result.TotalCount;
            }
            else
            {
                eventHandler.OnNoContentFound("Highlights");
            }
        }

        if (config.DownloadMessages)
        {
            counts.MessagesCount = await DownloadContentTypeAsync("Messages",
                async statusReporter =>
                    await apiService.GetMessages($"/chats/{userId}/messages", path, statusReporter),
                messages => messages.Messages.Count,
                messages => messages.MessageObjects.Count,
                messages => messages.Messages.Values.ToList(),
                async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users,
                    clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter),
                eventHandler);
        }

        if (config.DownloadPaidMessages)
        {
            counts.PaidMessagesCount = await DownloadContentTypeAsync("Paid Messages",
                async statusReporter =>
                    await apiService.GetPaidMessages("/posts/paid/chat", path, username, statusReporter),
                paidMessages => paidMessages.PaidMessages.Count,
                paidMessages => paidMessages.PaidMessageObjects.Count,
                paidMessages => paidMessages.PaidMessages.Values.ToList(),
                async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users,
                    clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter),
                eventHandler);
        }

        eventHandler.OnUserComplete(username, counts);
        return counts;
    }

    public async Task DownloadSinglePostAsync(
        string username, long postId, string path,
        Dictionary<string, long> users,
        bool clientIdBlobMissing, bool devicePrivateKeyMissing,
        IDownloadEventHandler eventHandler)
    {
        Log.Debug($"Calling DownloadSinglePost - {postId}");
        eventHandler.OnMessage("Getting Post");

        PostEntities.SinglePostCollection post = await apiService.GetPost($"/posts/{postId}", path);
        if (post.SinglePosts.Count == 0)
        {
            eventHandler.OnMessage("Couldn't find post");
            Log.Debug("Couldn't find post");
            return;
        }

        Config config = configService.CurrentConfig;
        long totalSize = config.ShowScrapeSize
            ? await downloadService.CalculateTotalFileSize(post.SinglePosts.Values.ToList())
            : post.SinglePosts.Count;

        DownloadResult result = await eventHandler.WithProgressAsync(
            "Downloading Post", totalSize, config.ShowScrapeSize,
            async reporter => await downloadService.DownloadSinglePost(username, path, users,
                clientIdBlobMissing, devicePrivateKeyMissing, post, reporter));

        if (result.NewDownloads > 0)
        {
            eventHandler.OnMessage($"Post {postId} downloaded");
            Log.Debug($"Post {postId} downloaded");
        }
        else
        {
            eventHandler.OnMessage($"Post {postId} already downloaded");
            Log.Debug($"Post {postId} already downloaded");
        }
    }

    public async Task DownloadPurchasedTabAsync(
        Dictionary<string, long> users,
        bool clientIdBlobMissing, bool devicePrivateKeyMissing,
        IDownloadEventHandler eventHandler)
    {
        Config config = configService.CurrentConfig;

        Dictionary<string, long> purchasedTabUsers =
            await apiService.GetPurchasedTabUsers("/posts/paid/all", users);

        eventHandler.OnMessage("Checking folders for Users in Purchased Tab");

        foreach (KeyValuePair<string, long> user in purchasedTabUsers)
        {
            string path = ResolveDownloadPath(user.Key);
            Log.Debug($"Download path: {path}");

            await dbService.CheckUsername(user, path);

            if (!Directory.Exists(path))
            {
                Directory.CreateDirectory(path);
                Log.Debug($"Created folder for {user.Key}");
            }

            await apiService.GetUserInfo($"/users/{user.Key}");
            await dbService.CreateDB(path);
        }

        string basePath = !string.IsNullOrEmpty(config.DownloadPath)
            ? config.DownloadPath
            : "__user_data__/sites/OnlyFans/";

        Log.Debug($"Download path: {basePath}");

        List<PurchasedEntities.PurchasedTabCollection> purchasedTabCollections =
            await apiService.GetPurchasedTab("/posts/paid/all", basePath, users);

        foreach (PurchasedEntities.PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
        {
            eventHandler.OnUserStarting(purchasedTabCollection.Username);
            string path = ResolveDownloadPath(purchasedTabCollection.Username);
            Log.Debug($"Download path: {path}");

            int paidPostCount = 0;
            int paidMessagesCount = 0;

            // Download paid posts
            if (purchasedTabCollection.PaidPosts.PaidPosts.Count > 0)
            {
                eventHandler.OnContentFound("Paid Posts",
                    purchasedTabCollection.PaidPosts.PaidPosts.Count,
                    purchasedTabCollection.PaidPosts.PaidPostObjects.Count);

                long totalSize = config.ShowScrapeSize
                    ? await downloadService.CalculateTotalFileSize(
                        purchasedTabCollection.PaidPosts.PaidPosts.Values.ToList())
                    : purchasedTabCollection.PaidPosts.PaidPosts.Count;

                DownloadResult postResult = await eventHandler.WithProgressAsync(
                    $"Downloading {purchasedTabCollection.PaidPosts.PaidPosts.Count} Paid Posts",
                    totalSize, config.ShowScrapeSize,
                    async reporter => await downloadService.DownloadPaidPostsPurchasedTab(
                        purchasedTabCollection.Username, path, users,
                        clientIdBlobMissing, devicePrivateKeyMissing,
                        purchasedTabCollection.PaidPosts, reporter));

                eventHandler.OnDownloadComplete("Paid Posts", postResult);
                paidPostCount = postResult.TotalCount;
            }
            else
            {
                eventHandler.OnNoContentFound("Paid Posts");
            }

            // Download paid messages
            if (purchasedTabCollection.PaidMessages.PaidMessages.Count > 0)
            {
                eventHandler.OnContentFound("Paid Messages",
                    purchasedTabCollection.PaidMessages.PaidMessages.Count,
                    purchasedTabCollection.PaidMessages.PaidMessageObjects.Count);

                long totalSize = config.ShowScrapeSize
                    ? await downloadService.CalculateTotalFileSize(
                        purchasedTabCollection.PaidMessages.PaidMessages.Values.ToList())
                    : purchasedTabCollection.PaidMessages.PaidMessages.Count;

                DownloadResult msgResult = await eventHandler.WithProgressAsync(
                    $"Downloading {purchasedTabCollection.PaidMessages.PaidMessages.Count} Paid Messages",
                    totalSize, config.ShowScrapeSize,
                    async reporter => await downloadService.DownloadPaidMessagesPurchasedTab(
                        purchasedTabCollection.Username, path, users,
                        clientIdBlobMissing, devicePrivateKeyMissing,
                        purchasedTabCollection.PaidMessages, reporter));

                eventHandler.OnDownloadComplete("Paid Messages", msgResult);
                paidMessagesCount = msgResult.TotalCount;
            }
            else
            {
                eventHandler.OnNoContentFound("Paid Messages");
            }

            eventHandler.OnPurchasedTabUserComplete(purchasedTabCollection.Username, paidPostCount, paidMessagesCount);
        }
    }

    public async Task DownloadSinglePaidMessageAsync(
        string username, long messageId, string path,
        Dictionary<string, long> users,
        bool clientIdBlobMissing, bool devicePrivateKeyMissing,
        IDownloadEventHandler eventHandler)
    {
        Log.Debug($"Calling DownloadSinglePaidMessage - {username}");
        eventHandler.OnMessage("Getting Paid Message");

        PurchasedEntities.SinglePaidMessageCollection singlePaidMessageCollection =
            await apiService.GetPaidMessage($"/messages/{messageId}", path);

        if (singlePaidMessageCollection.SingleMessages.Count == 0)
        {
            eventHandler.OnNoContentFound("Paid Messages");
            return;
        }

        Config config = configService.CurrentConfig;

        // Handle preview messages
        if (singlePaidMessageCollection.PreviewSingleMessages.Count > 0)
        {
            eventHandler.OnContentFound("Preview Paid Messages",
                singlePaidMessageCollection.PreviewSingleMessages.Count,
                singlePaidMessageCollection.SingleMessageObjects.Count);

            long previewSize = config.ShowScrapeSize
                ? await downloadService.CalculateTotalFileSize(
                    singlePaidMessageCollection.PreviewSingleMessages.Values.ToList())
                : singlePaidMessageCollection.PreviewSingleMessages.Count;

            DownloadResult previewResult = await eventHandler.WithProgressAsync(
                $"Downloading {singlePaidMessageCollection.PreviewSingleMessages.Count} Preview Paid Messages",
                previewSize, config.ShowScrapeSize,
                async reporter => await downloadService.DownloadSinglePaidMessage(username, path, users,
                    clientIdBlobMissing, devicePrivateKeyMissing, singlePaidMessageCollection, reporter));

            eventHandler.OnDownloadComplete("Paid Messages", previewResult);
        }
        else if (singlePaidMessageCollection.SingleMessages.Count > 0)
        {
            // Only actual paid messages, no preview
            eventHandler.OnContentFound("Paid Messages",
                singlePaidMessageCollection.SingleMessages.Count,
                singlePaidMessageCollection.SingleMessageObjects.Count);

            long totalSize = config.ShowScrapeSize
                ? await downloadService.CalculateTotalFileSize(
                    singlePaidMessageCollection.SingleMessages.Values.ToList())
                : singlePaidMessageCollection.SingleMessages.Count;

            DownloadResult result = await eventHandler.WithProgressAsync(
                $"Downloading {singlePaidMessageCollection.SingleMessages.Count} Paid Messages",
                totalSize, config.ShowScrapeSize,
                async reporter => await downloadService.DownloadSinglePaidMessage(username, path, users,
                    clientIdBlobMissing, devicePrivateKeyMissing, singlePaidMessageCollection, reporter));

            eventHandler.OnDownloadComplete("Paid Messages", result);
        }
        else
        {
            eventHandler.OnNoContentFound("Paid Messages");
        }
    }

    public async Task<string?> ResolveUsernameAsync(long userId)
    {
        JObject? user = await apiService.GetUserInfoById($"/users/list?x[]={userId}");
        if (user == null)
        {
            return $"Deleted User - {userId}";
        }

        string? username = user[userId.ToString()]?["username"]?.ToString();
        return !string.IsNullOrEmpty(username) ? username : $"Deleted User - {userId}";
    }

    /// <summary>
    ///     Generic helper for the common pattern: fetch with status -> check count -> download with progress.
    /// </summary>
    private async Task<int> DownloadContentTypeAsync<T>(
        string contentType,
        Func<IStatusReporter, Task<T>> fetchData,
        Func<T, int> getMediaCount,
        Func<T, int> getObjectCount,
        Func<T, List<string>?> getUrls,
        Func<T, IProgressReporter, Task<DownloadResult>> downloadData,
        IDownloadEventHandler eventHandler)
    {
        T data = await eventHandler.WithStatusAsync($"Getting {contentType}",
            async statusReporter => await fetchData(statusReporter));

        int mediaCount = getMediaCount(data);
        if (mediaCount <= 0)
        {
            eventHandler.OnNoContentFound(contentType);
            Log.Debug($"Found 0 {contentType}");
            return 0;
        }

        int objectCount = getObjectCount(data);
        eventHandler.OnContentFound(contentType, mediaCount, objectCount);
        Log.Debug($"Found {mediaCount} Media from {objectCount} {contentType}");

        Config config = configService.CurrentConfig;
        List<string>? urls = getUrls(data);
        long totalSize = config.ShowScrapeSize && urls != null
            ? await downloadService.CalculateTotalFileSize(urls)
            : mediaCount;

        DownloadResult result = await eventHandler.WithProgressAsync(
            $"Downloading {mediaCount} {contentType}", totalSize, config.ShowScrapeSize,
            async reporter => await downloadData(data, reporter));

        eventHandler.OnDownloadComplete(contentType, result);
        Log.Debug(
            $"{contentType} Already Downloaded: {result.ExistingDownloads} New {contentType} Downloaded: {result.NewDownloads}");

        return result.TotalCount;
    }
}