Reordered methods in CajetanApiService

Cajetan.OF-DL/CLI/CajetanDownloadEventHandler.cs

@@ -21,7 +21,7 @@ public class CajetanDownloadEventHandler : ICajetanDownloadEventHandler
         => _eventHandler.OnMessage(message);
 
     public void OnMessage(string message, string color)
-        => AnsiConsole.Markup($"[{color.ToLowerInvariant()}]{Markup.Escape(message)}\n[/]");
+        => AnsiConsole.MarkupLine($"[{color.ToLowerInvariant()}]{Markup.Escape(message)}[/]");
 
     public void OnNoContentFound(string contentType)
         => _eventHandler.OnNoContentFound(contentType);

Cajetan.OF-DL/Cajetan.OF-DL.csproj

@@ -31,6 +31,7 @@
 
   <ItemGroup>
     <PackageReference Include="Serilog.Sinks.Seq" Version="7.0.1" />
+    <PackageReference Include="Serilog.Sinks.Debug" Version="3.0.0" />
     <PackageReference Include="Akka" Version="1.5.60"/>
     <PackageReference Include="BouncyCastle.NetCore" Version="2.2.1"/>
     <PackageReference Include="HtmlAgilityPack" Version="1.12.4"/>

Cajetan.OF-DL/Models/CajetanConfig.cs

@@ -5,7 +5,5 @@ public class CajetanConfig
     public string[] NonInteractiveSpecificLists { get; set; } = [];
     public string[] NonInteractiveSpecificUsers { get; set; } = [];
 
-    public EMode Mode { get; set; } = EMode.None;
-
-    public string ErrorMessage { get; set; } = string.Empty;
+    public EMode Mode { get; set; } = EMode.DownloadCreatorContent;
 }

Cajetan.OF-DL/Models/Dtos/Lists/ListUsersDto.cs

@@ -0,0 +1,8 @@
+namespace OF_DL.Models.Dtos.Lists;
+
+public class ListUsersDto
+{
+    [JsonProperty("list")] public List<UsersListDto> List { get; set; } = [];
+    [JsonProperty("hasMore")] public bool? HasMore { get; set; }
+    [JsonProperty("nextOffset")] public int NextOffset { get; set; }
+}

Cajetan.OF-DL/Models/EMode.cs

@@ -2,7 +2,6 @@ namespace OF_DL.Models;
 
 public enum EMode
 {
-    None,
     DownloadCreatorContent,
     OutputBlockedUsers,
     UpdateAllUserInfo

Cajetan.OF-DL/ProgramCajetan.cs

@@ -20,7 +20,7 @@ static async Task<ServiceCollection> ConfigureServices(string[] args)
 {
     // Set up dependency injection with LoggingService and ConfigService
     ServiceCollection services = new();
-    services.AddSingleton<ILoggingService, SeqLoggingService>();
+    services.AddSingleton<ILoggingService, CajetanLoggingService>();
     services.AddSingleton<IConfigService, ConfigService>();
     services.AddSingleton(new ExitHelper(new SpectreDownloadEventHandler()));
     ServiceProvider tempServiceProvider = services.BuildServiceProvider();
@@ -49,14 +49,7 @@ static async Task<ServiceCollection> ConfigureServices(string[] args)
 
     AnsiConsole.Markup("[green]config.conf located successfully!\n[/]");
 
-    if (!ParseCommandlineArgs(args, configService.CurrentConfig, out CajetanConfig cajetanConfig))
-    {
-        AnsiConsole.MarkupLine($"\n[red]{cajetanConfig.ErrorMessage}[/]\n");
-        AnsiConsole.MarkupLine("[red]Press any key to exit.[/]");
-        Console.ReadKey();
-
-        exitHelper.ExitWithCode(3);
-    }
+    CajetanConfig cajetanConfig = ParseCommandlineArgs(args, configService.CurrentConfig);
 
     // Set up full dependency injection with loaded config
     services = [];
@@ -90,26 +83,26 @@ static async Task<ServiceCollection> ConfigureServices(string[] args)
     return services;
 }
 
-static bool ParseCommandlineArgs(string[] args, Config currentConfig, out CajetanConfig parsedConfig)
+static CajetanConfig ParseCommandlineArgs(string[] args, Config currentConfig)
 {
     const string SPECIFIC_LISTS_ARG = "--specific-lists";
     const string SPECIFIC_USERS_ARG = "--specific-users";
     const string OUTPUT_BLOCKED_USERS_ARG = "--output-blocked";
     const string UPDATE_ALL_USER_INFO_ARG = "--update-userinfo";
 
-    parsedConfig = new();
+    CajetanConfig parsedConfig = new();
 
     if (ParseListAndUserArguments(ref parsedConfig))
-        return true;
+        return parsedConfig;
 
     if (ParseFlagArgument(OUTPUT_BLOCKED_USERS_ARG, EMode.OutputBlockedUsers, ref parsedConfig))
-        return true;
+        return parsedConfig;
 
     if (ParseFlagArgument(UPDATE_ALL_USER_INFO_ARG, EMode.UpdateAllUserInfo, ref parsedConfig))
-        return true;
+        return parsedConfig;
 
-    parsedConfig.ErrorMessage = "No mode argument provided!";
-    return false;
+    // Will process all active subscriptions
+    return parsedConfig;
 
     bool ParseListAndUserArguments(ref CajetanConfig parsedConfig)
     {

Cajetan.OF-DL/Properties/GlobalUsings.cs

@@ -11,6 +11,7 @@ global using OF_DL.Services;
 global using Serilog;
 global using Serilog.Context;
 global using Spectre.Console;
+global using ListsDtos = OF_DL.Models.Dtos.Lists;
 global using MessageDtos = OF_DL.Models.Dtos.Messages;
 global using MessageEntities = OF_DL.Models.Entities.Messages;
 global using SubscriptionsDtos = OF_DL.Models.Dtos.Subscriptions;

Cajetan.OF-DL/Services/CajetanApiService.cs

@@ -27,6 +27,95 @@ public class CajetanApiService(IAuthService authService, IConfigService configSe
         return GetAllSubscriptions(endpoint, includeRestricted, "expired");
     }
 
+    public new async Task<MessageEntities.MessageCollection> GetMessages(string endpoint, string folder, IStatusReporter statusReporter)
+    {
+        (bool couldExtract, long userId) = ExtractUserId(endpoint);
+
+        _eventHandler.OnMessage("Getting Unread Chats", "grey");
+        HashSet<long> usersWithUnread = couldExtract ? await GetUsersWithUnreadMessagesAsync() : [];
+
+        MessageEntities.MessageCollection messages = await base.GetMessages(endpoint, folder, statusReporter);
+
+        if (usersWithUnread.Contains(userId))
+        {
+            _eventHandler.OnMessage("Restoring unread state", "grey");
+            await MarkAsUnreadAsync($"/chats/{userId}/mark-as-read");
+        }
+
+        return messages;
+
+        static (bool couldExtract, long userId) ExtractUserId(string endpoint)
+        {
+            string withoutChatsAndMessages = endpoint
+                .Replace("chats", "", StringComparison.OrdinalIgnoreCase)
+                .Replace("messages", "", StringComparison.OrdinalIgnoreCase);
+            string trimmed = withoutChatsAndMessages.Trim(' ', '/', '\\');
+
+            if (long.TryParse(trimmed, out long userId))
+                return (true, userId);
+
+            return (false, default);
+        }
+    }
+
+    public new async Task<Dictionary<string, long>?> GetListUsers(string endpoint)
+    {
+        if (!HasSignedRequestAuth())
+            return null;
+
+        try
+        {
+            Dictionary<string, long> users = new(StringComparer.OrdinalIgnoreCase);
+
+            Log.Debug($"Calling GetListUsers - {endpoint}");
+
+            const int limit = 50;
+            int offset = 0;
+
+            Dictionary<string, string> getParams = new()
+            {
+                ["format"] = "infinite",
+                ["limit"] = limit.ToString()
+            };
+
+            while (true)
+            {
+                getParams["offset"] = offset.ToString();
+
+                string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
+
+                if (string.IsNullOrWhiteSpace(body))
+                    break;
+
+                ListsDtos.ListUsersDto? listUsers = DeserializeJson<ListsDtos.ListUsersDto>(body, s_mJsonSerializerSettings);
+
+                if (listUsers?.List is null)
+                    break;
+
+                foreach (ListsDtos.UsersListDto item in listUsers.List)
+                {
+                    if (item.Id is null)
+                        continue;
+
+                    users.TryAdd(item.Username, item.Id.Value);
+                }
+
+                if (listUsers.HasMore is false)
+                    break;
+
+                offset = listUsers.NextOffset;
+            }
+
+            return users;
+        }
+        catch (Exception ex)
+        {
+            ExceptionLoggerHelper.LogException(ex);
+        }
+
+        return null;
+    }
+
     public async Task<UserEntities.UserInfo?> GetDetailedUserInfoAsync(string endpoint)
     {
         Log.Debug($"Calling GetDetailedUserInfo: {endpoint}");
@@ -64,29 +153,6 @@ public class CajetanApiService(IAuthService authService, IConfigService configSe
         return null;
     }
 
-    public async Task SortBlockedAsync(string endpoint, string order = "recent", string direction = "desc")
-    {
-        Log.Debug($"Calling SortBlocked - {endpoint}");
-
-        try
-        {
-            var reqBody = new { order, direction };
-            var result = new { success = false, canAddFriends = false };
-
-            string? body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(), HttpMethod.Post, reqBody);
-
-            if (!string.IsNullOrWhiteSpace(body))
-                result = JsonConvert.DeserializeAnonymousType(body, result);
-
-            if (result?.success != true)
-                _eventHandler.OnMessage($"Failed to sort blocked (order: {order}, direction; {direction})! Endpoint: {endpoint}", "yellow");
-        }
-        catch (Exception ex)
-        {
-            ExceptionLoggerHelper.LogException(ex);
-        }
-    }
-
     public async Task<Dictionary<string, long>> GetUsersWithProgressAsync(string typeDisplay, string endpoint, string? typeParam, bool offsetByCount)
     {
         Dictionary<string, long> usersOfType = await _eventHandler.WithStatusAsync(
@@ -172,37 +238,6 @@ public class CajetanApiService(IAuthService authService, IConfigService configSe
         }
     }
 
-    public new async Task<MessageEntities.MessageCollection> GetMessages(string endpoint, string folder, IStatusReporter statusReporter)
-    {
-        (bool couldExtract, long userId) = ExtractUserId(endpoint);
-
-        _eventHandler.OnMessage("Getting Unread Chats", "grey");
-        HashSet<long> usersWithUnread = couldExtract ? await GetUsersWithUnreadMessagesAsync() : [];
-
-        MessageEntities.MessageCollection messages = await base.GetMessages(endpoint, folder, statusReporter);
-
-        if (usersWithUnread.Contains(userId))
-        {
-            _eventHandler.OnMessage("Restoring unread state", "grey");
-            await MarkAsUnreadAsync($"/chats/{userId}/mark-as-read");
-        }
-
-        return messages;
-
-        static (bool couldExtract, long userId) ExtractUserId(string endpoint)
-        {
-            string withoutChatsAndMessages = endpoint
-                .Replace("chats", "", StringComparison.OrdinalIgnoreCase)
-                .Replace("messages", "", StringComparison.OrdinalIgnoreCase);
-            string trimmed = withoutChatsAndMessages.Trim(' ', '/', '\\');
-
-            if (long.TryParse(trimmed, out long userId))
-                return (true, userId);
-
-            return (false, default);
-        }
-    }
-
     public async Task<HashSet<long>> GetUsersWithUnreadMessagesAsync()
     {
         MessageDtos.ChatsDto unreadChats = await GetChatsAsync("/chats", onlyUnread: true);
@@ -244,6 +279,29 @@ public class CajetanApiService(IAuthService authService, IConfigService configSe
         }
     }
 
+    public async Task SortBlockedAsync(string endpoint, string order = "recent", string direction = "desc")
+    {
+        Log.Debug($"Calling SortBlocked - {endpoint}");
+
+        try
+        {
+            var reqBody = new { order, direction };
+            var result = new { success = false, canAddFriends = false };
+
+            string? body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(), HttpMethod.Post, reqBody);
+
+            if (!string.IsNullOrWhiteSpace(body))
+                result = JsonConvert.DeserializeAnonymousType(body, result);
+
+            if (result?.success != true)
+                _eventHandler.OnMessage($"Failed to sort blocked (order: {order}, direction; {direction})! Endpoint: {endpoint}", "yellow");
+        }
+        catch (Exception ex)
+        {
+            ExceptionLoggerHelper.LogException(ex);
+        }
+    }
+
     private async Task<Dictionary<string, long>?> GetAllSubscriptions(string endpoint, bool includeRestricted, string type)
     {
         if (!HasSignedRequestAuth())

Cajetan.OF-DL/Services/CajetanLoggingService.cs

@@ -1,13 +1,11 @@
-using OF_DL.Enumerations;
-using Serilog;
 using Serilog.Core;
 using Serilog.Events;
 
 namespace OF_DL.Services;
 
-public class SeqLoggingService : ILoggingService
+public class CajetanLoggingService : ILoggingService
 {
-    public SeqLoggingService()
+    public CajetanLoggingService()
     {
         LevelSwitch = new LoggingLevelSwitch();
         InitializeLoggerWithSeq();
@@ -35,15 +33,21 @@ public class SeqLoggingService : ILoggingService
 
     private void InitializeLoggerWithSeq()
     {
-        Log.Logger = new LoggerConfiguration()
+        LevelSwitch.MinimumLevel = LogEventLevel.Warning;
+
+        LoggerConfiguration loggerConfig = new LoggerConfiguration()
             .Enrich.FromLogContext()
             .Enrich.WithProperty("Application", "OF_DL")
             .Enrich.WithProperty("StartTime", $"{DateTime.Now:yyyy-MM-dd HH:mm:ss} ")
             .Enrich.WithProperty("MachineName", Environment.MachineName)
-            .MinimumLevel.ControlledBy(LevelSwitch)
-            .WriteTo.File("logs/OFDL.txt", rollingInterval: RollingInterval.Day, restrictedToMinimumLevel: LogEventLevel.Error)
-            .WriteTo.Seq("https://seq.cajetan.dk")
-            .CreateLogger();
+            .MinimumLevel.Verbose()
+            .WriteTo.File("logs/OFDL.txt", rollingInterval: RollingInterval.Day, restrictedToMinimumLevel: LogEventLevel.Error, levelSwitch: LevelSwitch)
+            .WriteTo.Seq("https://seq.cajetan.dk", controlLevelSwitch: LevelSwitch);
+
+        if (System.Diagnostics.Debugger.IsAttached)
+            loggerConfig.WriteTo.Debug(restrictedToMinimumLevel: LogEventLevel.Debug);
+
+        Log.Logger = loggerConfig.CreateLogger();
 
         Log.Debug("Logging service initialized");
     }

Cajetan.OF-DL/Services/ICajetanApiService.cs

@@ -2,6 +2,8 @@ namespace OF_DL.Services;
 
 public interface ICajetanApiService : IApiService
 {
+    new Task<Dictionary<string, long>?> GetListUsers(string endpoint);
+
     Task<UserEntities.UserInfo?> GetDetailedUserInfoAsync(string endpoint);
     Task<Dictionary<string, long>> GetUsersWithProgressAsync(string typeDisplay, string endpoint, string? typeParam, bool offsetByCount);
     Task<HashSet<long>> GetUsersWithUnreadMessagesAsync();

Cajetan.OF-DL/Worker.cs

@@ -134,7 +134,7 @@ internal class Worker(IServiceProvider serviceProvider)
         DateTime startTime = DateTime.Now;
 
         UserListResult allUsersAndLists = await GetAvailableUsersAsync();
-        Dictionary<string, long> usersToDownload = [];
+        Dictionary<string, long> usersToDownload = allUsersAndLists.Users;
 
         if (_cajetanConfig.NonInteractiveSpecificLists is not null && _cajetanConfig.NonInteractiveSpecificLists.Length > 0)
             usersToDownload = await GetUsersFromSpecificListsAsync(allUsersAndLists, _cajetanConfig.NonInteractiveSpecificLists);
@@ -142,9 +142,6 @@ internal class Worker(IServiceProvider serviceProvider)
         else if (_cajetanConfig.NonInteractiveSpecificUsers is not null && _cajetanConfig.NonInteractiveSpecificUsers.Length > 0)
             usersToDownload = GetUsersFromSpecificUsernames(allUsersAndLists, [.. _cajetanConfig.NonInteractiveSpecificUsers]);
 
-        if (usersToDownload.Count == 0)
-            return;
-
         int userNum = 0;
         int userCount = usersToDownload.Count;
         CajetanDownloadEventHandler eventHandler = new();
@@ -152,7 +149,11 @@ internal class Worker(IServiceProvider serviceProvider)
 
         LoggerWithConfigContext(_configService.CurrentConfig, _cajetanConfig)
             .Information("Scraping Data for {UserCount} user(s)", usersToDownload.Count);
-        eventHandler.OnMessage($"Scraping Data for {usersToDownload.Count} user(s)\n");
+
+        eventHandler.OnMessage(
+            $"\nScraping Data for {usersToDownload.Count} user(s)\n" +
+            $"{"======================================================================================================"}\n"
+        );
 
         foreach ((string username, long userId) in usersToDownload)
         {
@@ -182,17 +183,19 @@ internal class Worker(IServiceProvider serviceProvider)
                 DateTime userEndTime = DateTime.Now;
                 TimeSpan userTotalTime = userEndTime - userStartTime;
 
-                Log.ForContext("Paid Posts", newResults.PaidPostCount)
-                   .ForContext("Posts", newResults.PostCount)
+                Log.ForContext("Posts", newResults.PostCount)
+                   .ForContext("PaidPosts", newResults.PaidPostCount)
+                   .ForContext("AllPosts", newResults.PostCount + newResults.PaidPostCount)
                    .ForContext("Archived", newResults.ArchivedCount)
                    .ForContext("Streams", newResults.StreamsCount)
                    .ForContext("Stories", newResults.StoriesCount)
                    .ForContext("Highlights", newResults.HighlightsCount)
                    .ForContext("Messages", newResults.MessagesCount)
-                   .ForContext("Paid Messages", newResults.PaidMessagesCount)
+                   .ForContext("PaidMessages", newResults.PaidMessagesCount)
+                   .ForContext("AllMessages", newResults.MessagesCount + newResults.PaidMessagesCount)
                    .ForContext("Username", username)
                    .ForContext("TotalMinutes", userTotalTime.TotalMinutes)
-                   .Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes");
+                   .Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes [P: {AllPosts}] [M: {AllMessages}]");
             }
             catch (Exception ex)
             {
@@ -209,16 +212,18 @@ internal class Worker(IServiceProvider serviceProvider)
 
         eventHandler.OnScrapeComplete(totalTime);
 
-        Log.ForContext("Paid Posts", totalResults.PaidPostCount)
-           .ForContext("Posts", totalResults.PostCount)
+        Log.ForContext("Posts", totalResults.PostCount)
+           .ForContext("PaidPosts", totalResults.PaidPostCount)
+           .ForContext("AllPosts", totalResults.PostCount + totalResults.PaidPostCount)
            .ForContext("Archived", totalResults.ArchivedCount)
            .ForContext("Streams", totalResults.StreamsCount)
            .ForContext("Stories", totalResults.StoriesCount)
            .ForContext("Highlights", totalResults.HighlightsCount)
            .ForContext("Messages", totalResults.MessagesCount)
-           .ForContext("Paid Messages", totalResults.PaidMessagesCount)
+           .ForContext("PaidMessages", totalResults.PaidMessagesCount)
+           .ForContext("AllMessages", totalResults.MessagesCount + totalResults.PaidMessagesCount)
            .ForContext("TotalMinutes", totalTime.TotalMinutes)
-           .Information("Scrape Completed in {TotalMinutes:0.00} minutes");
+           .Information("Scrape Completed in {TotalMinutes:0.00} minutes [P: {AllPosts}] [M: {AllMessages}]");
 
         await Task.Delay(2000);
     }
@@ -322,19 +327,16 @@ internal class Worker(IServiceProvider serviceProvider)
                 continue;
 
             Log.Information("Getting Users from list '{ListName:l}' (Include Restricted: {IncludeRestrictedSubscriptions})", name, currentConfig.IncludeRestrictedSubscriptions);
-            AnsiConsole.Markup($"[green]Getting Users from list '{name}' (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
+            AnsiConsole.MarkupLine($"[green]Getting Users from list '{name}' (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})[/]");
 
-            List<string> listUsernames = await _apiService.GetListUsers($"/lists/{listId}/users") ?? [];
+            Dictionary<string, long> listUsernames = await _apiService.GetListUsers($"/lists/{listId}/users") ?? [];
 
-            foreach (string u in listUsernames)
+            foreach ((string username, long userId) in listUsernames)
             {
-                if (usersFromLists.ContainsKey(u))
+                if (usersFromLists.ContainsKey(username))
                     continue;
 
-                if (!allUsersAndLists.Users.TryGetValue(u, out long userId))
-                    continue;
-
-                usersFromLists[u] = userId;
+                usersFromLists[username] = userId;
             }
         }
 
@@ -373,7 +375,7 @@ internal class Worker(IServiceProvider serviceProvider)
         async Task FetchUsersAsync()
         {
             Log.Information("Getting Active Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
-            AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
+            AnsiConsole.MarkupLine($"[green]Getting Active Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})[/]");
 
             Dictionary<string, long>? activeSubs = await _apiService.GetActiveSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
             AddToResult(activeSubs);
@@ -381,7 +383,7 @@ internal class Worker(IServiceProvider serviceProvider)
             if (currentConfig.IncludeExpiredSubscriptions)
             {
                 Log.Information("Getting Expired Subscriptions (Include Restricted: {IncludeRestrictedSubscriptions})", currentConfig.IncludeRestrictedSubscriptions);
-                AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})\n[/]");
+                AnsiConsole.MarkupLine($"[green]Getting Expired Subscriptions (Include Restricted: {currentConfig.IncludeRestrictedSubscriptions})[/]");
 
                 Dictionary<string, long>? expiredSubs = await _apiService.GetExpiredSubscriptions("/subscriptions/subscribes", currentConfig.IncludeRestrictedSubscriptions);
                 AddToResult(expiredSubs);
@@ -410,7 +412,7 @@ internal class Worker(IServiceProvider serviceProvider)
 
             Dictionary<string, long> usersInNonNudeLists = await GetUsersFromSpecificListsAsync(result, [.. listNames]);
 
-            AnsiConsole.Markup($"[green]Updating Non-Nude collection with {usersInNonNudeLists.Count} Users[/]");
+            AnsiConsole.MarkupLine($"[grey]Updating Non-Nude collection with {usersInNonNudeLists.Count} Users[/]");
             await _dbService.UpdateNonNudeCollectionAsync(usersInNonNudeLists);
 
             AnsiConsole.WriteLine();