Enabled updating of User Info during scrape

Cajetan.OF-DL/ProgramCajetan.cs

@@ -62,8 +62,13 @@ static async Task<ServiceCollection> ConfigureServices(string[] args)
     services.AddSingleton(cajetanConfig);
 
     services.AddSingleton<IAuthService, AuthService>();
-    services.AddSingleton<IApiService, ApiService>();
+
-    services.AddSingleton<IDbService, DbService>();
+    services.AddSingleton<ICajetanApiService, CajetanApiService>();
+    services.AddSingleton<IApiService>(sp => sp.GetRequiredService<ICajetanApiService>());
+
+    services.AddSingleton<ICajetanDbService, CajetanDbService>();
+    services.AddSingleton<IDbService>(sp => sp.GetRequiredService<ICajetanDbService>());
+
     services.AddSingleton<IDownloadService, DownloadService>();
     services.AddSingleton<IFileNameService, FileNameService>();
     services.AddSingleton<IStartupService, StartupService>();

Cajetan.OF-DL/Properties/GlobalUsings.cs

@@ -1,5 +1,4 @@
-global using Serilog;
+global using Newtonsoft.Json;
-global using Spectre.Console;
 global using OF_DL;
 global using OF_DL.CLI;
 global using OF_DL.Crypto;
@@ -8,5 +7,13 @@ global using OF_DL.Exceptions;
 global using OF_DL.Helpers;
 global using OF_DL.Models;
 global using OF_DL.Models.Config;
+global using OF_DL.Models.Downloads;
 global using OF_DL.Services;
-global using OF_DL.Utils;
+global using Serilog;
+global using Serilog.Context;
+global using Spectre.Console;
+global using MessageDtos = OF_DL.Models.Dtos.Messages;
+global using MessageEntities = OF_DL.Models.Entities.Messages;
+global using SubscriptionDtos = OF_DL.Models.Dtos.Subscriptions;
+global using UserDtos = OF_DL.Models.Dtos.Users;
+global using UserEntities = OF_DL.Models.Entities.Users;

Cajetan.OF-DL/Services/CajetanApiService.cs

@@ -0,0 +1,94 @@
+namespace OF_DL.Services;
+
+public interface ICajetanApiService : IApiService
+{
+    Task<UserEntities.UserInfo?> GetDetailedUserInfo(string endpoint);
+}
+
+public class CajetanApiService(IAuthService authService, IConfigService configService, ICajetanDbService dbService)
+    : ApiService(authService, configService, dbService), ICajetanApiService
+{
+    public new async Task<UserEntities.User?> GetUserInfo(string endpoint)
+    {
+        UserEntities.UserInfo? userInfo = await GetDetailedUserInfoAsync(endpoint);
+
+        if (userInfo is not null && !endpoint.EndsWith("/me"))
+            await dbService.UpdateUserInfoAsync(userInfo);
+
+        return userInfo;
+    }
+
+    public async Task<UserEntities.UserInfo?> GetDetailedUserInfoAsync(string endpoint)
+    {
+        Log.Debug($"Calling GetDetailedUserInfo: {endpoint}");
+
+        if (!HasSignedRequestAuth())
+            return null;
+
+        try
+        {
+            UserEntities.UserInfo userInfo = new();
+            Dictionary<string, string> getParams = new()
+            {
+                { "limit", Constants.ApiPageSize.ToString() }, { "order", "publish_date_asc" }
+            };
+
+            HttpClient client = new();
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
+
+            using HttpResponseMessage response = await client.SendAsync(request);
+
+            if (!response.IsSuccessStatusCode)
+                return userInfo;
+
+            response.EnsureSuccessStatusCode();
+            string body = await response.Content.ReadAsStringAsync();
+            UserDtos.UserDto? userDto = JsonConvert.DeserializeObject<UserDtos.UserDto>(body, s_mJsonSerializerSettings);
+            userInfo = FromDto(userDto);
+            return userInfo;
+        }
+        catch (Exception ex)
+        {
+            ExceptionLoggerHelper.LogException(ex);
+        }
+
+        return null;
+    }
+
+    private static UserEntities.UserInfo FromDto(UserDtos.UserDto? userDto)
+    {
+        if (userDto is null)
+            return new();
+
+        return new()
+        {
+            Id = userDto.Id,
+
+            Avatar = userDto.Avatar,
+            Header = userDto.Header,
+            Name = userDto.Name,
+            Username = userDto.Username,
+
+            SubscribePrice = userDto.SubscribePrice,
+            CurrentSubscribePrice = userDto.CurrentSubscribePrice,
+            IsPaywallRequired = userDto.IsPaywallRequired,
+            IsRestricted = userDto.IsRestricted,
+            SubscribedBy = userDto.SubscribedBy,
+            SubscribedByExpire = userDto.SubscribedByExpire,
+            SubscribedByExpireDate = userDto.SubscribedByExpireDate,
+            SubscribedByAutoprolong = userDto.SubscribedByAutoprolong,
+            SubscribedIsExpiredNow = userDto.SubscribedIsExpiredNow,
+            SubscribedOn = userDto.SubscribedOn,
+            SubscribedOnExpiredNow = userDto.SubscribedOnExpiredNow,
+            SubscribedOnDuration = userDto.SubscribedOnDuration,
+            About = userDto.About,
+            PostsCount = userDto.PostsCount,
+            ArchivedPostsCount = userDto.ArchivedPostsCount,
+            PrivateArchivedPostsCount = userDto.PrivateArchivedPostsCount,
+            PhotosCount = userDto.PhotosCount,
+            VideosCount = userDto.VideosCount,
+            AudiosCount = userDto.AudiosCount,
+            MediasCount = userDto.MediasCount,
+        };
+    }
+}

Cajetan.OF-DL/Services/CajetanDbService.cs

@@ -0,0 +1,107 @@
+using Microsoft.Data.Sqlite;
+
+namespace OF_DL.Services;
+
+public class CajetanDbService(IConfigService configService)
+    : DbService(configService), ICajetanDbService
+{
+    public async Task InitializeUserInfoTablesAsync()
+    {
+        await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+
+        connection.Open();
+
+        using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, about VARCHAR NULL, expires_on TIMESTAMP NULL, photo_count INT NOT NULL, video_count INT NOT NULL, PRIMARY KEY(user_id));", connection))
+        {
+            await cmdInfo.ExecuteNonQueryAsync();
+        }
+
+        using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info_blob (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, blob TEXT NULL, PRIMARY KEY(user_id));", connection))
+        {
+            await cmdInfo.ExecuteNonQueryAsync();
+        }
+
+    }
+
+    public async Task<Dictionary<string, long>> GetUsersAsync()
+    {
+        await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+
+        using SqliteCommand cmd = new("SELECT user_id, username FROM users", connection);
+        using SqliteDataReader reader = cmd.ExecuteReader();
+
+        Dictionary<string, long> result = new(StringComparer.OrdinalIgnoreCase);
+
+        while (reader.Read())
+        {
+            long userId = reader.GetInt64(0);
+            string username = reader.GetString(1);
+
+            result[username] = userId;
+        }
+
+        return result;
+    }
+
+    public async Task UpdateUserInfoAsync(UserEntities.UserInfo? userInfo)
+    {
+        if (userInfo?.Id is null || userInfo?.Username is null)
+            return;
+
+        await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+
+        Log.Debug("Database data source: " + connection.DataSource);
+
+        await UpdateAsync();
+        await UpdateBlobAsync();
+
+        async Task UpdateAsync()
+        {
+            using SqliteCommand cmdInfo = new(
+                "INSERT OR REPLACE INTO user_info (user_id, name, about, expires_on, photo_count, video_count) " +
+                "VALUES (@userId, @name, @about, @expiresOn, @photoCount, @videoCount);",
+                connection
+            );
+
+            cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
+            cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
+            cmdInfo.Parameters.AddWithValue("@about", userInfo.About);
+            cmdInfo.Parameters.AddWithValue("@expiresOn", userInfo.SubscribedByExpireDate);
+            cmdInfo.Parameters.AddWithValue("@photoCount", userInfo.PhotosCount ?? 0);
+            cmdInfo.Parameters.AddWithValue("@videoCount", userInfo.VideosCount ?? 0);
+
+            try
+            {
+                await cmdInfo.ExecuteNonQueryAsync();
+                Log.Debug("Inserted or updated creator info: {Username:l}", userInfo.Username);
+            }
+            catch (Exception ex)
+            {
+                Log.Warning(ex, "Failed to update User Info for: {Username:l}", userInfo.Username);
+            }
+        }
+
+        async Task UpdateBlobAsync()
+        {
+            using SqliteCommand cmdInfo = new(
+                "INSERT OR REPLACE INTO user_info_blob (user_id, name, blob) " +
+                "VALUES (@userId, @name, @blob);",
+                connection
+            );
+
+            cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
+            cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
+            cmdInfo.Parameters.AddWithValue("@blob", Newtonsoft.Json.JsonConvert.SerializeObject(userInfo));
+
+            try
+            {
+                await cmdInfo.ExecuteNonQueryAsync();
+                Log.Debug("Inserted or updated creator blob: {Username:l}", userInfo.Username);
+            }
+            catch (Exception ex)
+            {
+                Log.Warning(ex, "Failed to update User Info Blob for: {Username:l}", userInfo.Username);
+            }
+        }
+    }
+}

Cajetan.OF-DL/Services/ICajetanApiService.cs

@@ -0,0 +1,9 @@
+namespace OF_DL.Services;
+
+public interface ICajetanApiService : IApiService
+{
+    Task<UserEntities.UserInfo?> GetDetailedUserInfoAsync(string endpoint);
+    Task<Dictionary<string, long>> GetUsersWithProgressAsync(string typeDisplay, string endpoint, string? typeParam, bool offsetByCount);
+    Task<HashSet<long>> GetUsersWithUnreadMessagesAsync();
+    Task MarkAsUnreadAsync(string endpoint);
+}

Cajetan.OF-DL/Services/ICajetanDbService.cs

@@ -0,0 +1,9 @@
+namespace OF_DL.Services;
+
+public interface ICajetanDbService : IDbService
+{
+    Task InitializeUserInfoTablesAsync();
+
+    Task<Dictionary<string, long>> GetUsersAsync();
+    Task UpdateUserInfoAsync(UserEntities.UserInfo? userInfo);
+}

Cajetan.OF-DL/Worker.cs

@@ -1,6 +1,4 @@
 using Microsoft.Extensions.DependencyInjection;
-using OF_DL.Models.Downloads;
-using OF_DL.Models.Entities.Users;
 
 namespace OF_DL;
 
@@ -10,8 +8,8 @@ internal class Worker(IServiceProvider serviceProvider)
     private readonly IAuthService _authService = serviceProvider.GetRequiredService<IAuthService>();
     private readonly IStartupService _startupService = serviceProvider.GetRequiredService<IStartupService>();
     private readonly IDownloadOrchestrationService _orchestrationService = serviceProvider.GetRequiredService<IDownloadOrchestrationService>();
-    private readonly IDbService _dbService = serviceProvider.GetRequiredService<IDbService>();
+    private readonly ICajetanDbService _dbService = serviceProvider.GetRequiredService<ICajetanDbService>();
-    private readonly IApiService _apiService = serviceProvider.GetRequiredService<IApiService>();
+    private readonly ICajetanApiService _apiService = serviceProvider.GetRequiredService<ICajetanApiService>();
     private readonly ExitHelper _exitHelper = serviceProvider.GetRequiredService<ExitHelper>();
 
     private readonly CajetanConfig _cajetanConfig = serviceProvider.GetRequiredService<CajetanConfig>();
@@ -115,7 +113,7 @@ internal class Worker(IServiceProvider serviceProvider)
             // Validate cookie string
             _authService.ValidateCookieString();
 
-            User? user = await _authService.ValidateAuthAsync();
+            UserEntities.User? user = await _authService.ValidateAuthAsync();
             if (user is null || (user.Name is null && user.Username is null))
             {
                 Log.Error("Auth failed");
@@ -288,6 +286,7 @@ internal class Worker(IServiceProvider serviceProvider)
         AnsiConsole.WriteLine();
 
         await _dbService.CreateUsersDb(result.Users);
+        await _dbService.InitializeUserInfoTablesAsync();
 
         return result;