Enabled updating of User Info during scrape

Cajetan.OF-DL/ProgramCajetan.cs

@@ -62,8 +62,13 @@ static async Task<ServiceCollection> ConfigureServices(string[] args)
     services.AddSingleton(cajetanConfig);
 
     services.AddSingleton<IAuthService, AuthService>();
-    services.AddSingleton<IApiService, ApiService>();
+
-    services.AddSingleton<IDbService, DbService>();
+    services.AddSingleton<ICajetanApiService, CajetanApiService>();
+    services.AddSingleton<IApiService>(sp => sp.GetRequiredService<ICajetanApiService>());
+
+    services.AddSingleton<ICajetanDbService, CajetanDbService>();
+    services.AddSingleton<IDbService>(sp => sp.GetRequiredService<ICajetanDbService>());
+
     services.AddSingleton<IDownloadService, DownloadService>();
     services.AddSingleton<IFileNameService, FileNameService>();
     services.AddSingleton<IStartupService, StartupService>();

Cajetan.OF-DL/Services/CajetanApiService.cs

@@ -0,0 +1,103 @@
+using Newtonsoft.Json;
+using UserDtos = OF_DL.Models.Dtos.Users;
+using UserEntities = OF_DL.Models.Entities.Users;
+
+namespace OF_DL.Services;
+
+public interface ICajetanApiService : IApiService
+{
+    Task<UserEntities.UserInfo?> GetDetailedUserInfo(string endpoint);
+}
+
+public class CajetanApiService(IAuthService authService, IConfigService configService, ICajetanDbService dbService)
+    : ApiService(authService, configService, dbService), ICajetanApiService
+{
+    public new async Task<UserEntities.User?> GetUserInfo(string endpoint)
+    {
+        UserEntities.UserInfo? userInfo = await GetDetailedUserInfo(endpoint);
+
+        if (userInfo is not null && !endpoint.EndsWith("/me"))
+            await dbService.UpdateUserInfoAsync(userInfo);
+
+        return userInfo;
+    }
+
+    /// <summary>
+    /// Retrieves detailed user information from the API.
+    /// </summary>
+    /// <param name="endpoint">The user endpoint.</param>
+    /// <returns>The user entity when available.</returns>
+    public async Task<UserEntities.UserInfo?> GetDetailedUserInfo(string endpoint)
+    {
+        Log.Debug($"Calling GetDetailedUserInfo: {endpoint}");
+
+        if (!HasSignedRequestAuth())
+            return null;
+
+        try
+        {
+            UserEntities.UserInfo userInfo = new();
+            Dictionary<string, string> getParams = new()
+            {
+                { "limit", Constants.ApiPageSize.ToString() }, { "order", "publish_date_asc" }
+            };
+
+            HttpClient client = new();
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
+
+            using HttpResponseMessage response = await client.SendAsync(request);
+
+            if (!response.IsSuccessStatusCode)
+                return userInfo;
+
+            response.EnsureSuccessStatusCode();
+            string body = await response.Content.ReadAsStringAsync();
+            UserDtos.UserDto? userDto = JsonConvert.DeserializeObject<UserDtos.UserDto>(body, s_mJsonSerializerSettings);
+            userInfo = FromDto(userDto);
+            return userInfo;
+        }
+        catch (Exception ex)
+        {
+            ExceptionLoggerHelper.LogException(ex);
+        }
+
+        return null;
+    }
+
+    private static UserEntities.UserInfo FromDto(UserDtos.UserDto? userDto)
+    {
+        if (userDto is null)
+            return new();
+
+        return new()
+        {
+            Id = userDto.Id,
+
+            Avatar = userDto.Avatar,
+            Header = userDto.Header,
+            Name = userDto.Name,
+            Username = userDto.Username,
+
+            SubscribePrice = userDto.SubscribePrice,
+            CurrentSubscribePrice = userDto.CurrentSubscribePrice,
+            IsPaywallRequired = userDto.IsPaywallRequired,
+            IsRestricted = userDto.IsRestricted,
+            SubscribedBy = userDto.SubscribedBy,
+            SubscribedByExpire = userDto.SubscribedByExpire,
+            SubscribedByExpireDate = userDto.SubscribedByExpireDate,
+            SubscribedByAutoprolong = userDto.SubscribedByAutoprolong,
+            SubscribedIsExpiredNow = userDto.SubscribedIsExpiredNow,
+            SubscribedOn = userDto.SubscribedOn,
+            SubscribedOnExpiredNow = userDto.SubscribedOnExpiredNow,
+            SubscribedOnDuration = userDto.SubscribedOnDuration,
+            About = userDto.About,
+            PostsCount = userDto.PostsCount,
+            ArchivedPostsCount = userDto.ArchivedPostsCount,
+            PrivateArchivedPostsCount = userDto.PrivateArchivedPostsCount,
+            PhotosCount = userDto.PhotosCount,
+            VideosCount = userDto.VideosCount,
+            AudiosCount = userDto.AudiosCount,
+            MediasCount = userDto.MediasCount,
+        };
+    }
+}

Cajetan.OF-DL/Services/CajetanDbService.cs

@@ -0,0 +1,107 @@
+using Microsoft.Data.Sqlite;
+using OF_DL.Models.Entities.Users;
+
+namespace OF_DL.Services;
+
+public class CajetanDbService(IConfigService configService)
+    : DbService(configService), ICajetanDbService
+{
+    public async Task InitializeUserInfoTablesAsync()
+    {
+        await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+
+
+        using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, about VARCHAR NULL, expires_on TIMESTAMP NULL, photo_count INT NOT NULL, video_count INT NOT NULL, PRIMARY KEY(user_id));", connection))
+        {
+            await cmdInfo.ExecuteNonQueryAsync();
+        }
+
+        using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info_blob (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, blob TEXT NULL, PRIMARY KEY(user_id));", connection))
+        {
+            await cmdInfo.ExecuteNonQueryAsync();
+        }
+
+    }
+
+    public async Task<Dictionary<string, long>> GetUsersAsync()
+    {
+        await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+
+        using SqliteCommand cmd = new("SELECT user_id, username FROM users", connection);
+        using SqliteDataReader reader = cmd.ExecuteReader();
+
+        Dictionary<string, long> result = new(StringComparer.OrdinalIgnoreCase);
+
+        while (reader.Read())
+        {
+            long userId = reader.GetInt64(0);
+            string username = reader.GetString(1);
+
+            result[username] = userId;
+        }
+
+        return result;
+    }
+
+    public async Task UpdateUserInfoAsync(UserInfo? userInfo)
+    {
+        if (userInfo?.Id is null || userInfo?.Username is null)
+            return;
+
+        await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+
+        Log.Debug("Database data source: " + connection.DataSource);
+
+        await UpdateAsync();
+        await UpdateBlobAsync();
+
+        async Task UpdateAsync()
+        {
+            using SqliteCommand cmdInfo = new(
+                "INSERT OR REPLACE INTO user_info (user_id, name, about, expires_on, photo_count, video_count) " +
+                "VALUES (@userId, @name, @about, @expiresOn, @photoCount, @videoCount);",
+                connection
+            );
+
+            cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
+            cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
+            cmdInfo.Parameters.AddWithValue("@about", userInfo.About);
+            cmdInfo.Parameters.AddWithValue("@expiresOn", userInfo.SubscribedByExpireDate);
+            cmdInfo.Parameters.AddWithValue("@photoCount", userInfo.PhotosCount ?? 0);
+            cmdInfo.Parameters.AddWithValue("@videoCount", userInfo.VideosCount ?? 0);
+
+            try
+            {
+                await cmdInfo.ExecuteNonQueryAsync();
+                Log.Debug("Inserted or updated creator info: {Username:l}", userInfo.Username);
+            }
+            catch (Exception ex)
+            {
+                Log.Warning(ex, "Failed to update User Info for: {Username:l}", userInfo.Username);
+            }
+        }
+
+        async Task UpdateBlobAsync()
+        {
+            using SqliteCommand cmdInfo = new(
+                "INSERT OR REPLACE INTO user_info_blob (user_id, name, blob) " +
+                "VALUES (@userId, @name, @blob);",
+                connection
+            );
+
+            cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
+            cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
+            cmdInfo.Parameters.AddWithValue("@blob", Newtonsoft.Json.JsonConvert.SerializeObject(userInfo));
+
+            try
+            {
+                await cmdInfo.ExecuteNonQueryAsync();
+                Log.Debug("Inserted or updated creator blob: {Username:l}", userInfo.Username);
+            }
+            catch (Exception ex)
+            {
+                Log.Warning(ex, "Failed to update User Info Blob for: {Username:l}", userInfo.Username);
+            }
+        }
+    }
+}

Cajetan.OF-DL/Services/ICajetanDbService.cs

@@ -0,0 +1,12 @@
+
+using OF_DL.Models.Entities.Users;
+
+namespace OF_DL.Services;
+
+public interface ICajetanDbService : IDbService
+{
+    Task InitializeUserInfoTablesAsync();
+
+    Task<Dictionary<string, long>> GetUsersAsync();
+    Task UpdateUserInfoAsync(UserInfo? userInfo);
+}

Cajetan.OF-DL/Worker.cs

@@ -10,8 +10,8 @@ internal class Worker(IServiceProvider serviceProvider)
     private readonly IAuthService _authService = serviceProvider.GetRequiredService<IAuthService>();
     private readonly IStartupService _startupService = serviceProvider.GetRequiredService<IStartupService>();
     private readonly IDownloadOrchestrationService _orchestrationService = serviceProvider.GetRequiredService<IDownloadOrchestrationService>();
-    private readonly IDbService _dbService = serviceProvider.GetRequiredService<IDbService>();
+    private readonly ICajetanDbService _dbService = serviceProvider.GetRequiredService<ICajetanDbService>();
-    private readonly IApiService _apiService = serviceProvider.GetRequiredService<IApiService>();
+    private readonly ICajetanApiService _apiService = serviceProvider.GetRequiredService<ICajetanApiService>();
     private readonly ExitHelper _exitHelper = serviceProvider.GetRequiredService<ExitHelper>();
 
     private readonly CajetanConfig _cajetanConfig = serviceProvider.GetRequiredService<CajetanConfig>();
@@ -285,6 +285,7 @@ internal class Worker(IServiceProvider serviceProvider)
         AnsiConsole.WriteLine();
 
         await _dbService.CreateUsersDb(result.Users);
+        await _dbService.InitializeUserInfoTablesAsync();
 
         return result;
 

OF DL.Core/Models/Entities/Users/UserInfo.cs

@@ -0,0 +1,36 @@
+namespace OF_DL.Models.Entities.Users;
+
+public class UserInfo : User
+{
+    public long? Id { get; set; }
+
+    public string? SubscribePrice { get; set; }
+    public string? CurrentSubscribePrice { get; set; }
+
+    public bool? IsPaywallRequired { get; set; }
+    public bool? IsActive { get; set; }
+    public bool? IsRestricted { get; set; }
+
+    public bool? SubscribedBy { get; set; }
+    public bool? SubscribedByExpire { get; set; }
+    public DateTimeOffset? SubscribedByExpireDate { get; set; }
+    public bool? SubscribedByAutoprolong { get; set; }
+    public bool? IsPendingAutoprolong { get; set; }
+
+    public bool? SubscribedIsExpiredNow { get; set; }
+
+    public bool? SubscribedOn { get; set; }
+    public bool? SubscribedOnExpiredNow { get; set; }
+    public string? SubscribedOnDuration { get; set; }
+
+    public string? About { get; set; }
+
+    public int? PostsCount { get; set; }
+    public int? ArchivedPostsCount { get; set; }
+    public int? PrivateArchivedPostsCount { get; set; }
+
+    public int? PhotosCount { get; set; }
+    public int? VideosCount { get; set; }
+    public int? AudiosCount { get; set; }
+    public int? MediasCount { get; set; }
+}

OF DL.Core/OF DL.Core.csproj

@@ -23,4 +23,8 @@
     <PackageReference Include="xFFmpeg.NET" Version="7.2.0"/>
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="Cajetan.OF-DL" />
+  </ItemGroup>
+
 </Project>

OF DL.Core/Services/ApiService.cs

@@ -43,7 +43,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
 {
     private const int MaxAttempts = 30;
     private const int DelayBetweenAttempts = 3000;
-    private static readonly JsonSerializerSettings s_mJsonSerializerSettings;
+    protected static readonly JsonSerializerSettings s_mJsonSerializerSettings;
     private static DateTime? s_cachedDynamicRulesExpiration;
     private static DynamicRules? s_cachedDynamicRules;
 
@@ -147,7 +147,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
         return headers;
     }
 
-    private bool HasSignedRequestAuth()
+    protected bool HasSignedRequestAuth()
     {
         Auth? currentAuth = authService.CurrentAuth;
         return currentAuth is { UserId: not null, Cookie: not null, UserAgent: not null, XBc: not null };
@@ -2788,7 +2788,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
     }
 
 
-    private Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
+    protected Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
         string endpoint)
     {
         Log.Debug("Calling BuildHttpRequestMessage");

OF DL/OF DL.csproj

@@ -51,4 +51,8 @@
     </None>
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="Cajetan.OF-DL" />
+  </ItemGroup>
+  
 </Project>