Enabled updating of User Info during scrape

Cajetan.OF-DL/ProgramCajetan.cs

@@ -1,3 +1,5 @@
+using System.Diagnostics;
+using System.Reflection;
 using Microsoft.Extensions.DependencyInjection;
 
 AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
@@ -5,6 +7,8 @@ AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
 ServiceCollection services = await ConfigureServices(args);
 ServiceProvider serviceProvider = services.BuildServiceProvider();
 
+ExitIfOtherProcess(serviceProvider);
+
 Worker worker = serviceProvider.GetRequiredService<Worker>();
 await worker.RunAsync();
 
@@ -58,8 +62,13 @@ static async Task<ServiceCollection> ConfigureServices(string[] args)
     services.AddSingleton(cajetanConfig);
 
     services.AddSingleton<IAuthService, AuthService>();
-    services.AddSingleton<IApiService, ApiService>();
-    services.AddSingleton<IDbService, DbService>();
+
+    services.AddSingleton<ICajetanApiService, CajetanApiService>();
+    services.AddSingleton<IApiService>(sp => sp.GetRequiredService<ICajetanApiService>());
+
+    services.AddSingleton<ICajetanDbService, CajetanDbService>();
+    services.AddSingleton<IDbService>(sp => sp.GetRequiredService<ICajetanDbService>());
+
     services.AddSingleton<IDownloadService, DownloadService>();
     services.AddSingleton<IFileNameService, FileNameService>();
     services.AddSingleton<IStartupService, StartupService>();
@@ -149,3 +158,23 @@ static bool ParseCommandlineArgs(string[] args, Config currentConfig, out Cajeta
     }
 }
 
+static void ExitIfOtherProcess(ServiceProvider serviceProvider)
+{
+    Assembly? entryAssembly = Assembly.GetEntryAssembly();
+    AssemblyName? entryAssemblyName = entryAssembly?.GetName();
+
+    if (entryAssemblyName?.Name is null)
+        return;
+
+    Process thisProcess = Process.GetCurrentProcess();
+    Process[] otherProcesses = [.. Process.GetProcessesByName(entryAssemblyName.Name).Where(p => p.Id != thisProcess.Id)];
+
+    if (otherProcesses.Length <= 0)
+        return;
+
+    AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]");
+    Log.Warning("Other OF DL process detected, exiting..");
+
+    serviceProvider.GetRequiredService<ExitHelper>().ExitWithCode(0);
+}
+

Cajetan.OF-DL/Services/CajetanApiService.cs

@@ -0,0 +1,103 @@
+using Newtonsoft.Json;
+using UserDtos = OF_DL.Models.Dtos.Users;
+using UserEntities = OF_DL.Models.Entities.Users;
+
+namespace OF_DL.Services;
+
+public interface ICajetanApiService : IApiService
+{
+    Task<UserEntities.UserInfo?> GetDetailedUserInfo(string endpoint);
+}
+
+public class CajetanApiService(IAuthService authService, IConfigService configService, ICajetanDbService dbService)
+    : ApiService(authService, configService, dbService), ICajetanApiService
+{
+    public new async Task<UserEntities.User?> GetUserInfo(string endpoint)
+    {
+        UserEntities.UserInfo? userInfo = await GetDetailedUserInfo(endpoint);
+
+        if (userInfo is not null && !endpoint.EndsWith("/me"))
+            await dbService.UpdateUserInfoAsync(userInfo);
+
+        return userInfo;
+    }
+
+    /// <summary>
+    /// Retrieves detailed user information from the API.
+    /// </summary>
+    /// <param name="endpoint">The user endpoint.</param>
+    /// <returns>The user entity when available.</returns>
+    public async Task<UserEntities.UserInfo?> GetDetailedUserInfo(string endpoint)
+    {
+        Log.Debug($"Calling GetDetailedUserInfo: {endpoint}");
+
+        if (!HasSignedRequestAuth())
+            return null;
+
+        try
+        {
+            UserEntities.UserInfo userInfo = new();
+            Dictionary<string, string> getParams = new()
+            {
+                { "limit", Constants.ApiPageSize.ToString() }, { "order", "publish_date_asc" }
+            };
+
+            HttpClient client = new();
+            HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
+
+            using HttpResponseMessage response = await client.SendAsync(request);
+
+            if (!response.IsSuccessStatusCode)
+                return userInfo;
+
+            response.EnsureSuccessStatusCode();
+            string body = await response.Content.ReadAsStringAsync();
+            UserDtos.UserDto? userDto = JsonConvert.DeserializeObject<UserDtos.UserDto>(body, s_mJsonSerializerSettings);
+            userInfo = FromDto(userDto);
+            return userInfo;
+        }
+        catch (Exception ex)
+        {
+            ExceptionLoggerHelper.LogException(ex);
+        }
+
+        return null;
+    }
+
+    private static UserEntities.UserInfo FromDto(UserDtos.UserDto? userDto)
+    {
+        if (userDto is null)
+            return new();
+
+        return new()
+        {
+            Id = userDto.Id,
+
+            Avatar = userDto.Avatar,
+            Header = userDto.Header,
+            Name = userDto.Name,
+            Username = userDto.Username,
+
+            SubscribePrice = userDto.SubscribePrice,
+            CurrentSubscribePrice = userDto.CurrentSubscribePrice,
+            IsPaywallRequired = userDto.IsPaywallRequired,
+            IsRestricted = userDto.IsRestricted,
+            SubscribedBy = userDto.SubscribedBy,
+            SubscribedByExpire = userDto.SubscribedByExpire,
+            SubscribedByExpireDate = userDto.SubscribedByExpireDate,
+            SubscribedByAutoprolong = userDto.SubscribedByAutoprolong,
+            SubscribedIsExpiredNow = userDto.SubscribedIsExpiredNow,
+            SubscribedOn = userDto.SubscribedOn,
+            SubscribedOnExpiredNow = userDto.SubscribedOnExpiredNow,
+            SubscribedOnDuration = userDto.SubscribedOnDuration,
+            About = userDto.About,
+            PostsCount = userDto.PostsCount,
+            ArchivedPostsCount = userDto.ArchivedPostsCount,
+            PrivateArchivedPostsCount = userDto.PrivateArchivedPostsCount,
+            PhotosCount = userDto.PhotosCount,
+            VideosCount = userDto.VideosCount,
+            AudiosCount = userDto.AudiosCount,
+            MediasCount = userDto.MediasCount,
+        };
+    }
+}

Cajetan.OF-DL/Services/CajetanDbService.cs

@@ -0,0 +1,107 @@
+using Microsoft.Data.Sqlite;
+using OF_DL.Models.Entities.Users;
+
+namespace OF_DL.Services;
+
+public class CajetanDbService(IConfigService configService)
+    : DbService(configService), ICajetanDbService
+{
+    public async Task InitializeUserInfoTablesAsync()
+    {
+        await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+
+
+        using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, about VARCHAR NULL, expires_on TIMESTAMP NULL, photo_count INT NOT NULL, video_count INT NOT NULL, PRIMARY KEY(user_id));", connection))
+        {
+            await cmdInfo.ExecuteNonQueryAsync();
+        }
+
+        using (SqliteCommand cmdInfo = new("CREATE TABLE IF NOT EXISTS user_info_blob (user_id INTEGER NOT NULL, name VARCHAR NOT NULL, blob TEXT NULL, PRIMARY KEY(user_id));", connection))
+        {
+            await cmdInfo.ExecuteNonQueryAsync();
+        }
+
+    }
+
+    public async Task<Dictionary<string, long>> GetUsersAsync()
+    {
+        await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+
+        using SqliteCommand cmd = new("SELECT user_id, username FROM users", connection);
+        using SqliteDataReader reader = cmd.ExecuteReader();
+
+        Dictionary<string, long> result = new(StringComparer.OrdinalIgnoreCase);
+
+        while (reader.Read())
+        {
+            long userId = reader.GetInt64(0);
+            string username = reader.GetString(1);
+
+            result[username] = userId;
+        }
+
+        return result;
+    }
+
+    public async Task UpdateUserInfoAsync(UserInfo? userInfo)
+    {
+        if (userInfo?.Id is null || userInfo?.Username is null)
+            return;
+
+        await using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
+
+        Log.Debug("Database data source: " + connection.DataSource);
+
+        await UpdateAsync();
+        await UpdateBlobAsync();
+
+        async Task UpdateAsync()
+        {
+            using SqliteCommand cmdInfo = new(
+                "INSERT OR REPLACE INTO user_info (user_id, name, about, expires_on, photo_count, video_count) " +
+                "VALUES (@userId, @name, @about, @expiresOn, @photoCount, @videoCount);",
+                connection
+            );
+
+            cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
+            cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
+            cmdInfo.Parameters.AddWithValue("@about", userInfo.About);
+            cmdInfo.Parameters.AddWithValue("@expiresOn", userInfo.SubscribedByExpireDate);
+            cmdInfo.Parameters.AddWithValue("@photoCount", userInfo.PhotosCount ?? 0);
+            cmdInfo.Parameters.AddWithValue("@videoCount", userInfo.VideosCount ?? 0);
+
+            try
+            {
+                await cmdInfo.ExecuteNonQueryAsync();
+                Log.Debug("Inserted or updated creator info: {Username:l}", userInfo.Username);
+            }
+            catch (Exception ex)
+            {
+                Log.Warning(ex, "Failed to update User Info for: {Username:l}", userInfo.Username);
+            }
+        }
+
+        async Task UpdateBlobAsync()
+        {
+            using SqliteCommand cmdInfo = new(
+                "INSERT OR REPLACE INTO user_info_blob (user_id, name, blob) " +
+                "VALUES (@userId, @name, @blob);",
+                connection
+            );
+
+            cmdInfo.Parameters.AddWithValue("@userId", userInfo.Id);
+            cmdInfo.Parameters.AddWithValue("@name", userInfo.Name ?? userInfo.Username);
+            cmdInfo.Parameters.AddWithValue("@blob", Newtonsoft.Json.JsonConvert.SerializeObject(userInfo));
+
+            try
+            {
+                await cmdInfo.ExecuteNonQueryAsync();
+                Log.Debug("Inserted or updated creator blob: {Username:l}", userInfo.Username);
+            }
+            catch (Exception ex)
+            {
+                Log.Warning(ex, "Failed to update User Info Blob for: {Username:l}", userInfo.Username);
+            }
+        }
+    }
+}

Cajetan.OF-DL/Services/ICajetanDbService.cs

@@ -0,0 +1,12 @@
+
+using OF_DL.Models.Entities.Users;
+
+namespace OF_DL.Services;
+
+public interface ICajetanDbService : IDbService
+{
+    Task InitializeUserInfoTablesAsync();
+
+    Task<Dictionary<string, long>> GetUsersAsync();
+    Task UpdateUserInfoAsync(UserInfo? userInfo);
+}

Cajetan.OF-DL/Worker.cs

@@ -10,8 +10,8 @@ internal class Worker(IServiceProvider serviceProvider)
     private readonly IAuthService _authService = serviceProvider.GetRequiredService<IAuthService>();
     private readonly IStartupService _startupService = serviceProvider.GetRequiredService<IStartupService>();
     private readonly IDownloadOrchestrationService _orchestrationService = serviceProvider.GetRequiredService<IDownloadOrchestrationService>();
-    private readonly IDbService _dbService = serviceProvider.GetRequiredService<IDbService>();
-    private readonly IApiService _apiService = serviceProvider.GetRequiredService<IApiService>();
+    private readonly ICajetanDbService _dbService = serviceProvider.GetRequiredService<ICajetanDbService>();
+    private readonly ICajetanApiService _apiService = serviceProvider.GetRequiredService<ICajetanApiService>();
     private readonly ExitHelper _exitHelper = serviceProvider.GetRequiredService<ExitHelper>();
 
     private readonly CajetanConfig _cajetanConfig = serviceProvider.GetRequiredService<CajetanConfig>();
@@ -177,20 +177,21 @@ internal class Worker(IServiceProvider serviceProvider)
                     devicePrivateKeyMissing: _devicePrivateKeyMissing,
                     eventHandler: eventHandler
                 );
+                CreatorDownloadResult newResults = results.NewDownloads ?? results;
 
-                totalResults.Add(results);
+                totalResults.Add(newResults);
 
                 DateTime userEndTime = DateTime.Now;
                 TimeSpan userTotalTime = userEndTime - userStartTime;
 
-                Log.ForContext("Paid Posts", results.PaidPostCount)
-                   .ForContext("Posts", results.PostCount)
-                   .ForContext("Archived", results.ArchivedCount)
-                   .ForContext("Streams", results.StreamsCount)
-                   .ForContext("Stories", results.StoriesCount)
-                   .ForContext("Highlights", results.HighlightsCount)
-                   .ForContext("Messages", results.MessagesCount)
-                   .ForContext("Paid Messages", results.PaidMessagesCount)
+                Log.ForContext("Paid Posts", newResults.PaidPostCount)
+                   .ForContext("Posts", newResults.PostCount)
+                   .ForContext("Archived", newResults.ArchivedCount)
+                   .ForContext("Streams", newResults.StreamsCount)
+                   .ForContext("Stories", newResults.StoriesCount)
+                   .ForContext("Highlights", newResults.HighlightsCount)
+                   .ForContext("Messages", newResults.MessagesCount)
+                   .ForContext("Paid Messages", newResults.PaidMessagesCount)
                    .ForContext("Username", username)
                    .ForContext("TotalMinutes", userTotalTime.TotalMinutes)
                    .Information("Scraped Data for '{Username:l}', took {TotalMinutes:0.000} minutes");
@@ -220,6 +221,8 @@ internal class Worker(IServiceProvider serviceProvider)
            .ForContext("Paid Messages", totalResults.PaidMessagesCount)
            .ForContext("TotalMinutes", totalTime.TotalMinutes)
            .Information("Scrape Completed in {TotalMinutes:0.00} minutes");
+
+        await Task.Delay(2000);
     }
 
     private async Task OutputBlockedUsersAsync()
@@ -285,6 +288,7 @@ internal class Worker(IServiceProvider serviceProvider)
         AnsiConsole.WriteLine();
 
         await _dbService.CreateUsersDb(result.Users);
+        await _dbService.InitializeUserInfoTablesAsync();
 
         return result;
 

OF DL.Core/Models/Downloads/CreatorDownloadResult.cs

@@ -17,6 +17,8 @@ public class CreatorDownloadResult
     public int MessagesCount { get; set; }
 
     public int PaidMessagesCount { get; set; }
+
+    public CreatorDownloadResult? NewDownloads { get; set; }
 }
 
 public class UserListResult

OF DL.Core/Models/Entities/Users/UserInfo.cs

@@ -0,0 +1,36 @@
+namespace OF_DL.Models.Entities.Users;
+
+public class UserInfo : User
+{
+    public long? Id { get; set; }
+
+    public string? SubscribePrice { get; set; }
+    public string? CurrentSubscribePrice { get; set; }
+
+    public bool? IsPaywallRequired { get; set; }
+    public bool? IsActive { get; set; }
+    public bool? IsRestricted { get; set; }
+
+    public bool? SubscribedBy { get; set; }
+    public bool? SubscribedByExpire { get; set; }
+    public DateTimeOffset? SubscribedByExpireDate { get; set; }
+    public bool? SubscribedByAutoprolong { get; set; }
+    public bool? IsPendingAutoprolong { get; set; }
+
+    public bool? SubscribedIsExpiredNow { get; set; }
+
+    public bool? SubscribedOn { get; set; }
+    public bool? SubscribedOnExpiredNow { get; set; }
+    public string? SubscribedOnDuration { get; set; }
+
+    public string? About { get; set; }
+
+    public int? PostsCount { get; set; }
+    public int? ArchivedPostsCount { get; set; }
+    public int? PrivateArchivedPostsCount { get; set; }
+
+    public int? PhotosCount { get; set; }
+    public int? VideosCount { get; set; }
+    public int? AudiosCount { get; set; }
+    public int? MediasCount { get; set; }
+}

OF DL.Core/OF DL.Core.csproj

@@ -23,4 +23,8 @@
     <PackageReference Include="xFFmpeg.NET" Version="7.2.0"/>
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="Cajetan.OF-DL" />
+  </ItemGroup>
+
 </Project>

OF DL.Core/Services/ApiService.cs

@@ -43,7 +43,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
 {
     private const int MaxAttempts = 30;
     private const int DelayBetweenAttempts = 3000;
-    private static readonly JsonSerializerSettings s_mJsonSerializerSettings;
+    protected static readonly JsonSerializerSettings s_mJsonSerializerSettings;
     private static DateTime? s_cachedDynamicRulesExpiration;
     private static DynamicRules? s_cachedDynamicRules;
 
@@ -147,7 +147,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
         return headers;
     }
 
-    private bool HasSignedRequestAuth()
+    protected bool HasSignedRequestAuth()
     {
         Auth? currentAuth = authService.CurrentAuth;
         return currentAuth is { UserId: not null, Cookie: not null, UserAgent: not null, XBc: not null };
@@ -2788,7 +2788,7 @@ public class ApiService(IAuthService authService, IConfigService configService,
     }
 
 
-    private Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
+    protected Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams,
         string endpoint)
     {
         Log.Debug("Calling BuildHttpRequestMessage");

OF DL.Core/Services/DownloadOrchestrationService.cs

@@ -160,6 +160,7 @@ public class DownloadOrchestrationService(
     {
         Config config = configService.CurrentConfig;
         CreatorDownloadResult counts = new();
+        CreatorDownloadResult newCounts = new();
 
         eventHandler.OnUserStarting(username);
         Log.Debug($"Scraping Data for {username}");
@@ -185,7 +186,8 @@ public class DownloadOrchestrationService(
                 posts => posts.PaidPosts.Values.ToList(),
                 async (posts, reporter) => await downloadService.DownloadPaidPosts(username, userId, path, users,
                     clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
-                eventHandler);
+                eventHandler,
+                n => newCounts.PaidPostCount = n);
         }
 
         if (config.DownloadPosts)
@@ -202,7 +204,8 @@ public class DownloadOrchestrationService(
                 posts => posts.Posts.Values.ToList(),
                 async (posts, reporter) => await downloadService.DownloadFreePosts(username, userId, path, users,
                     clientIdBlobMissing, devicePrivateKeyMissing, posts, reporter),
-                eventHandler);
+                eventHandler,
+                n => newCounts.PostCount = n);
         }
 
         if (config.DownloadArchived)
@@ -215,7 +218,8 @@ public class DownloadOrchestrationService(
                 archived => archived.ArchivedPosts.Values.ToList(),
                 async (archived, reporter) => await downloadService.DownloadArchived(username, userId, path, users,
                     clientIdBlobMissing, devicePrivateKeyMissing, archived, reporter),
-                eventHandler);
+                eventHandler,
+                n => newCounts.ArchivedCount = n);
         }
 
         if (config.DownloadStreams)
@@ -228,7 +232,8 @@ public class DownloadOrchestrationService(
                 streams => streams.Streams.Values.ToList(),
                 async (streams, reporter) => await downloadService.DownloadStreams(username, userId, path, users,
                     clientIdBlobMissing, devicePrivateKeyMissing, streams, reporter),
-                eventHandler);
+                eventHandler,
+                n => newCounts.StreamsCount = n);
         }
 
         if (config.DownloadStories)
@@ -252,6 +257,7 @@ public class DownloadOrchestrationService(
 
                 eventHandler.OnDownloadComplete("Stories", result);
                 counts.StoriesCount = result.TotalCount;
+                newCounts.StoriesCount = result.NewDownloads;
             }
             else
             {
@@ -280,6 +286,7 @@ public class DownloadOrchestrationService(
 
                 eventHandler.OnDownloadComplete("Highlights", result);
                 counts.HighlightsCount = result.TotalCount;
+                newCounts.HighlightsCount = result.NewDownloads;
             }
             else
             {
@@ -297,7 +304,8 @@ public class DownloadOrchestrationService(
                 messages => messages.Messages.Values.ToList(),
                 async (messages, reporter) => await downloadService.DownloadMessages(username, userId, path, users,
                     clientIdBlobMissing, devicePrivateKeyMissing, messages, reporter),
-                eventHandler);
+                eventHandler,
+                n => newCounts.MessagesCount = n);
         }
 
         if (config.DownloadPaidMessages)
@@ -310,10 +318,13 @@ public class DownloadOrchestrationService(
                 paidMessages => paidMessages.PaidMessages.Values.ToList(),
                 async (paidMessages, reporter) => await downloadService.DownloadPaidMessages(username, path, users,
                     clientIdBlobMissing, devicePrivateKeyMissing, paidMessages, reporter),
-                eventHandler);
+                eventHandler,
+                n => newCounts.PaidMessagesCount = n);
         }
 
         eventHandler.OnUserComplete(username, counts);
+
+        counts.NewDownloads = newCounts;
         return counts;
     }
 
@@ -612,7 +623,8 @@ public class DownloadOrchestrationService(
         Func<T, int> getObjectCount,
         Func<T, List<string>?> getUrls,
         Func<T, IProgressReporter, Task<DownloadResult>> downloadData,
-        IDownloadEventHandler eventHandler)
+        IDownloadEventHandler eventHandler,
+        Action<int>? newPostAssignmentAction)
     {
         T data = await eventHandler.WithStatusAsync($"Getting {contentType}",
             async statusReporter => await fetchData(statusReporter));
@@ -643,6 +655,8 @@ public class DownloadOrchestrationService(
         Log.Debug(
             $"{contentType} Already Downloaded: {result.ExistingDownloads} New {contentType} Downloaded: {result.NewDownloads}");
 
+        newPostAssignmentAction?.Invoke(result.NewDownloads);
+
         return result.TotalCount;
     }
 }

OF DL/OF DL.csproj

@@ -51,4 +51,8 @@
     </None>
   </ItemGroup>
 
+  <ItemGroup>
+    <InternalsVisibleTo Include="Cajetan.OF-DL" />
+  </ItemGroup>
+  
 </Project>