reptail/OF-DL
1
0
Fork 0
forked from sim0n00ps/OF-DL
Code Pull Requests Activity

Compare commits

base: reptail:dev
Branches Tags
reptail:master
reptail:dev
reptail:improvements
sim0n00ps:master
reptail:OFDLV1.9.9
reptail:OFDLV1.9.8
reptail:OFDLV1.9.7
sim0n00ps:OFDLV1.9.12
sim0n00ps:OFDLV1.9.11
sim0n00ps:OFDLV1.9.10
sim0n00ps:OFDLV1.9.9
sim0n00ps:OFDLV1.9.8
sim0n00ps:OFDLV1.9.7
..
compare: sim0n00ps:master
Branches Tags
sim0n00ps:master
reptail:master
reptail:dev
reptail:improvements
sim0n00ps:OFDLV1.9.12
sim0n00ps:OFDLV1.9.11
sim0n00ps:OFDLV1.9.10
sim0n00ps:OFDLV1.9.9
sim0n00ps:OFDLV1.9.8
sim0n00ps:OFDLV1.9.7
reptail:OFDLV1.9.9
reptail:OFDLV1.9.8
reptail:OFDLV1.9.7

No commits in common. "dev" and "master" have entirely different histories.
dev ... master

14 changed files with 186 additions and 626 deletions
Show Stats Expand all files Collapse all files

7
.editorconfig
View File

@ -5,12 +5,5 @@ root = true
charset = utf-8 charset = utf-8
indent_style = space indent_style = space
indent_size = 4 indent_size = 4
tab_width = 4
end_of_line = crlf
insert_final_newline = true insert_final_newline = true
trim_trailing_whitespace = true trim_trailing_whitespace = true
[*.{sln,csproj,xml,json,config}]
indent_size = 2
indent_style = space
tab_width = 2

2
.gitignore vendored
View File

@ -371,5 +371,3 @@ FodyWeavers.xsd
# venv # venv
venv/ venv/
Publish/

7
OF DL/Entities/Chats/ChatCollection.cs
View File

@ -1,7 +0,0 @@
namespace OF_DL.Entities.Chats
{
public class ChatCollection
{
public Dictionary<int, Chats.Chat> Chats { get; set; } = [];
}
}

20
OF DL/Entities/Chats/Chats.cs
View File

@ -1,20 +0,0 @@
namespace OF_DL.Entities.Chats
{
public class Chats
{
public List<Chat> list { get; set; }
public bool hasMore { get; set; }
public int nextOffset { get; set; }
public class Chat
{
public User withUser { get; set; }
public int unreadMessagesCount { get; set; }
}
public class User
{
public int id { get; set; }
}
}
}

5
OF DL/Entities/Config.cs
View File

@ -103,11 +103,6 @@ namespace OF_DL.Entities
[JsonConverter(typeof(StringEnumConverter))] [JsonConverter(typeof(StringEnumConverter))]
public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source; public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
public string[] NonInteractiveSpecificUsers { get; set; } = [];
public string[] NonInteractiveSpecificLists { get; set; } = [];
public bool OutputBlockedUsers { get; set; }
} }
public class CreatorConfig : IFileNameFormatConfig public class CreatorConfig : IFileNameFormatConfig

1
OF DL/Entities/Subscriptions.cs
View File

@ -98,7 +98,6 @@ namespace OF_DL.Entities
public object id { get; set; } public object id { get; set; }
public int? userId { get; set; } public int? userId { get; set; }
public int? subscriberId { get; set; } public int? subscriberId { get; set; }
public long? earningId { get; set; }
public DateTime? date { get; set; } public DateTime? date { get; set; }
public int? duration { get; set; } public int? duration { get; set; }
public DateTime? startDate { get; set; } public DateTime? startDate { get; set; }

295
OF DL/Helpers/APIHelper.cs
View File

@ -2,7 +2,6 @@ using Newtonsoft.Json;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using OF_DL.Entities; using OF_DL.Entities;
using OF_DL.Entities.Archived; using OF_DL.Entities.Archived;
using OF_DL.Entities.Chats;
using OF_DL.Entities.Highlights; using OF_DL.Entities.Highlights;
using OF_DL.Entities.Lists; using OF_DL.Entities.Lists;
using OF_DL.Entities.Messages; using OF_DL.Entities.Messages;
@ -14,7 +13,6 @@ using OF_DL.Enumerations;
using OF_DL.Enumurations; using OF_DL.Enumurations;
using Serilog; using Serilog;
using Spectre.Console; using Spectre.Console;
using System.Diagnostics;
using System.Globalization; using System.Globalization;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
@ -27,15 +25,10 @@ namespace OF_DL.Helpers;
public class APIHelper : IAPIHelper public class APIHelper : IAPIHelper
{ {
private const int MAX_RETRIES = 10;
private const int DELAY_BEFORE_RETRY = 1000;
private static readonly JsonSerializerSettings m_JsonSerializerSettings; private static readonly JsonSerializerSettings m_JsonSerializerSettings;
private readonly IDBHelper m_DBHelper; private readonly IDBHelper m_DBHelper;
private readonly IDownloadConfig downloadConfig; private readonly IDownloadConfig downloadConfig;
private readonly Auth auth; private readonly Auth auth;
private HttpClient httpClient = new();
private static DateTime? cachedDynamicRulesExpiration; private static DateTime? cachedDynamicRulesExpiration;
private static DynamicRules? cachedDynamicRules; private static DynamicRules? cachedDynamicRules;
@ -123,21 +116,12 @@ public class APIHelper : IAPIHelper
} }
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null, int retryCount = 0) private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
{ {
Log.Debug("Calling BuildHeaderAndExecuteRequests -- Attempt number: {AttemptNumber}", retryCount + 1); Log.Debug("Calling BuildHeaderAndExecuteRequests");
try
{
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
Debug.WriteLine($"Executing {request.Method.Method.ToUpper()} request: {request.RequestUri}\r\n\t{GetParamsString(getParams)}");
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
using var response = await client.SendAsync(request); using var response = await client.SendAsync(request);
if (Debugger.IsAttached && !response.IsSuccessStatusCode)
Debugger.Break();
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
string body = await response.Content.ReadAsStringAsync(); string body = await response.Content.ReadAsStringAsync();
@ -145,36 +129,17 @@ public class APIHelper : IAPIHelper
return body; return body;
} }
catch (HttpRequestException ex)
{
if (ex.StatusCode == System.Net.HttpStatusCode.TooManyRequests && retryCount < MAX_RETRIES)
{
await Task.Delay(DELAY_BEFORE_RETRY);
return await BuildHeaderAndExecuteRequests(getParams, endpoint, client, method, ++retryCount);
}
throw;
}
static string GetParamsString(Dictionary<string, string> getParams)
=> string.Join(" | ", getParams.Select(kv => $"{kv.Key}={kv.Value}"));
}
private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint, HttpMethod? method = null) private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint)
{ {
Log.Debug("Calling BuildHttpRequestMessage"); Log.Debug("Calling BuildHttpRequestMessage");
method ??= HttpMethod.Get; string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
string queryParams = "";
if (getParams.Any())
queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams); Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
HttpRequestMessage request = new(method, $"{Constants.API_URL}{endpoint}{queryParams}"); HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{queryParams}");
Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}"); Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}");
@ -199,16 +164,18 @@ public class APIHelper : IAPIHelper
return input.All(char.IsDigit); return input.All(char.IsDigit);
} }
private HttpClient GetHttpClient(IDownloadConfig? config = null)
private static HttpClient GetHttpClient(IDownloadConfig? config = null)
{ {
httpClient ??= new HttpClient(); var client = new HttpClient();
if (config?.Timeout != null && config.Timeout > 0) if (config?.Timeout != null && config.Timeout > 0)
{ {
httpClient.Timeout = TimeSpan.FromSeconds(config.Timeout.Value); client.Timeout = TimeSpan.FromSeconds(config.Timeout.Value);
} }
return httpClient; return client;
} }
/// <summary> /// <summary>
/// this one is used during initialization only /// this one is used during initialization only
/// if the config option is not available then no modificatiotns will be done on the getParams /// if the config option is not available then no modificatiotns will be done on the getParams
@ -331,44 +298,47 @@ public class APIHelper : IAPIHelper
try try
{ {
Dictionary<string, int> users = new(); Dictionary<string, int> users = new();
Subscriptions subscriptions = new();
int limit = 25;
int offset = 0;
getParams["limit"] = limit.ToString();
getParams["offset"] = offset.ToString();
Log.Debug("Calling GetAllSubscrptions"); Log.Debug("Calling GetAllSubscrptions");
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
if (subscriptions != null && subscriptions.hasMore)
{
getParams["offset"] = subscriptions.list.Count.ToString();
while (true) while (true)
{ {
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient); Subscriptions newSubscriptions = new();
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
if (string.IsNullOrWhiteSpace(body)) if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
break;
Subscriptions? subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body, m_JsonSerializerSettings);
if (subscriptions?.list is null)
break;
foreach (Subscriptions.List item in subscriptions.list)
{ {
if (users.ContainsKey(item.username)) newSubscriptions = JsonConvert.DeserializeObject<Subscriptions>(loopbody, m_JsonSerializerSettings);
continue; }
else
bool isRestricted = item.isRestricted ?? false; {
bool isRestrictedButAllowed = isRestricted && includeRestricted; break;
if (!isRestricted || isRestrictedButAllowed)
users.Add(item.username, item.id);
} }
if (!subscriptions.hasMore) subscriptions.list.AddRange(newSubscriptions.list);
if (!newSubscriptions.hasMore)
{
break; break;
}
getParams["offset"] = subscriptions.list.Count.ToString();
}
}
offset += limit; foreach (Subscriptions.List subscription in subscriptions.list)
getParams["offset"] = offset.ToString(); {
if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted))
&& !users.ContainsKey(subscription.username))
{
users.Add(subscription.username, subscription.id);
}
} }
return users; return users;
@ -391,20 +361,23 @@ public class APIHelper : IAPIHelper
{ {
Dictionary<string, string> getParams = new() Dictionary<string, string> getParams = new()
{ {
{ "offset", "0" },
{ "limit", "50" },
{ "type", "active" }, { "type", "active" },
{ "format", "infinite"} { "format", "infinite"}
}; };
Log.Debug("Calling GetActiveSubscriptions");
return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config); return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
} }
public async Task<Dictionary<string, int>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config) public async Task<Dictionary<string, int>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config)
{ {
Dictionary<string, string> getParams = new() Dictionary<string, string> getParams = new()
{ {
{ "offset", "0" },
{ "limit", "50" },
{ "type", "expired" }, { "type", "expired" },
{ "format", "infinite"} { "format", "infinite"}
}; };
@ -414,18 +387,6 @@ public class APIHelper : IAPIHelper
return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config); return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
} }
public async Task<Dictionary<string, int>?> GetBlockedUsers(string endpoint, IDownloadConfig config)
{
Dictionary<string, string> getParams = new()
{
{ "type", "expired" },
{ "format", "infinite"}
};
Log.Debug("Calling GetBlockedUsers");
return await GetAllSubscriptions(getParams, endpoint, true, config);
}
public async Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config) public async Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config)
{ {
@ -444,7 +405,7 @@ public class APIHelper : IAPIHelper
Dictionary<string, int> lists = new(); Dictionary<string, int> lists = new();
while (true) while (true)
{ {
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
if (body == null) if (body == null)
{ {
@ -509,7 +470,7 @@ public class APIHelper : IAPIHelper
while (true) while (true)
{ {
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
if (body == null) if (body == null)
{ {
break; break;
@ -553,66 +514,6 @@ public class APIHelper : IAPIHelper
} }
public async Task<Dictionary<string, int>?> GetUsersFromList(string endpoint, bool includeRestricted, IDownloadConfig config)
{
var model = new { list = new[] { new { id = int.MaxValue, username = string.Empty, isRestricted = false, isBlocked = false } }, nextOffset = 0, hasMore = false };
Log.Debug($"Calling GetUsersFromList - {endpoint}");
int limit = 50;
int offset = 0;
Dictionary<string, string> getParams = new()
{
{ "offset", offset.ToString() },
{ "limit", limit.ToString() },
{ "format", "infinite" }
};
try
{
Dictionary<string, int> users = [];
while (true)
{
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
if (string.IsNullOrWhiteSpace(body))
break;
var data = JsonConvert.DeserializeAnonymousType(body, model, m_JsonSerializerSettings);
if (data is null)
break;
foreach (var item in data.list)
{
if (users.ContainsKey(item.username))
continue;
bool isRestricted = item.isRestricted;
bool isRestrictedButAllowed = isRestricted && includeRestricted;
if (!isRestricted || isRestrictedButAllowed)
users.Add(item.username, item.id);
}
if (!data.hasMore)
break;
offset += data.nextOffset;
getParams["offset"] = offset.ToString();
}
return users;
}
catch (Exception ex)
{
throw;
}
}
public async Task<Dictionary<long, string>> GetMedia(MediaType mediatype, public async Task<Dictionary<long, string>> GetMedia(MediaType mediatype,
string endpoint, string endpoint,
string? username, string? username,
@ -652,7 +553,7 @@ public class APIHelper : IAPIHelper
break; break;
} }
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
if (mediatype == MediaType.Stories) if (mediatype == MediaType.Stories)
@ -1031,7 +932,7 @@ public class APIHelper : IAPIHelper
ref getParams, ref getParams,
downloadAsOf); downloadAsOf);
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
posts = JsonConvert.DeserializeObject<Post>(body, m_JsonSerializerSettings); posts = JsonConvert.DeserializeObject<Post>(body, m_JsonSerializerSettings);
ctx.Status($"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.list.Count}[/]"); ctx.Status($"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.list.Count}[/]");
ctx.Spinner(Spinner.Known.Dots); ctx.Spinner(Spinner.Known.Dots);
@ -1189,7 +1090,7 @@ public class APIHelper : IAPIHelper
{ "skip_users", "all" } { "skip_users", "all" }
}; };
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
singlePost = JsonConvert.DeserializeObject<SinglePost>(body, m_JsonSerializerSettings); singlePost = JsonConvert.DeserializeObject<SinglePost>(body, m_JsonSerializerSettings);
if (singlePost != null) if (singlePost != null)
@ -1350,7 +1251,7 @@ public class APIHelper : IAPIHelper
ref getParams, ref getParams,
config.CustomDate); config.CustomDate);
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config)); var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
streams = JsonConvert.DeserializeObject<Streams>(body, m_JsonSerializerSettings); streams = JsonConvert.DeserializeObject<Streams>(body, m_JsonSerializerSettings);
ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.list.Count}[/]"); ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.list.Count}[/]");
ctx.Spinner(Spinner.Known.Dots); ctx.Spinner(Spinner.Known.Dots);
@ -2684,94 +2585,6 @@ public class APIHelper : IAPIHelper
return null; return null;
} }
public async Task<ChatCollection> GetChats(string endpoint, IDownloadConfig config, bool onlyUnread)
{
Log.Debug($"Calling GetChats - {endpoint}");
try
{
Chats chats = new();
ChatCollection collection = new();
int limit = 60;
Dictionary<string, string> getParams = new()
{
{ "limit", $"{limit}" },
{ "offset", "0" },
{ "skip_users", "all" },
{ "order", "recent" }
};
if (onlyUnread)
getParams["filter"] = "unread";
string body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
chats = JsonConvert.DeserializeObject<Chats>(body, m_JsonSerializerSettings);
if (chats.hasMore)
{
getParams["offset"] = $"{chats.nextOffset}";
while (true)
{
string loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
Chats newChats = JsonConvert.DeserializeObject<Chats>(loopbody, m_JsonSerializerSettings);
chats.list.AddRange(newChats.list);
if (!newChats.hasMore)
break;
getParams["offset"] = $"{newChats.nextOffset}";
}
}
foreach (Chats.Chat chat in chats.list)
collection.Chats.Add(chat.withUser.id, chat);
return collection;
}
catch (Exception ex)
{
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
if (ex.InnerException != null)
{
Console.WriteLine("\nInner Exception:");
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
}
}
return null;
}
public async Task MarkAsUnread(string endpoint, IDownloadConfig config)
{
Log.Debug($"Calling MarkAsUnread - {endpoint}");
try
{
var result = new { success = false };
string body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(config), HttpMethod.Delete);
result = JsonConvert.DeserializeAnonymousType(body, result);
if (result?.success != true)
Console.WriteLine($"Failed to mark chat as unread! Endpoint: {endpoint}");
}
catch (Exception ex)
{
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
if (ex.InnerException != null)
{
Console.WriteLine("\nInner Exception:");
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
}
}
}
public async Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp) public async Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp)
{ {
@ -2890,7 +2703,7 @@ public class APIHelper : IAPIHelper
using var response = await client.SendAsync(request); using var response = await client.SendAsync(request);
Log.Debug($"CDRM Project Response (Attempt {attempt}): {await response.Content.ReadAsStringAsync()}"); Log.Debug($"CDRM Project Response (Attempt {attempt}): {response.Content.ReadAsStringAsync().Result}");
response.EnsureSuccessStatusCode(); response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync(); var body = await response.Content.ReadAsStringAsync();

107
OF DL/Helpers/DBHelper.cs
View File

@ -1,14 +1,18 @@
using Microsoft.Data.Sqlite; using System;
using OF_DL.Entities; using System.Collections.Generic;
using Serilog; using System.Linq;
using System.Text; using System.Text;
using System.Threading.Tasks;
using OF_DL.Enumurations;
using System.IO;
using Microsoft.Data.Sqlite;
using Serilog;
using OF_DL.Entities;
namespace OF_DL.Helpers namespace OF_DL.Helpers
{ {
public class DBHelper : IDBHelper public class DBHelper : IDBHelper
{ {
private static readonly Dictionary<string, SqliteConnection> _connections = [];
private readonly IDownloadConfig downloadConfig; private readonly IDownloadConfig downloadConfig;
public DBHelper(IDownloadConfig downloadConfig) public DBHelper(IDownloadConfig downloadConfig)
@ -28,7 +32,9 @@ namespace OF_DL.Helpers
string dbFilePath = $"{folder}/Metadata/user_data.db"; string dbFilePath = $"{folder}/Metadata/user_data.db";
// connect to the new database file // connect to the new database file
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={dbFilePath}"); using SqliteConnection connection = new($"Data Source={dbFilePath}");
// open the connection
connection.Open();
// create the 'medias' table // create the 'medias' table
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection)) using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection))
@ -133,9 +139,11 @@ namespace OF_DL.Helpers
{ {
try try
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db"); using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
Log.Debug("Database data source: " + connection.DataSource); Log.Debug("Database data source: " + connection.DataSource);
connection.Open();
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection)) using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection))
{ {
await cmd.ExecuteNonQueryAsync(); await cmd.ExecuteNonQueryAsync();
@ -186,7 +194,9 @@ namespace OF_DL.Helpers
{ {
try try
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db"); using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
connection.Open();
using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection)) using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection))
{ {
@ -237,8 +247,8 @@ namespace OF_DL.Helpers
{ {
try try
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
await EnsureCreatedAtColumnExists(connection, "messages"); await EnsureCreatedAtColumnExists(connection, "messages");
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection); using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection);
cmd.Parameters.AddWithValue("@post_id", post_id); cmd.Parameters.AddWithValue("@post_id", post_id);
@ -276,8 +286,8 @@ namespace OF_DL.Helpers
{ {
try try
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
await EnsureCreatedAtColumnExists(connection, "posts"); await EnsureCreatedAtColumnExists(connection, "posts");
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection); using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection);
cmd.Parameters.AddWithValue("@post_id", post_id); cmd.Parameters.AddWithValue("@post_id", post_id);
@ -314,8 +324,8 @@ namespace OF_DL.Helpers
{ {
try try
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
await EnsureCreatedAtColumnExists(connection, "stories"); await EnsureCreatedAtColumnExists(connection, "stories");
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection); using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection);
cmd.Parameters.AddWithValue("@post_id", post_id); cmd.Parameters.AddWithValue("@post_id", post_id);
@ -352,8 +362,8 @@ namespace OF_DL.Helpers
{ {
try try
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
await EnsureCreatedAtColumnExists(connection, "medias"); await EnsureCreatedAtColumnExists(connection, "medias");
StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id"); StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id");
if (downloadConfig.DownloadDuplicatedMedia) if (downloadConfig.DownloadDuplicatedMedia)
@ -390,8 +400,10 @@ namespace OF_DL.Helpers
{ {
try try
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); bool downloaded = false;
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
{
StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id"); StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id");
if(downloadConfig.DownloadDuplicatedMedia) if(downloadConfig.DownloadDuplicatedMedia)
{ {
@ -402,9 +414,8 @@ namespace OF_DL.Helpers
using SqliteCommand cmd = new (sql.ToString(), connection); using SqliteCommand cmd = new (sql.ToString(), connection);
cmd.Parameters.AddWithValue("@media_id", media_id); cmd.Parameters.AddWithValue("@media_id", media_id);
cmd.Parameters.AddWithValue("@api_type", api_type); cmd.Parameters.AddWithValue("@api_type", api_type);
downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
bool downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync()); }
return downloaded; return downloaded;
} }
catch (Exception ex) catch (Exception ex)
@ -424,7 +435,8 @@ namespace OF_DL.Helpers
public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at) public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at)
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
// Construct the update command // Construct the update command
StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id"); StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id");
@ -451,20 +463,24 @@ namespace OF_DL.Helpers
public async Task<long> GetStoredFileSize(string folder, long media_id, string api_type) public async Task<long> GetStoredFileSize(string folder, long media_id, string api_type)
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); long size;
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
{
connection.Open();
using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection); using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection);
cmd.Parameters.AddWithValue("@media_id", media_id); cmd.Parameters.AddWithValue("@media_id", media_id);
cmd.Parameters.AddWithValue("@api_type", api_type); cmd.Parameters.AddWithValue("@api_type", api_type);
size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
long size = Convert.ToInt64(await cmd.ExecuteScalarAsync()); }
return size; return size;
} }
public async Task<DateTime?> GetMostRecentPostDate(string folder) public async Task<DateTime?> GetMostRecentPostDate(string folder)
{ {
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db"); DateTime? mostRecentDate = null;
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
{
connection.Open();
using SqliteCommand cmd = new(@" using SqliteCommand cmd = new(@"
SELECT SELECT
MIN(created_at) AS created_at MIN(created_at) AS created_at
@ -481,14 +497,13 @@ namespace OF_DL.Helpers
ON P.post_id = m.post_id ON P.post_id = m.post_id
WHERE m.downloaded = 0 WHERE m.downloaded = 0
)", connection); )", connection);
var scalarValue = await cmd.ExecuteScalarAsync(); var scalarValue = await cmd.ExecuteScalarAsync();
if(scalarValue != null && scalarValue != DBNull.Value) if(scalarValue != null && scalarValue != DBNull.Value)
{ {
return Convert.ToDateTime(scalarValue); mostRecentDate = Convert.ToDateTime(scalarValue);
} }
}
return null; return mostRecentDate;
} }
private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName) private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName)
@ -512,35 +527,5 @@ namespace OF_DL.Helpers
await alterCmd.ExecuteNonQueryAsync(); await alterCmd.ExecuteNonQueryAsync();
} }
} }
public static void CloseAllConnections()
{
foreach (SqliteConnection cn in _connections.Values)
{
cn?.Close();
cn?.Dispose();
}
_connections.Clear();
}
private static async Task<SqliteConnection> GetAndOpenConnectionAsync(string connectionString, int numberOfRetries = 2)
{
try
{
SqliteConnection connection = new(connectionString);
connection.Open();
return connection;
}
catch (Exception)
{
if (--numberOfRetries <= 0)
throw;
await Task.Delay(300);
return await GetAndOpenConnectionAsync(connectionString, numberOfRetries);
}
}
} }
} }

4
OF DL/Helpers/DownloadHelper.cs
View File

@ -855,7 +855,7 @@ public class DownloadHelper : IDownloadHelper
memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.Seek(0, SeekOrigin.Begin);
MD5 md5 = MD5.Create(); MD5 md5 = MD5.Create();
byte[] hash = await md5.ComputeHashAsync(memoryStream); byte[] hash = md5.ComputeHash(memoryStream);
memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.Seek(0, SeekOrigin.Begin);
if (!avatarMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant())) if (!avatarMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant()))
{ {
@ -898,7 +898,7 @@ public class DownloadHelper : IDownloadHelper
memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.Seek(0, SeekOrigin.Begin);
MD5 md5 = MD5.Create(); MD5 md5 = MD5.Create();
byte[] hash = await md5.ComputeHashAsync(memoryStream); byte[] hash = md5.ComputeHash(memoryStream);
memoryStream.Seek(0, SeekOrigin.Begin); memoryStream.Seek(0, SeekOrigin.Begin);
if (!headerMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant())) if (!headerMD5Hashes.Contains(BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant()))
{ {

1
OF DL/Helpers/Interfaces/IAPIHelper.cs
View File

@ -18,7 +18,6 @@ namespace OF_DL.Helpers
Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp); Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp);
Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config); Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config);
Task<List<string>> GetListUsers(string endpoint, IDownloadConfig config); Task<List<string>> GetListUsers(string endpoint, IDownloadConfig config);
Task<Dictionary<string, int>?> GetUsersFromList(string endpoint, bool includeRestricted, IDownloadConfig config);
Task<Dictionary<long, string>> GetMedia(MediaType mediatype, string endpoint, string? username, string folder, IDownloadConfig config, List<long> paid_post_ids); Task<Dictionary<long, string>> GetMedia(MediaType mediatype, string endpoint, string? username, string folder, IDownloadConfig config, List<long> paid_post_ids);
Task<PaidPostCollection> GetPaidPosts(string endpoint, string folder, string username, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx); Task<PaidPostCollection> GetPaidPosts(string endpoint, string folder, string username, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx);
Task<PostCollection> GetPosts(string endpoint, string folder, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx); Task<PostCollection> GetPosts(string endpoint, string folder, IDownloadConfig config, List<long> paid_post_ids, StatusContext ctx);

10
OF DL/OF DL.csproj
View File

@ -7,13 +7,6 @@
<ImplicitUsings>enable</ImplicitUsings> <ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable> <Nullable>enable</Nullable>
<ApplicationIcon>Icon\download.ico</ApplicationIcon> <ApplicationIcon>Icon\download.ico</ApplicationIcon>
<LangVersion>12</LangVersion>
<PublishSingleFile>true</PublishSingleFile>
<SelfContained>true</SelfContained>
</PropertyGroup>
<PropertyGroup>
<NoWarn>CS0168;CS0219;CS0472;CS1998;CS8073;CS8600;CS8602;CS8603;CS8604;CS8605;CS8613;CS8618;CS8622;CS8625;CS8629;SYSLIB0021;AsyncFixer01;AsyncFixer02</NoWarn>
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
@ -44,15 +37,12 @@
<ItemGroup> <ItemGroup>
<None Update="auth.json"> <None Update="auth.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
<ExcludeFromSingleFile>true</ExcludeFromSingleFile>
</None> </None>
<None Update="config.conf"> <None Update="config.conf">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
<ExcludeFromSingleFile>true</ExcludeFromSingleFile>
</None> </None>
<None Update="rules.json"> <None Update="rules.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory> <CopyToOutputDirectory>Always</CopyToOutputDirectory>
<ExcludeFromSingleFile>true</ExcludeFromSingleFile>
</None> </None>
</ItemGroup> </ItemGroup>

206
OF DL/Program.cs
View File

@ -1,9 +1,7 @@
using Akka.Configuration;
using Newtonsoft.Json; using Newtonsoft.Json;
using Newtonsoft.Json.Linq; using Newtonsoft.Json.Linq;
using OF_DL.Entities; using OF_DL.Entities;
using OF_DL.Entities.Archived; using OF_DL.Entities.Archived;
using OF_DL.Entities.Chats;
using OF_DL.Entities.Messages; using OF_DL.Entities.Messages;
using OF_DL.Entities.Post; using OF_DL.Entities.Post;
using OF_DL.Entities.Purchased; using OF_DL.Entities.Purchased;
@ -15,12 +13,14 @@ using Serilog;
using Serilog.Core; using Serilog.Core;
using Serilog.Events; using Serilog.Events;
using Spectre.Console; using Spectre.Console;
using System.Diagnostics; using System.IO;
using System.Reflection; using System.Reflection;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using static OF_DL.Entities.Messages.Messages; using static OF_DL.Entities.Messages.Messages;
using Akka.Configuration;
using System.Text;
using static Akka.Actor.ProviderSelection;
namespace OF_DL; namespace OF_DL;
@ -44,15 +44,15 @@ public class Program
AuthHelper authHelper = new(); AuthHelper authHelper = new();
Task setupBrowserTask = authHelper.SetupBrowser(runningInDocker); Task setupBrowserTask = authHelper.SetupBrowser(runningInDocker);
await Task.Delay(1000); Task.Delay(1000).Wait();
if (!setupBrowserTask.IsCompleted) if (!setupBrowserTask.IsCompleted)
{ {
AnsiConsole.MarkupLine($"[yellow]Downloading dependencies. Please wait ...[/]"); AnsiConsole.MarkupLine($"[yellow]Downloading dependencies. Please wait ...[/]");
} }
await setupBrowserTask; setupBrowserTask.Wait();
Task<Auth?> getAuthTask = authHelper.GetAuthFromBrowser(); Task<Auth?> getAuthTask = authHelper.GetAuthFromBrowser();
await Task.Delay(5000); Task.Delay(5000).Wait();
if (!getAuthTask.IsCompleted) if (!getAuthTask.IsCompleted)
{ {
if (runningInDocker) if (runningInDocker)
@ -117,15 +117,13 @@ public class Program
AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red)); AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
ExitIfOtherProcess();
//Remove config.json and convert to config.conf //Remove config.json and convert to config.conf
if (File.Exists("config.json")) if (File.Exists("config.json"))
{ {
AnsiConsole.Markup("[green]config.json located successfully!\n[/]"); AnsiConsole.Markup("[green]config.json located successfully!\n[/]");
try try
{ {
string jsonText = await File.ReadAllTextAsync("config.json"); string jsonText = File.ReadAllText("config.json");
var jsonConfig = JsonConvert.DeserializeObject<Entities.Config>(jsonText); var jsonConfig = JsonConvert.DeserializeObject<Entities.Config>(jsonText);
if (jsonConfig != null) if (jsonConfig != null)
@ -221,7 +219,7 @@ public class Program
hoconConfig.AppendLine($" LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\""); hoconConfig.AppendLine($" LoggingLevel = \"{jsonConfig.LoggingLevel.ToString().ToLower()}\"");
hoconConfig.AppendLine("}"); hoconConfig.AppendLine("}");
await File.WriteAllTextAsync("config.conf", hoconConfig.ToString()); File.WriteAllText("config.conf", hoconConfig.ToString());
File.Delete("config.json"); File.Delete("config.json");
AnsiConsole.Markup("[green]config.conf created successfully from config.json!\n[/]"); AnsiConsole.Markup("[green]config.conf created successfully from config.json!\n[/]");
} }
@ -247,7 +245,7 @@ public class Program
AnsiConsole.Markup("[green]config.conf located successfully!\n[/]"); AnsiConsole.Markup("[green]config.conf located successfully!\n[/]");
try try
{ {
string hoconText = await File.ReadAllTextAsync("config.conf"); string hoconText = File.ReadAllText("config.conf");
var hoconConfig = ConfigurationFactory.ParseString(hoconText); var hoconConfig = ConfigurationFactory.ParseString(hoconText);
@ -476,48 +474,11 @@ public class Program
if (args is not null && args.Length > 0) if (args is not null && args.Length > 0)
{ {
const string NON_INTERACTIVE_ARG = "--non-interactive"; const string NON_INTERACTIVE_ARG = "--non-interactive";
const string SPECIFIC_LISTS_ARG = "--specific-lists";
const string SPECIFIC_USERS_ARG = "--specific-users";
if (args.Any(a => NON_INTERACTIVE_ARG.Equals(NON_INTERACTIVE_ARG, StringComparison.OrdinalIgnoreCase))) if (args.Any(a => NON_INTERACTIVE_ARG.Equals(NON_INTERACTIVE_ARG, StringComparison.OrdinalIgnoreCase)))
{ {
AnsiConsole.Markup($"[grey]Non-Interactive Mode enabled through command-line argument![/]\n"); cliNonInteractive = true;
Log.Debug("NonInteractiveMode set via command line");
config.NonInteractiveMode = true;
int indexOfSpecificListsArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_LISTS_ARG, StringComparison.OrdinalIgnoreCase));
int indexOfSpecificUsersArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_USERS_ARG, StringComparison.OrdinalIgnoreCase));
char[] separator = [','];
if (indexOfSpecificListsArg >= 0)
{
int indexOfListValues = indexOfSpecificListsArg + 1;
string[] strListValues = args.ElementAtOrDefault(indexOfListValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
if (strListValues.Length > 0)
{
config.NonInteractiveSpecificLists = strListValues;
config.NonInteractiveModeListName = string.Empty;
}
}
if (indexOfSpecificUsersArg >= 0)
{
int indexOfUserValues = indexOfSpecificUsersArg + 1;
string[] strUserValues = args.ElementAtOrDefault(indexOfUserValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
if (strUserValues.Length > 0)
{
config.NonInteractiveSpecificUsers = strUserValues;
}
}
}
const string OUTPUT_BLOCKED_USERS_ARG = "--output-blocked";
if (args.Any(a => OUTPUT_BLOCKED_USERS_ARG.Equals(a, StringComparison.OrdinalIgnoreCase)))
{
config.NonInteractiveMode = true;
config.OutputBlockedUsers = true;
} }
Log.Debug("Additional arguments:"); Log.Debug("Additional arguments:");
@ -685,10 +646,9 @@ public class Program
AnsiConsole.Markup("[green]rules.json located successfully!\n[/]"); AnsiConsole.Markup("[green]rules.json located successfully!\n[/]");
try try
{ {
string rulesJson = await File.ReadAllTextAsync("rules.json"); JsonConvert.DeserializeObject<DynamicRules>(File.ReadAllText("rules.json"));
DynamicRules? dynamicRules = JsonConvert.DeserializeObject<DynamicRules>(rulesJson);
Log.Debug($"Rules.json: "); Log.Debug($"Rules.json: ");
Log.Debug(JsonConvert.SerializeObject(dynamicRules, Formatting.Indented)); Log.Debug(JsonConvert.SerializeObject(File.ReadAllText("rules.json"), Formatting.Indented));
} }
catch (Exception e) catch (Exception e)
{ {
@ -852,22 +812,8 @@ public class Program
} }
AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]"); AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]");
try
{
if (config.OutputBlockedUsers)
{
await DownloadBlockedUsers(apiHelper, config);
return;
}
await DownloadAllData(apiHelper, auth, config); await DownloadAllData(apiHelper, auth, config);
} }
finally
{
DBHelper.CloseAllConnections();
}
}
catch (Exception ex) catch (Exception ex)
{ {
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
@ -887,23 +833,6 @@ public class Program
} }
} }
private static async Task DownloadBlockedUsers(APIHelper m_ApiHelper, Entities.Config Config)
{
const string OUTPUT_FILE = "blocked-users.json";
Dictionary<string, int>? blockedUsers = await m_ApiHelper.GetBlockedUsers("/users/blocked", Config);
if (blockedUsers is null || blockedUsers.Count == 0)
{
AnsiConsole.Markup($"[green]No Blocked Users found.\n[/]");
}
else
{
AnsiConsole.Markup($"[green]Found {blockedUsers.Count} Blocked Users, saving to '{OUTPUT_FILE}'\n[/]");
string json = JsonConvert.SerializeObject(blockedUsers, Formatting.Indented);
await File.WriteAllTextAsync(OUTPUT_FILE, json);
}
}
private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config) private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config)
{ {
@ -915,12 +844,11 @@ public class Program
{ {
DateTime startTime = DateTime.Now; DateTime startTime = DateTime.Now;
Dictionary<string, int> users = new(); Dictionary<string, int> users = new();
Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
AnsiConsole.Markup($"[green]Getting Active Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
Dictionary<string, int> subsActive = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? [];
Log.Debug("Subscriptions: "); Log.Debug("Subscriptions: ");
foreach (KeyValuePair<string, int> activeSub in subsActive)
foreach (KeyValuePair<string, int> activeSub in activeSubs)
{ {
if (!users.ContainsKey(activeSub.Key)) if (!users.ContainsKey(activeSub.Key))
{ {
@ -928,15 +856,12 @@ public class Program
Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}"); Log.Debug($"Name: {activeSub.Key} ID: {activeSub.Value}");
} }
} }
if (Config!.IncludeExpiredSubscriptions) if (Config!.IncludeExpiredSubscriptions)
{ {
Log.Debug("Inactive Subscriptions: "); Log.Debug("Inactive Subscriptions: ");
AnsiConsole.Markup($"[green]Getting Expired Subscriptions (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
Dictionary<string, int> subsExpired = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config) ?? []; foreach (KeyValuePair<string, int> expiredSub in expiredSubs)
foreach (KeyValuePair<string, int> expiredSub in subsExpired)
{ {
if (!users.ContainsKey(expiredSub.Key)) if (!users.ContainsKey(expiredSub.Key))
{ {
@ -963,36 +888,12 @@ public class Program
} }
} }
KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP = new(false, []); await dBHelper.CreateUsersDB(users);
KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP;
if(Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab) if(Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab)
{ {
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, new Dictionary<string, int> { { "PurchasedTab", 0 } }); hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, new Dictionary<string, int> { { "PurchasedTab", 0 } });
} }
else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificLists is not null && Config.NonInteractiveSpecificLists.Length > 0)
{
Dictionary<string, int> usersFromLists = new(StringComparer.OrdinalIgnoreCase);
foreach (string listName in Config.NonInteractiveSpecificLists)
{
if (!lists.TryGetValue(listName, out int listId))
continue;
AnsiConsole.Markup($"[green]Getting Users from list '{listName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]");
Dictionary<string, int> list = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config);
foreach ((string username, int id) in list)
usersFromLists.TryAdd(username, id);
}
users = usersFromLists;
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
}
else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificUsers is not null && Config.NonInteractiveSpecificUsers.Length > 0)
{
HashSet<string> usernames = [.. Config.NonInteractiveSpecificUsers];
users = users.Where(u => usernames.Contains(u.Key)).ToDictionary(u => u.Key, u => u.Value);
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
}
else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName)) else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName))
{ {
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users); hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
@ -1000,17 +901,11 @@ public class Program
else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName)) else if (Config.NonInteractiveMode && !string.IsNullOrEmpty(Config.NonInteractiveModeListName))
{ {
var listId = lists[Config.NonInteractiveModeListName]; var listId = lists[Config.NonInteractiveModeListName];
AnsiConsole.Markup($"[green]Getting Users from list '{Config.NonInteractiveModeListName}' (Include Restricted: {Config.IncludeRestrictedSubscriptions})\n[/]"); var listUsernames = await m_ApiHelper.GetListUsers($"/lists/{listId}/users", Config) ?? [];
users = await m_ApiHelper.GetUsersFromList($"/lists/{listId}/users", config.IncludeRestrictedSubscriptions, Config); var selectedUsers = users.Where(x => listUsernames.Contains(x.Key)).Distinct().ToDictionary(x => x.Key, x => x.Value);
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users); hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, selectedUsers);
} }
else
if (users.Count <= 0)
throw new InvalidOperationException("No users found!");
await dBHelper.CreateUsersDB(users);
if (hasSelectedUsersKVP.Key == false)
{ {
var userSelectionResult = await HandleUserSelection(m_ApiHelper, Config, users, lists); var userSelectionResult = await HandleUserSelection(m_ApiHelper, Config, users, lists);
@ -1129,11 +1024,9 @@ public class Program
Log.Debug($"Download path: {p}"); Log.Debug($"Download path: {p}");
List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users); List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
int userNum = 1;
int userCount = purchasedTabCollections.Count;
foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections) foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
{ {
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]"); AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]");
string path = ""; string path = "";
if (!string.IsNullOrEmpty(Config.DownloadPath)) if (!string.IsNullOrEmpty(Config.DownloadPath))
{ {
@ -1243,8 +1136,6 @@ public class Program
else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged")) else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
{ {
//Iterate over each user in the list of users //Iterate over each user in the list of users
int userNum = 1;
int userCount = hasSelectedUsersKVP.Value.Count;
foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value) foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
{ {
int paidPostCount = 0; int paidPostCount = 0;
@ -1255,7 +1146,7 @@ public class Program
int highlightsCount = 0; int highlightsCount = 0;
int messagesCount = 0; int messagesCount = 0;
int paidMessagesCount = 0; int paidMessagesCount = 0;
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]"); AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]");
Log.Debug($"Scraping Data for {user.Key}"); Log.Debug($"Scraping Data for {user.Key}");
@ -1540,9 +1431,6 @@ public class Program
{ {
Log.Debug($"Calling DownloadMessages - {user.Key}"); Log.Debug($"Calling DownloadMessages - {user.Key}");
AnsiConsole.Markup($"[grey]Getting Unread Chats\n[/]");
HashSet<int> unreadChats = await GetUsersWithUnreadChats(downloadContext.ApiHelper, downloadContext.DownloadConfig);
MessageCollection messages = new MessageCollection(); MessageCollection messages = new MessageCollection();
await AnsiConsole.Status() await AnsiConsole.Status()
@ -1550,13 +1438,6 @@ public class Program
{ {
messages = await downloadContext.ApiHelper.GetMessages($"/chats/{user.Value}/messages", path, downloadContext.DownloadConfig!, ctx); messages = await downloadContext.ApiHelper.GetMessages($"/chats/{user.Value}/messages", path, downloadContext.DownloadConfig!, ctx);
}); });
if (unreadChats.Contains(user.Value))
{
AnsiConsole.Markup($"[grey]Restoring unread state\n[/]");
await downloadContext.ApiHelper.MarkAsUnread($"/chats/{user.Value}/mark-as-read", downloadContext.DownloadConfig);
}
int oldMessagesCount = 0; int oldMessagesCount = 0;
int newMessagesCount = 0; int newMessagesCount = 0;
if (messages != null && messages.Messages.Count > 0) if (messages != null && messages.Messages.Count > 0)
@ -3082,7 +2963,7 @@ public class Program
hoconConfig.AppendLine($" LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\""); hoconConfig.AppendLine($" LoggingLevel = \"{newConfig.LoggingLevel.ToString().ToLower()}\"");
hoconConfig.AppendLine("}"); hoconConfig.AppendLine("}");
await File.WriteAllTextAsync("config.conf", hoconConfig.ToString()); File.WriteAllText("config.conf", hoconConfig.ToString());
string newConfigString = JsonConvert.SerializeObject(newConfig, Formatting.Indented); string newConfigString = JsonConvert.SerializeObject(newConfig, Formatting.Indented);
@ -3306,17 +3187,6 @@ public class Program
} }
} }
private static async Task<HashSet<int>> GetUsersWithUnreadChats(APIHelper apiHelper, IDownloadConfig currentConfig)
{
ChatCollection chats = await apiHelper.GetChats($"/chats", currentConfig, onlyUnread: true);
var unreadChats = chats.Chats
.Where(c => c.Value.unreadMessagesCount > 0)
.ToList();
return [.. unreadChats.Select(c => c.Key)];
}
static bool ValidateFilePath(string path) static bool ValidateFilePath(string path)
{ {
char[] invalidChars = System.IO.Path.GetInvalidPathChars(); char[] invalidChars = System.IO.Path.GetInvalidPathChars();
@ -3420,24 +3290,4 @@ public class Program
return Enum.Parse<VideoResolution>("_" + value, ignoreCase: true); return Enum.Parse<VideoResolution>("_" + value, ignoreCase: true);
} }
static void ExitIfOtherProcess()
{
Assembly entryAssembly = Assembly.GetEntryAssembly();
AssemblyName entryAssemblyName = entryAssembly?.GetName();
if (entryAssemblyName?.Name is null)
return;
Process thisProcess = Process.GetCurrentProcess();
Process[] otherProcesses = [.. Process.GetProcessesByName(entryAssemblyName.Name).Where(p => p.Id != thisProcess.Id)];
if (otherProcesses.Length <= 0)
return;
AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]");
Log.Warning("Other OF DL process detected, exiting..");
Environment.Exit(0);
}
} }

33
Publish_OF-DL.bat
View File

@ -1,33 +0,0 @@
@ECHO OFF
ECHO.
ECHO ==============================
ECHO == Cleaning Output ===========
ECHO ==============================
dotnet clean ".\OF DL\OF DL.csproj" -v minimal
DEL /Q /F ".\Publish"
ECHO.
ECHO ==============================
ECHO == Publishing OF-DL ==========
ECHO ==============================
dotnet publish ".\OF DL\OF DL.csproj" -o ".\Publish" -c Debug
ECHO.
ECHO ==============================
ECHO == Copy to network drive? ====
ECHO ==============================
CHOICE /C yn /m "Copy published files to network drive? "
IF %ERRORLEVEL%==1 (GOTO Copy) ELSE (GOTO Exit)
:Copy
xcopy .\Publish\* p:\_Utils\OF_DL /I /Y /Q /EXCLUDE:.\excludes.txt
:Exit
ECHO.
ECHO.
PAUSE

2
excludes.txt
View File

@ -1,2 +0,0 @@
excludes.txt
rules.json