forked from sim0n00ps/OF-DL
Compare commits
No commits in common. "ddbc98b12cf144bcf94a8896fc93f2be766e16b2" and "9f4dbccc020fa866a23510d9a2e0e56f916b9f92" have entirely different histories.
ddbc98b12c
...
9f4dbccc02
@ -1,7 +0,0 @@
|
|||||||
namespace OF_DL.Entities.Chats
|
|
||||||
{
|
|
||||||
public class ChatCollection
|
|
||||||
{
|
|
||||||
public Dictionary<int, Chats.Chat> Chats { get; set; } = [];
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,20 +0,0 @@
|
|||||||
namespace OF_DL.Entities.Chats
|
|
||||||
{
|
|
||||||
public class Chats
|
|
||||||
{
|
|
||||||
public List<Chat> list { get; set; }
|
|
||||||
public bool hasMore { get; set; }
|
|
||||||
public int nextOffset { get; set; }
|
|
||||||
|
|
||||||
public class Chat
|
|
||||||
{
|
|
||||||
public User withUser { get; set; }
|
|
||||||
public int unreadMessagesCount { get; set; }
|
|
||||||
}
|
|
||||||
|
|
||||||
public class User
|
|
||||||
{
|
|
||||||
public int id { get; set; }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
@ -103,11 +103,6 @@ namespace OF_DL.Entities
|
|||||||
|
|
||||||
[JsonConverter(typeof(StringEnumConverter))]
|
[JsonConverter(typeof(StringEnumConverter))]
|
||||||
public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
|
public VideoResolution DownloadVideoResolution { get; set; } = VideoResolution.source;
|
||||||
|
|
||||||
public string[] NonInteractiveSpecificUsers { get; set; } = [];
|
|
||||||
public string[] NonInteractiveSpecificLists { get; set; } = [];
|
|
||||||
|
|
||||||
public bool OutputBlockedUsers { get; set; }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public class CreatorConfig : IFileNameFormatConfig
|
public class CreatorConfig : IFileNameFormatConfig
|
||||||
|
@ -98,7 +98,6 @@ namespace OF_DL.Entities
|
|||||||
public object id { get; set; }
|
public object id { get; set; }
|
||||||
public int? userId { get; set; }
|
public int? userId { get; set; }
|
||||||
public int? subscriberId { get; set; }
|
public int? subscriberId { get; set; }
|
||||||
public long? earningId { get; set; }
|
|
||||||
public DateTime? date { get; set; }
|
public DateTime? date { get; set; }
|
||||||
public int? duration { get; set; }
|
public int? duration { get; set; }
|
||||||
public DateTime? startDate { get; set; }
|
public DateTime? startDate { get; set; }
|
||||||
|
@ -2,7 +2,6 @@ using Newtonsoft.Json;
|
|||||||
using Newtonsoft.Json.Linq;
|
using Newtonsoft.Json.Linq;
|
||||||
using OF_DL.Entities;
|
using OF_DL.Entities;
|
||||||
using OF_DL.Entities.Archived;
|
using OF_DL.Entities.Archived;
|
||||||
using OF_DL.Entities.Chats;
|
|
||||||
using OF_DL.Entities.Highlights;
|
using OF_DL.Entities.Highlights;
|
||||||
using OF_DL.Entities.Lists;
|
using OF_DL.Entities.Lists;
|
||||||
using OF_DL.Entities.Messages;
|
using OF_DL.Entities.Messages;
|
||||||
@ -30,8 +29,6 @@ public class APIHelper : IAPIHelper
|
|||||||
private readonly IDBHelper m_DBHelper;
|
private readonly IDBHelper m_DBHelper;
|
||||||
private readonly IDownloadConfig downloadConfig;
|
private readonly IDownloadConfig downloadConfig;
|
||||||
private readonly Auth auth;
|
private readonly Auth auth;
|
||||||
private HttpClient httpClient = new();
|
|
||||||
|
|
||||||
private static DateTime? cachedDynamicRulesExpiration;
|
private static DateTime? cachedDynamicRulesExpiration;
|
||||||
private static DynamicRules? cachedDynamicRules;
|
private static DynamicRules? cachedDynamicRules;
|
||||||
|
|
||||||
@ -119,11 +116,11 @@ public class APIHelper : IAPIHelper
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client, HttpMethod? method = null)
|
private async Task<string?> BuildHeaderAndExecuteRequests(Dictionary<string, string> getParams, string endpoint, HttpClient client)
|
||||||
{
|
{
|
||||||
Log.Debug("Calling BuildHeaderAndExecuteRequests");
|
Log.Debug("Calling BuildHeaderAndExecuteRequests");
|
||||||
|
|
||||||
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint, method);
|
HttpRequestMessage request = await BuildHttpRequestMessage(getParams, endpoint);
|
||||||
using var response = await client.SendAsync(request);
|
using var response = await client.SendAsync(request);
|
||||||
response.EnsureSuccessStatusCode();
|
response.EnsureSuccessStatusCode();
|
||||||
string body = await response.Content.ReadAsStringAsync();
|
string body = await response.Content.ReadAsStringAsync();
|
||||||
@ -134,20 +131,15 @@ public class APIHelper : IAPIHelper
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint, HttpMethod? method = null)
|
private async Task<HttpRequestMessage> BuildHttpRequestMessage(Dictionary<string, string> getParams, string endpoint)
|
||||||
{
|
{
|
||||||
Log.Debug("Calling BuildHttpRequestMessage");
|
Log.Debug("Calling BuildHttpRequestMessage");
|
||||||
|
|
||||||
method ??= HttpMethod.Get;
|
string queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
||||||
|
|
||||||
string queryParams = "";
|
|
||||||
|
|
||||||
if (getParams.Any())
|
|
||||||
queryParams = "?" + string.Join("&", getParams.Select(kvp => $"{kvp.Key}={kvp.Value}"));
|
|
||||||
|
|
||||||
Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
|
Dictionary<string, string> headers = GetDynamicHeaders($"/api2/v2{endpoint}", queryParams);
|
||||||
|
|
||||||
HttpRequestMessage request = new(method, $"{Constants.API_URL}{endpoint}{queryParams}");
|
HttpRequestMessage request = new(HttpMethod.Get, $"{Constants.API_URL}{endpoint}{queryParams}");
|
||||||
|
|
||||||
Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}");
|
Log.Debug($"Full request URL: {Constants.API_URL}{endpoint}{queryParams}");
|
||||||
|
|
||||||
@ -172,16 +164,18 @@ public class APIHelper : IAPIHelper
|
|||||||
return input.All(char.IsDigit);
|
return input.All(char.IsDigit);
|
||||||
}
|
}
|
||||||
|
|
||||||
private HttpClient GetHttpClient(IDownloadConfig? config = null)
|
|
||||||
|
private static HttpClient GetHttpClient(IDownloadConfig? config = null)
|
||||||
{
|
{
|
||||||
httpClient ??= new HttpClient();
|
var client = new HttpClient();
|
||||||
if (config?.Timeout != null && config.Timeout > 0)
|
if (config?.Timeout != null && config.Timeout > 0)
|
||||||
{
|
{
|
||||||
httpClient.Timeout = TimeSpan.FromSeconds(config.Timeout.Value);
|
client.Timeout = TimeSpan.FromSeconds(config.Timeout.Value);
|
||||||
}
|
}
|
||||||
return httpClient;
|
return client;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// this one is used during initialization only
|
/// this one is used during initialization only
|
||||||
/// if the config option is not available then no modificatiotns will be done on the getParams
|
/// if the config option is not available then no modificatiotns will be done on the getParams
|
||||||
@ -308,7 +302,7 @@ public class APIHelper : IAPIHelper
|
|||||||
|
|
||||||
Log.Debug("Calling GetAllSubscrptions");
|
Log.Debug("Calling GetAllSubscrptions");
|
||||||
|
|
||||||
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
|
|
||||||
subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
|
subscriptions = JsonConvert.DeserializeObject<Subscriptions>(body);
|
||||||
if (subscriptions != null && subscriptions.hasMore)
|
if (subscriptions != null && subscriptions.hasMore)
|
||||||
@ -318,7 +312,7 @@ public class APIHelper : IAPIHelper
|
|||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
Subscriptions newSubscriptions = new();
|
Subscriptions newSubscriptions = new();
|
||||||
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, httpClient);
|
string? loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
|
|
||||||
if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
|
if (!string.IsNullOrEmpty(loopbody) && (!loopbody.Contains("[]") || loopbody.Trim() != "[]"))
|
||||||
{
|
{
|
||||||
@ -340,15 +334,12 @@ public class APIHelper : IAPIHelper
|
|||||||
|
|
||||||
foreach (Subscriptions.List subscription in subscriptions.list)
|
foreach (Subscriptions.List subscription in subscriptions.list)
|
||||||
{
|
{
|
||||||
if (users.ContainsKey(subscription.username))
|
if ((!(subscription.isRestricted ?? false) || ((subscription.isRestricted ?? false) && includeRestricted))
|
||||||
continue;
|
&& !users.ContainsKey(subscription.username))
|
||||||
|
{
|
||||||
bool isRestricted = subscription.isRestricted ?? false;
|
|
||||||
bool isRestrictedButAllowed = isRestricted && includeRestricted;
|
|
||||||
|
|
||||||
if (!isRestricted || isRestrictedButAllowed)
|
|
||||||
users.Add(subscription.username, subscription.id);
|
users.Add(subscription.username, subscription.id);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return users;
|
return users;
|
||||||
}
|
}
|
||||||
@ -382,6 +373,7 @@ public class APIHelper : IAPIHelper
|
|||||||
|
|
||||||
public async Task<Dictionary<string, int>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config)
|
public async Task<Dictionary<string, int>?> GetExpiredSubscriptions(string endpoint, bool includeRestricted, IDownloadConfig config)
|
||||||
{
|
{
|
||||||
|
|
||||||
Dictionary<string, string> getParams = new()
|
Dictionary<string, string> getParams = new()
|
||||||
{
|
{
|
||||||
{ "offset", "0" },
|
{ "offset", "0" },
|
||||||
@ -395,20 +387,6 @@ public class APIHelper : IAPIHelper
|
|||||||
return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
|
return await GetAllSubscriptions(getParams, endpoint, includeRestricted, config);
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<Dictionary<string, int>?> GetBlockedUsers(string endpoint, IDownloadConfig config)
|
|
||||||
{
|
|
||||||
Dictionary<string, string> getParams = new()
|
|
||||||
{
|
|
||||||
{ "offset", "0" },
|
|
||||||
{ "limit", "50" },
|
|
||||||
{ "type", "expired" },
|
|
||||||
{ "format", "infinite"}
|
|
||||||
};
|
|
||||||
|
|
||||||
Log.Debug("Calling GetBlockedUsers");
|
|
||||||
|
|
||||||
return await GetAllSubscriptions(getParams, endpoint, true, config);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config)
|
public async Task<Dictionary<string, int>> GetLists(string endpoint, IDownloadConfig config)
|
||||||
{
|
{
|
||||||
@ -427,7 +405,7 @@ public class APIHelper : IAPIHelper
|
|||||||
Dictionary<string, int> lists = new();
|
Dictionary<string, int> lists = new();
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
string? body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
|
|
||||||
if (body == null)
|
if (body == null)
|
||||||
{
|
{
|
||||||
@ -492,7 +470,7 @@ public class APIHelper : IAPIHelper
|
|||||||
|
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
if (body == null)
|
if (body == null)
|
||||||
{
|
{
|
||||||
break;
|
break;
|
||||||
@ -575,7 +553,7 @@ public class APIHelper : IAPIHelper
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
|
|
||||||
|
|
||||||
if (mediatype == MediaType.Stories)
|
if (mediatype == MediaType.Stories)
|
||||||
@ -954,7 +932,7 @@ public class APIHelper : IAPIHelper
|
|||||||
ref getParams,
|
ref getParams,
|
||||||
downloadAsOf);
|
downloadAsOf);
|
||||||
|
|
||||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
posts = JsonConvert.DeserializeObject<Post>(body, m_JsonSerializerSettings);
|
posts = JsonConvert.DeserializeObject<Post>(body, m_JsonSerializerSettings);
|
||||||
ctx.Status($"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.list.Count}[/]");
|
ctx.Status($"[red]Getting Posts (this may take a long time, depending on the number of Posts the creator has)\n[/] [red]Found {posts.list.Count}[/]");
|
||||||
ctx.Spinner(Spinner.Known.Dots);
|
ctx.Spinner(Spinner.Known.Dots);
|
||||||
@ -1112,7 +1090,7 @@ public class APIHelper : IAPIHelper
|
|||||||
{ "skip_users", "all" }
|
{ "skip_users", "all" }
|
||||||
};
|
};
|
||||||
|
|
||||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
singlePost = JsonConvert.DeserializeObject<SinglePost>(body, m_JsonSerializerSettings);
|
singlePost = JsonConvert.DeserializeObject<SinglePost>(body, m_JsonSerializerSettings);
|
||||||
|
|
||||||
if (singlePost != null)
|
if (singlePost != null)
|
||||||
@ -1273,7 +1251,7 @@ public class APIHelper : IAPIHelper
|
|||||||
ref getParams,
|
ref getParams,
|
||||||
config.CustomDate);
|
config.CustomDate);
|
||||||
|
|
||||||
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
var body = await BuildHeaderAndExecuteRequests(getParams, endpoint, new HttpClient());
|
||||||
streams = JsonConvert.DeserializeObject<Streams>(body, m_JsonSerializerSettings);
|
streams = JsonConvert.DeserializeObject<Streams>(body, m_JsonSerializerSettings);
|
||||||
ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.list.Count}[/]");
|
ctx.Status($"[red]Getting Streams\n[/] [red]Found {streams.list.Count}[/]");
|
||||||
ctx.Spinner(Spinner.Known.Dots);
|
ctx.Spinner(Spinner.Known.Dots);
|
||||||
@ -2607,94 +2585,6 @@ public class APIHelper : IAPIHelper
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<ChatCollection> GetChats(string endpoint, IDownloadConfig config, bool onlyUnread)
|
|
||||||
{
|
|
||||||
Log.Debug($"Calling GetChats - {endpoint}");
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
Chats chats = new();
|
|
||||||
ChatCollection collection = new();
|
|
||||||
|
|
||||||
int limit = 60;
|
|
||||||
Dictionary<string, string> getParams = new()
|
|
||||||
{
|
|
||||||
{ "limit", $"{limit}" },
|
|
||||||
{ "offset", "0" },
|
|
||||||
{ "skip_users", "all" },
|
|
||||||
{ "order", "recent" }
|
|
||||||
};
|
|
||||||
|
|
||||||
if (onlyUnread)
|
|
||||||
getParams["filter"] = "unread";
|
|
||||||
|
|
||||||
string body = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
|
||||||
chats = JsonConvert.DeserializeObject<Chats>(body, m_JsonSerializerSettings);
|
|
||||||
|
|
||||||
if (chats.hasMore)
|
|
||||||
{
|
|
||||||
getParams["offset"] = $"{chats.nextOffset}";
|
|
||||||
|
|
||||||
while (true)
|
|
||||||
{
|
|
||||||
string loopbody = await BuildHeaderAndExecuteRequests(getParams, endpoint, GetHttpClient(config));
|
|
||||||
Chats newChats = JsonConvert.DeserializeObject<Chats>(loopbody, m_JsonSerializerSettings);
|
|
||||||
|
|
||||||
chats.list.AddRange(newChats.list);
|
|
||||||
|
|
||||||
if (!newChats.hasMore)
|
|
||||||
break;
|
|
||||||
|
|
||||||
getParams["offset"] = $"{newChats.nextOffset}";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach (Chats.Chat chat in chats.list)
|
|
||||||
collection.Chats.Add(chat.withUser.id, chat);
|
|
||||||
|
|
||||||
return collection;
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
||||||
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
||||||
if (ex.InnerException != null)
|
|
||||||
{
|
|
||||||
Console.WriteLine("\nInner Exception:");
|
|
||||||
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
|
|
||||||
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task MarkAsUnread(string endpoint, IDownloadConfig config)
|
|
||||||
{
|
|
||||||
Log.Debug($"Calling MarkAsUnread - {endpoint}");
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
var result = new { success = false };
|
|
||||||
|
|
||||||
string body = await BuildHeaderAndExecuteRequests([], endpoint, GetHttpClient(config), HttpMethod.Delete);
|
|
||||||
result = JsonConvert.DeserializeAnonymousType(body, result);
|
|
||||||
|
|
||||||
if (result?.success != true)
|
|
||||||
Console.WriteLine($"Failed to mark chat as unread! Endpoint: {endpoint}");
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
||||||
Log.Error("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
|
||||||
if (ex.InnerException != null)
|
|
||||||
{
|
|
||||||
Console.WriteLine("\nInner Exception:");
|
|
||||||
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
|
|
||||||
Log.Error("Inner Exception: {0}\n\nStackTrace: {1}", ex.InnerException.Message, ex.InnerException.StackTrace);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp)
|
public async Task<string> GetDRMMPDPSSH(string mpdUrl, string policy, string signature, string kvp)
|
||||||
{
|
{
|
||||||
|
@ -1,14 +1,18 @@
|
|||||||
using Microsoft.Data.Sqlite;
|
using System;
|
||||||
using OF_DL.Entities;
|
using System.Collections.Generic;
|
||||||
using Serilog;
|
using System.Linq;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using OF_DL.Enumurations;
|
||||||
|
using System.IO;
|
||||||
|
using Microsoft.Data.Sqlite;
|
||||||
|
using Serilog;
|
||||||
|
using OF_DL.Entities;
|
||||||
|
|
||||||
namespace OF_DL.Helpers
|
namespace OF_DL.Helpers
|
||||||
{
|
{
|
||||||
public class DBHelper : IDBHelper
|
public class DBHelper : IDBHelper
|
||||||
{
|
{
|
||||||
private static readonly Dictionary<string, SqliteConnection> _connections = [];
|
|
||||||
|
|
||||||
private readonly IDownloadConfig downloadConfig;
|
private readonly IDownloadConfig downloadConfig;
|
||||||
|
|
||||||
public DBHelper(IDownloadConfig downloadConfig)
|
public DBHelper(IDownloadConfig downloadConfig)
|
||||||
@ -28,7 +32,9 @@ namespace OF_DL.Helpers
|
|||||||
string dbFilePath = $"{folder}/Metadata/user_data.db";
|
string dbFilePath = $"{folder}/Metadata/user_data.db";
|
||||||
|
|
||||||
// connect to the new database file
|
// connect to the new database file
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={dbFilePath}");
|
using SqliteConnection connection = new($"Data Source={dbFilePath}");
|
||||||
|
// open the connection
|
||||||
|
connection.Open();
|
||||||
|
|
||||||
// create the 'medias' table
|
// create the 'medias' table
|
||||||
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection))
|
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection))
|
||||||
@ -133,9 +139,11 @@ namespace OF_DL.Helpers
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
|
using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
|
||||||
Log.Debug("Database data source: " + connection.DataSource);
|
Log.Debug("Database data source: " + connection.DataSource);
|
||||||
|
|
||||||
|
connection.Open();
|
||||||
|
|
||||||
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection))
|
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection))
|
||||||
{
|
{
|
||||||
await cmd.ExecuteNonQueryAsync();
|
await cmd.ExecuteNonQueryAsync();
|
||||||
@ -186,7 +194,9 @@ namespace OF_DL.Helpers
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
|
using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
|
||||||
|
|
||||||
|
connection.Open();
|
||||||
|
|
||||||
using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection))
|
using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection))
|
||||||
{
|
{
|
||||||
@ -237,8 +247,8 @@ namespace OF_DL.Helpers
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
|
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
|
||||||
|
connection.Open();
|
||||||
await EnsureCreatedAtColumnExists(connection, "messages");
|
await EnsureCreatedAtColumnExists(connection, "messages");
|
||||||
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection);
|
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection);
|
||||||
cmd.Parameters.AddWithValue("@post_id", post_id);
|
cmd.Parameters.AddWithValue("@post_id", post_id);
|
||||||
@ -276,8 +286,8 @@ namespace OF_DL.Helpers
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
|
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
|
||||||
|
connection.Open();
|
||||||
await EnsureCreatedAtColumnExists(connection, "posts");
|
await EnsureCreatedAtColumnExists(connection, "posts");
|
||||||
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection);
|
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection);
|
||||||
cmd.Parameters.AddWithValue("@post_id", post_id);
|
cmd.Parameters.AddWithValue("@post_id", post_id);
|
||||||
@ -314,8 +324,8 @@ namespace OF_DL.Helpers
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
|
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
|
||||||
|
connection.Open();
|
||||||
await EnsureCreatedAtColumnExists(connection, "stories");
|
await EnsureCreatedAtColumnExists(connection, "stories");
|
||||||
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection);
|
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection);
|
||||||
cmd.Parameters.AddWithValue("@post_id", post_id);
|
cmd.Parameters.AddWithValue("@post_id", post_id);
|
||||||
@ -352,8 +362,8 @@ namespace OF_DL.Helpers
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
|
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
|
||||||
|
connection.Open();
|
||||||
await EnsureCreatedAtColumnExists(connection, "medias");
|
await EnsureCreatedAtColumnExists(connection, "medias");
|
||||||
StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id");
|
StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id");
|
||||||
if (downloadConfig.DownloadDuplicatedMedia)
|
if (downloadConfig.DownloadDuplicatedMedia)
|
||||||
@ -390,8 +400,10 @@ namespace OF_DL.Helpers
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
|
bool downloaded = false;
|
||||||
|
|
||||||
|
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
|
||||||
|
{
|
||||||
StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id");
|
StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id");
|
||||||
if(downloadConfig.DownloadDuplicatedMedia)
|
if(downloadConfig.DownloadDuplicatedMedia)
|
||||||
{
|
{
|
||||||
@ -402,9 +414,8 @@ namespace OF_DL.Helpers
|
|||||||
using SqliteCommand cmd = new (sql.ToString(), connection);
|
using SqliteCommand cmd = new (sql.ToString(), connection);
|
||||||
cmd.Parameters.AddWithValue("@media_id", media_id);
|
cmd.Parameters.AddWithValue("@media_id", media_id);
|
||||||
cmd.Parameters.AddWithValue("@api_type", api_type);
|
cmd.Parameters.AddWithValue("@api_type", api_type);
|
||||||
|
downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
|
||||||
bool downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
|
}
|
||||||
|
|
||||||
return downloaded;
|
return downloaded;
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
@ -424,7 +435,8 @@ namespace OF_DL.Helpers
|
|||||||
|
|
||||||
public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at)
|
public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at)
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
|
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
|
||||||
|
connection.Open();
|
||||||
|
|
||||||
// Construct the update command
|
// Construct the update command
|
||||||
StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id");
|
StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id");
|
||||||
@ -451,20 +463,24 @@ namespace OF_DL.Helpers
|
|||||||
|
|
||||||
public async Task<long> GetStoredFileSize(string folder, long media_id, string api_type)
|
public async Task<long> GetStoredFileSize(string folder, long media_id, string api_type)
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
|
long size;
|
||||||
|
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
|
||||||
|
{
|
||||||
|
connection.Open();
|
||||||
using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection);
|
using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection);
|
||||||
cmd.Parameters.AddWithValue("@media_id", media_id);
|
cmd.Parameters.AddWithValue("@media_id", media_id);
|
||||||
cmd.Parameters.AddWithValue("@api_type", api_type);
|
cmd.Parameters.AddWithValue("@api_type", api_type);
|
||||||
|
size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
|
||||||
long size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
|
}
|
||||||
return size;
|
return size;
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<DateTime?> GetMostRecentPostDate(string folder)
|
public async Task<DateTime?> GetMostRecentPostDate(string folder)
|
||||||
{
|
{
|
||||||
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
|
DateTime? mostRecentDate = null;
|
||||||
|
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
|
||||||
|
{
|
||||||
|
connection.Open();
|
||||||
using SqliteCommand cmd = new(@"
|
using SqliteCommand cmd = new(@"
|
||||||
SELECT
|
SELECT
|
||||||
MIN(created_at) AS created_at
|
MIN(created_at) AS created_at
|
||||||
@ -481,14 +497,13 @@ namespace OF_DL.Helpers
|
|||||||
ON P.post_id = m.post_id
|
ON P.post_id = m.post_id
|
||||||
WHERE m.downloaded = 0
|
WHERE m.downloaded = 0
|
||||||
)", connection);
|
)", connection);
|
||||||
|
|
||||||
var scalarValue = await cmd.ExecuteScalarAsync();
|
var scalarValue = await cmd.ExecuteScalarAsync();
|
||||||
if(scalarValue != null && scalarValue != DBNull.Value)
|
if(scalarValue != null && scalarValue != DBNull.Value)
|
||||||
{
|
{
|
||||||
return Convert.ToDateTime(scalarValue);
|
mostRecentDate = Convert.ToDateTime(scalarValue);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
return null;
|
return mostRecentDate;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName)
|
private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName)
|
||||||
@ -512,35 +527,5 @@ namespace OF_DL.Helpers
|
|||||||
await alterCmd.ExecuteNonQueryAsync();
|
await alterCmd.ExecuteNonQueryAsync();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void CloseAllConnections()
|
|
||||||
{
|
|
||||||
foreach (SqliteConnection cn in _connections.Values)
|
|
||||||
{
|
|
||||||
cn?.Close();
|
|
||||||
cn?.Dispose();
|
|
||||||
}
|
|
||||||
|
|
||||||
_connections.Clear();
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async Task<SqliteConnection> GetAndOpenConnectionAsync(string connectionString, int numberOfRetries = 2)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
SqliteConnection connection = new(connectionString);
|
|
||||||
connection.Open();
|
|
||||||
|
|
||||||
return connection;
|
|
||||||
}
|
|
||||||
catch (Exception)
|
|
||||||
{
|
|
||||||
if (--numberOfRetries <= 0)
|
|
||||||
throw;
|
|
||||||
|
|
||||||
await Task.Delay(300);
|
|
||||||
return await GetAndOpenConnectionAsync(connectionString, numberOfRetries);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
171
OF DL/Program.cs
171
OF DL/Program.cs
@ -22,8 +22,6 @@ using static OF_DL.Entities.Messages.Messages;
|
|||||||
using Akka.Configuration;
|
using Akka.Configuration;
|
||||||
using System.Text;
|
using System.Text;
|
||||||
using static Akka.Actor.ProviderSelection;
|
using static Akka.Actor.ProviderSelection;
|
||||||
using System.Diagnostics;
|
|
||||||
using OF_DL.Entities.Chats;
|
|
||||||
|
|
||||||
namespace OF_DL;
|
namespace OF_DL;
|
||||||
|
|
||||||
@ -120,8 +118,6 @@ public class Program
|
|||||||
|
|
||||||
AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
|
AnsiConsole.Write(new FigletText("Welcome to OF-DL").Color(Color.Red));
|
||||||
|
|
||||||
ExitIfOtherProcess();
|
|
||||||
|
|
||||||
//Remove config.json and convert to config.conf
|
//Remove config.json and convert to config.conf
|
||||||
if (File.Exists("config.json"))
|
if (File.Exists("config.json"))
|
||||||
{
|
{
|
||||||
@ -479,48 +475,11 @@ public class Program
|
|||||||
if (args is not null && args.Length > 0)
|
if (args is not null && args.Length > 0)
|
||||||
{
|
{
|
||||||
const string NON_INTERACTIVE_ARG = "--non-interactive";
|
const string NON_INTERACTIVE_ARG = "--non-interactive";
|
||||||
const string SPECIFIC_LISTS_ARG = "--specific-lists";
|
|
||||||
const string SPECIFIC_USERS_ARG = "--specific-users";
|
|
||||||
|
|
||||||
if (args.Any(a => NON_INTERACTIVE_ARG.Equals(NON_INTERACTIVE_ARG, StringComparison.OrdinalIgnoreCase)))
|
if (args.Any(a => NON_INTERACTIVE_ARG.Equals(NON_INTERACTIVE_ARG, StringComparison.OrdinalIgnoreCase)))
|
||||||
{
|
{
|
||||||
AnsiConsole.Markup($"[grey]Non-Interactive Mode enabled through command-line argument![/]\n");
|
cliNonInteractive = true;
|
||||||
|
Log.Debug("NonInteractiveMode set via command line");
|
||||||
config.NonInteractiveMode = true;
|
|
||||||
|
|
||||||
int indexOfSpecificListsArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_LISTS_ARG, StringComparison.OrdinalIgnoreCase));
|
|
||||||
int indexOfSpecificUsersArg = Array.FindIndex(args, a => a.Contains(SPECIFIC_USERS_ARG, StringComparison.OrdinalIgnoreCase));
|
|
||||||
char[] separator = [','];
|
|
||||||
|
|
||||||
if (indexOfSpecificListsArg >= 0)
|
|
||||||
{
|
|
||||||
int indexOfListValues = indexOfSpecificListsArg + 1;
|
|
||||||
|
|
||||||
string[] strListValues = args.ElementAtOrDefault(indexOfListValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
|
|
||||||
if (strListValues.Length > 0)
|
|
||||||
{
|
|
||||||
config.NonInteractiveSpecificLists = strListValues;
|
|
||||||
config.NonInteractiveModeListName = string.Empty;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (indexOfSpecificUsersArg >= 0)
|
|
||||||
{
|
|
||||||
int indexOfUserValues = indexOfSpecificUsersArg + 1;
|
|
||||||
string[] strUserValues = args.ElementAtOrDefault(indexOfUserValues)?.Split(separator, StringSplitOptions.RemoveEmptyEntries) ?? [];
|
|
||||||
if (strUserValues.Length > 0)
|
|
||||||
{
|
|
||||||
config.NonInteractiveSpecificUsers = strUserValues;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const string OUTPUT_BLOCKED_USERS_ARG = "--output-blocked";
|
|
||||||
|
|
||||||
if (args.Any(a => OUTPUT_BLOCKED_USERS_ARG.Equals(a, StringComparison.OrdinalIgnoreCase)))
|
|
||||||
{
|
|
||||||
config.NonInteractiveMode = true;
|
|
||||||
config.OutputBlockedUsers = true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Log.Debug("Additional arguments:");
|
Log.Debug("Additional arguments:");
|
||||||
@ -815,22 +774,8 @@ public class Program
|
|||||||
}
|
}
|
||||||
|
|
||||||
AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]");
|
AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]");
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (config.OutputBlockedUsers)
|
|
||||||
{
|
|
||||||
await DownloadBlockedUsers(apiHelper, config);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await DownloadAllData(apiHelper, auth, config);
|
await DownloadAllData(apiHelper, auth, config);
|
||||||
}
|
}
|
||||||
finally
|
|
||||||
{
|
|
||||||
DBHelper.CloseAllConnections();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);
|
||||||
@ -850,27 +795,6 @@ public class Program
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task DownloadBlockedUsers(APIHelper m_ApiHelper, Entities.Config Config)
|
|
||||||
{
|
|
||||||
const string OUTPUT_FILE = "blocked-users.json";
|
|
||||||
|
|
||||||
Log.Debug($"Calling GetBlockedUsers");
|
|
||||||
|
|
||||||
AnsiConsole.Markup($"[red]Getting Blocked Users\n[/]");
|
|
||||||
|
|
||||||
Dictionary<string, int>? blockedUsers = await m_ApiHelper.GetBlockedUsers("/users/blocked", Config);
|
|
||||||
|
|
||||||
if (blockedUsers is null || blockedUsers.Count == 0)
|
|
||||||
{
|
|
||||||
AnsiConsole.Markup($"[red]No Blocked Users found.\n[/]");
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
AnsiConsole.Markup($"[red]Found {blockedUsers.Count} Blocked Users, saving to '{OUTPUT_FILE}'\n[/]");
|
|
||||||
string json = JsonConvert.SerializeObject(blockedUsers, Formatting.Indented);
|
|
||||||
await File.WriteAllTextAsync(OUTPUT_FILE, json);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config)
|
private static async Task DownloadAllData(APIHelper m_ApiHelper, Auth Auth, Entities.Config Config)
|
||||||
{
|
{
|
||||||
@ -882,19 +806,11 @@ public class Program
|
|||||||
{
|
{
|
||||||
DateTime startTime = DateTime.Now;
|
DateTime startTime = DateTime.Now;
|
||||||
Dictionary<string, int> users = new();
|
Dictionary<string, int> users = new();
|
||||||
|
Dictionary<string, int> activeSubs = await m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
|
||||||
Task<Dictionary<string, int>?> taskActive = m_ApiHelper.GetActiveSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
|
|
||||||
Task<Dictionary<string, int>?> taskExpired = Config!.IncludeExpiredSubscriptions
|
|
||||||
? m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config)
|
|
||||||
: Task.FromResult<Dictionary<string, int>?>([]);
|
|
||||||
|
|
||||||
await Task.WhenAll(taskActive, taskExpired);
|
|
||||||
|
|
||||||
Dictionary<string, int> subsActive = await taskActive ?? [];
|
|
||||||
Dictionary<string, int> subsExpired = await taskExpired ?? [];
|
|
||||||
|
|
||||||
Log.Debug("Subscriptions: ");
|
Log.Debug("Subscriptions: ");
|
||||||
foreach (KeyValuePair<string, int> activeSub in subsActive)
|
|
||||||
|
foreach (KeyValuePair<string, int> activeSub in activeSubs)
|
||||||
{
|
{
|
||||||
if (!users.ContainsKey(activeSub.Key))
|
if (!users.ContainsKey(activeSub.Key))
|
||||||
{
|
{
|
||||||
@ -905,7 +821,9 @@ public class Program
|
|||||||
if (Config!.IncludeExpiredSubscriptions)
|
if (Config!.IncludeExpiredSubscriptions)
|
||||||
{
|
{
|
||||||
Log.Debug("Inactive Subscriptions: ");
|
Log.Debug("Inactive Subscriptions: ");
|
||||||
foreach (KeyValuePair<string, int> expiredSub in subsExpired)
|
|
||||||
|
Dictionary<string, int> expiredSubs = await m_ApiHelper.GetExpiredSubscriptions("/subscriptions/subscribes", Config.IncludeRestrictedSubscriptions, Config);
|
||||||
|
foreach (KeyValuePair<string, int> expiredSub in expiredSubs)
|
||||||
{
|
{
|
||||||
if (!users.ContainsKey(expiredSub.Key))
|
if (!users.ContainsKey(expiredSub.Key))
|
||||||
{
|
{
|
||||||
@ -932,36 +850,12 @@ public class Program
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (users.Count <= 0)
|
|
||||||
throw new InvalidOperationException("No users found!");
|
|
||||||
|
|
||||||
await dBHelper.CreateUsersDB(users);
|
await dBHelper.CreateUsersDB(users);
|
||||||
KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP;
|
KeyValuePair<bool, Dictionary<string, int>> hasSelectedUsersKVP;
|
||||||
if(Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab)
|
if(Config.NonInteractiveMode && Config.NonInteractiveModePurchasedTab)
|
||||||
{
|
{
|
||||||
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, new Dictionary<string, int> { { "PurchasedTab", 0 } });
|
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, new Dictionary<string, int> { { "PurchasedTab", 0 } });
|
||||||
}
|
}
|
||||||
else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificLists is not null && Config.NonInteractiveSpecificLists.Length > 0)
|
|
||||||
{
|
|
||||||
HashSet<string> listUsernames = [];
|
|
||||||
foreach (string listName in Config.NonInteractiveSpecificLists)
|
|
||||||
{
|
|
||||||
if (!lists.TryGetValue(listName, out int listId))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
List<string> usernames = await m_ApiHelper.GetListUsers($"/lists/{listId}/users", Config);
|
|
||||||
foreach (string user in usernames)
|
|
||||||
listUsernames.Add(user);
|
|
||||||
}
|
|
||||||
users = users.Where(x => listUsernames.Contains(x.Key)).Distinct().ToDictionary(x => x.Key, x => x.Value);
|
|
||||||
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
|
|
||||||
}
|
|
||||||
else if (Config.NonInteractiveMode && Config.NonInteractiveSpecificUsers is not null && Config.NonInteractiveSpecificUsers.Length > 0)
|
|
||||||
{
|
|
||||||
HashSet<string> usernames = [.. Config.NonInteractiveSpecificUsers];
|
|
||||||
users = users.Where(u => usernames.Contains(u.Key)).ToDictionary(u => u.Key, u => u.Value);
|
|
||||||
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
|
|
||||||
}
|
|
||||||
else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName))
|
else if (Config.NonInteractiveMode && string.IsNullOrEmpty(Config.NonInteractiveModeListName))
|
||||||
{
|
{
|
||||||
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
|
hasSelectedUsersKVP = new KeyValuePair<bool, Dictionary<string, int>>(true, users);
|
||||||
@ -1092,11 +986,9 @@ public class Program
|
|||||||
Log.Debug($"Download path: {p}");
|
Log.Debug($"Download path: {p}");
|
||||||
|
|
||||||
List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
|
List<PurchasedTabCollection> purchasedTabCollections = await m_ApiHelper.GetPurchasedTab("/posts/paid", p, Config, users);
|
||||||
int userNum = 1;
|
|
||||||
int userCount = purchasedTabCollections.Count;
|
|
||||||
foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
|
foreach(PurchasedTabCollection purchasedTabCollection in purchasedTabCollections)
|
||||||
{
|
{
|
||||||
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username} ({userNum++} of {userCount})\n[/]");
|
AnsiConsole.Markup($"[red]\nScraping Data for {purchasedTabCollection.Username}\n[/]");
|
||||||
string path = "";
|
string path = "";
|
||||||
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
if (!string.IsNullOrEmpty(Config.DownloadPath))
|
||||||
{
|
{
|
||||||
@ -1206,8 +1098,6 @@ public class Program
|
|||||||
else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
else if (hasSelectedUsersKVP.Key && !hasSelectedUsersKVP.Value.ContainsKey("ConfigChanged"))
|
||||||
{
|
{
|
||||||
//Iterate over each user in the list of users
|
//Iterate over each user in the list of users
|
||||||
int userNum = 1;
|
|
||||||
int userCount = hasSelectedUsersKVP.Value.Count;
|
|
||||||
foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
|
foreach (KeyValuePair<string, int> user in hasSelectedUsersKVP.Value)
|
||||||
{
|
{
|
||||||
int paidPostCount = 0;
|
int paidPostCount = 0;
|
||||||
@ -1218,7 +1108,7 @@ public class Program
|
|||||||
int highlightsCount = 0;
|
int highlightsCount = 0;
|
||||||
int messagesCount = 0;
|
int messagesCount = 0;
|
||||||
int paidMessagesCount = 0;
|
int paidMessagesCount = 0;
|
||||||
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key} ({userNum++} of {userCount})\n[/]");
|
AnsiConsole.Markup($"[red]\nScraping Data for {user.Key}\n[/]");
|
||||||
|
|
||||||
Log.Debug($"Scraping Data for {user.Key}");
|
Log.Debug($"Scraping Data for {user.Key}");
|
||||||
|
|
||||||
@ -1503,9 +1393,6 @@ public class Program
|
|||||||
{
|
{
|
||||||
Log.Debug($"Calling DownloadMessages - {user.Key}");
|
Log.Debug($"Calling DownloadMessages - {user.Key}");
|
||||||
|
|
||||||
AnsiConsole.Markup($"[grey]Getting Unread Chats\n[/]");
|
|
||||||
HashSet<int> unreadChats = await GetUsersWithUnreadChats(downloadContext.ApiHelper, downloadContext.DownloadConfig);
|
|
||||||
|
|
||||||
MessageCollection messages = new MessageCollection();
|
MessageCollection messages = new MessageCollection();
|
||||||
|
|
||||||
await AnsiConsole.Status()
|
await AnsiConsole.Status()
|
||||||
@ -1513,13 +1400,6 @@ public class Program
|
|||||||
{
|
{
|
||||||
messages = await downloadContext.ApiHelper.GetMessages($"/chats/{user.Value}/messages", path, downloadContext.DownloadConfig!, ctx);
|
messages = await downloadContext.ApiHelper.GetMessages($"/chats/{user.Value}/messages", path, downloadContext.DownloadConfig!, ctx);
|
||||||
});
|
});
|
||||||
|
|
||||||
if (unreadChats.Contains(user.Value))
|
|
||||||
{
|
|
||||||
AnsiConsole.Markup($"[grey]Restoring unread state\n[/]");
|
|
||||||
await downloadContext.ApiHelper.MarkAsUnread($"/chats/{user.Value}/mark-as-read", downloadContext.DownloadConfig);
|
|
||||||
}
|
|
||||||
|
|
||||||
int oldMessagesCount = 0;
|
int oldMessagesCount = 0;
|
||||||
int newMessagesCount = 0;
|
int newMessagesCount = 0;
|
||||||
if (messages != null && messages.Messages.Count > 0)
|
if (messages != null && messages.Messages.Count > 0)
|
||||||
@ -3269,17 +3149,6 @@ public class Program
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static async Task<HashSet<int>> GetUsersWithUnreadChats(APIHelper apiHelper, IDownloadConfig currentConfig)
|
|
||||||
{
|
|
||||||
ChatCollection chats = await apiHelper.GetChats($"/chats", currentConfig, onlyUnread: true);
|
|
||||||
|
|
||||||
var unreadChats = chats.Chats
|
|
||||||
.Where(c => c.Value.unreadMessagesCount > 0)
|
|
||||||
.ToList();
|
|
||||||
|
|
||||||
return [.. unreadChats.Select(c => c.Key)];
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool ValidateFilePath(string path)
|
static bool ValidateFilePath(string path)
|
||||||
{
|
{
|
||||||
char[] invalidChars = System.IO.Path.GetInvalidPathChars();
|
char[] invalidChars = System.IO.Path.GetInvalidPathChars();
|
||||||
@ -3383,24 +3252,4 @@ public class Program
|
|||||||
|
|
||||||
return Enum.Parse<VideoResolution>("_" + value, ignoreCase: true);
|
return Enum.Parse<VideoResolution>("_" + value, ignoreCase: true);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void ExitIfOtherProcess()
|
|
||||||
{
|
|
||||||
Assembly entryAssembly = Assembly.GetEntryAssembly();
|
|
||||||
AssemblyName entryAssemblyName = entryAssembly?.GetName();
|
|
||||||
|
|
||||||
if (entryAssemblyName?.Name is null)
|
|
||||||
return;
|
|
||||||
|
|
||||||
Process thisProcess = Process.GetCurrentProcess();
|
|
||||||
Process[] otherProcesses = [.. Process.GetProcessesByName(entryAssemblyName.Name).Where(p => p.Id != thisProcess.Id)];
|
|
||||||
|
|
||||||
if (otherProcesses.Length <= 0)
|
|
||||||
return;
|
|
||||||
|
|
||||||
AnsiConsole.Markup($"[green]Other OF DL process detected, exiting..\n[/]");
|
|
||||||
Log.Warning("Other OF DL process detected, exiting..");
|
|
||||||
|
|
||||||
Environment.Exit(0);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
@ -1,33 +1,7 @@
|
|||||||
@ECHO OFF
|
@ECHO OFF
|
||||||
|
|
||||||
ECHO.
|
ECHO.
|
||||||
|
|
||||||
ECHO ==============================
|
|
||||||
ECHO == Cleaning Output ===========
|
|
||||||
ECHO ==============================
|
|
||||||
dotnet clean ".\OF DL\OF DL.csproj" -v minimal
|
|
||||||
DEL /Q /F ".\Publish"
|
|
||||||
|
|
||||||
ECHO.
|
|
||||||
|
|
||||||
ECHO ==============================
|
|
||||||
ECHO == Publishing OF-DL ==========
|
|
||||||
ECHO ==============================
|
|
||||||
dotnet publish ".\OF DL\OF DL.csproj" -o ".\Publish"
|
dotnet publish ".\OF DL\OF DL.csproj" -o ".\Publish"
|
||||||
|
|
||||||
ECHO.
|
|
||||||
|
|
||||||
ECHO ==============================
|
|
||||||
ECHO == Copy to network drive? ====
|
|
||||||
ECHO ==============================
|
|
||||||
CHOICE /C yn /m "Copy published files to network drive? "
|
|
||||||
|
|
||||||
IF %ERRORLEVEL%==1 (GOTO Copy) ELSE (GOTO Exit)
|
|
||||||
|
|
||||||
:Copy
|
|
||||||
xcopy .\Publish\* p:\_Utils\OF_DL /I /Y /Q /EXCLUDE:.\excludes.txt
|
|
||||||
|
|
||||||
:Exit
|
|
||||||
ECHO.
|
|
||||||
ECHO.
|
ECHO.
|
||||||
PAUSE
|
PAUSE
|
@ -1,2 +0,0 @@
|
|||||||
excludes.txt
|
|
||||||
rules.json
|
|
Loading…
x
Reference in New Issue
Block a user