reptail/OF-DL
1
0
Fork 0
forked from sim0n00ps/OF-DL
Code Pull Requests Activity

Improved DB connection creation with delayed retry, and connection caching

Browse Source
This commit is contained in:
Casper Sparre 2025-03-08 15:02:15 +01:00
parent cb5f901396
commit 049965b0fb
2 changed files with 84 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
OF DL/Helpers/DBHelper.cs
View File

@ -1,18 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using OF_DL.Enumurations;
using System.IO;
using Microsoft.Data.Sqlite; using Microsoft.Data.Sqlite;
using Serilog;
using OF_DL.Entities; using OF_DL.Entities;
using Serilog;
using System.Text;
namespace OF_DL.Helpers namespace OF_DL.Helpers
{ {
public class DBHelper : IDBHelper public class DBHelper : IDBHelper
{ {
private static readonly Dictionary<string, SqliteConnection> _connections = [];
private readonly IDownloadConfig downloadConfig; private readonly IDownloadConfig downloadConfig;
public DBHelper(IDownloadConfig downloadConfig) public DBHelper(IDownloadConfig downloadConfig)
@ -32,9 +28,7 @@ namespace OF_DL.Helpers
string dbFilePath = $"{folder}/Metadata/user_data.db"; string dbFilePath = $"{folder}/Metadata/user_data.db";
// connect to the new database file // connect to the new database file
using SqliteConnection connection = new($"Data Source={dbFilePath}"); SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={dbFilePath}");
// open the connection
connection.Open();
// create the 'medias' table // create the 'medias' table
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection)) using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection))
@ -139,11 +133,9 @@ namespace OF_DL.Helpers
{ {
try try
{ {
using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db"); SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
Log.Debug("Database data source: " + connection.DataSource); Log.Debug("Database data source: " + connection.DataSource);
connection.Open();
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection)) using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection))
{ {
await cmd.ExecuteNonQueryAsync(); await cmd.ExecuteNonQueryAsync();
@ -194,9 +186,7 @@ namespace OF_DL.Helpers
{ {
try try
{ {
using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db"); SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
connection.Open();
using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection)) using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection))
{ {
@ -247,8 +237,8 @@ namespace OF_DL.Helpers
{ {
try try
{ {
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
await EnsureCreatedAtColumnExists(connection, "messages"); await EnsureCreatedAtColumnExists(connection, "messages");
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection); using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection);
cmd.Parameters.AddWithValue("@post_id", post_id); cmd.Parameters.AddWithValue("@post_id", post_id);
@ -286,8 +276,8 @@ namespace OF_DL.Helpers
{ {
try try
{ {
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
await EnsureCreatedAtColumnExists(connection, "posts"); await EnsureCreatedAtColumnExists(connection, "posts");
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection); using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection);
cmd.Parameters.AddWithValue("@post_id", post_id); cmd.Parameters.AddWithValue("@post_id", post_id);
@ -324,8 +314,8 @@ namespace OF_DL.Helpers
{ {
try try
{ {
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
await EnsureCreatedAtColumnExists(connection, "stories"); await EnsureCreatedAtColumnExists(connection, "stories");
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection); using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection);
cmd.Parameters.AddWithValue("@post_id", post_id); cmd.Parameters.AddWithValue("@post_id", post_id);
@ -362,8 +352,8 @@ namespace OF_DL.Helpers
{ {
try try
{ {
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
await EnsureCreatedAtColumnExists(connection, "medias"); await EnsureCreatedAtColumnExists(connection, "medias");
StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id"); StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id");
if (downloadConfig.DownloadDuplicatedMedia) if (downloadConfig.DownloadDuplicatedMedia)
@ -400,22 +390,21 @@ namespace OF_DL.Helpers
{ {
try try
{ {
bool downloaded = false; SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
{
StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id"); StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id");
if(downloadConfig.DownloadDuplicatedMedia) if (downloadConfig.DownloadDuplicatedMedia)
{ {
sql.Append(" and api_type=@api_type"); sql.Append(" and api_type=@api_type");
} }
connection.Open(); connection.Open();
using SqliteCommand cmd = new (sql.ToString(), connection); using SqliteCommand cmd = new(sql.ToString(), connection);
cmd.Parameters.AddWithValue("@media_id", media_id); cmd.Parameters.AddWithValue("@media_id", media_id);
cmd.Parameters.AddWithValue("@api_type", api_type); cmd.Parameters.AddWithValue("@api_type", api_type);
downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
} bool downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
return downloaded; return downloaded;
} }
catch (Exception ex) catch (Exception ex)
@ -435,8 +424,7 @@ namespace OF_DL.Helpers
public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at) public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at)
{ {
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"); SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
// Construct the update command // Construct the update command
StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id"); StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id");
@ -463,24 +451,20 @@ namespace OF_DL.Helpers
public async Task<long> GetStoredFileSize(string folder, long media_id, string api_type) public async Task<long> GetStoredFileSize(string folder, long media_id, string api_type)
{ {
long size; SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
{
connection.Open();
using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection); using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection);
cmd.Parameters.AddWithValue("@media_id", media_id); cmd.Parameters.AddWithValue("@media_id", media_id);
cmd.Parameters.AddWithValue("@api_type", api_type); cmd.Parameters.AddWithValue("@api_type", api_type);
size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
} long size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
return size; return size;
} }
public async Task<DateTime?> GetMostRecentPostDate(string folder) public async Task<DateTime?> GetMostRecentPostDate(string folder)
{ {
DateTime? mostRecentDate = null; SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
{
connection.Open();
using SqliteCommand cmd = new(@" using SqliteCommand cmd = new(@"
SELECT SELECT
MIN(created_at) AS created_at MIN(created_at) AS created_at
@ -497,13 +481,14 @@ namespace OF_DL.Helpers
ON P.post_id = m.post_id ON P.post_id = m.post_id
WHERE m.downloaded = 0 WHERE m.downloaded = 0
)", connection); )", connection);
var scalarValue = await cmd.ExecuteScalarAsync(); var scalarValue = await cmd.ExecuteScalarAsync();
if(scalarValue != null && scalarValue != DBNull.Value) if (scalarValue != null && scalarValue != DBNull.Value)
{ {
mostRecentDate = Convert.ToDateTime(scalarValue); return Convert.ToDateTime(scalarValue);
} }
}
return mostRecentDate; return null;
} }
private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName) private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName)
@ -527,5 +512,35 @@ namespace OF_DL.Helpers
await alterCmd.ExecuteNonQueryAsync(); await alterCmd.ExecuteNonQueryAsync();
} }
} }
public static void CloseAllConnections()
{
foreach (SqliteConnection cn in _connections.Values)
{
cn?.Close();
cn?.Dispose();
}
_connections.Clear();
}
private static async Task<SqliteConnection> GetAndOpenConnectionAsync(string connectionString, int numberOfRetries = 2)
{
try
{
SqliteConnection connection = new(connectionString);
connection.Open();
return connection;
}
catch (Exception)
{
if (--numberOfRetries <= 0)
throw;
await Task.Delay(300);
return await GetAndOpenConnectionAsync(connectionString, numberOfRetries);
}
}
} }
} }

8
OF DL/Program.cs
View File

@ -806,8 +806,16 @@ public class Program
} }
AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]"); AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]");
try
{
await DownloadAllData(apiHelper, auth, config); await DownloadAllData(apiHelper, auth, config);
} }
finally
{
DBHelper.CloseAllConnections();
}
}
catch (Exception ex) catch (Exception ex)
{ {
Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace); Console.WriteLine("Exception caught: {0}\n\nStackTrace: {1}", ex.Message, ex.StackTrace);