reptail/OF-DL
1
0
Fork 0
forked from sim0n00ps/OF-DL
Code Pull Requests Activity

Improved DB connection creation with delayed retry, and connection caching

Browse Source
This commit is contained in:
Casper Sparre 2025-03-08 15:02:15 +01:00
parent e649751c31
commit 98b95d2aeb
2 changed files with 84 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

135
OF DL/Helpers/DBHelper.cs
View File

@ -1,18 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using OF_DL.Enumurations;
using System.IO;
using Microsoft.Data.Sqlite;
using Serilog;
using OF_DL.Entities;
using Serilog;
using System.Text;
namespace OF_DL.Helpers
{
public class DBHelper : IDBHelper
{
private static readonly Dictionary<string, SqliteConnection> _connections = [];
private readonly IDownloadConfig downloadConfig;
public DBHelper(IDownloadConfig downloadConfig)
@ -32,9 +28,7 @@ namespace OF_DL.Helpers
string dbFilePath = $"{folder}/Metadata/user_data.db";
// connect to the new database file
using SqliteConnection connection = new($"Data Source={dbFilePath}");
// open the connection
connection.Open();
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={dbFilePath}");
// create the 'medias' table
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS medias (id INTEGER NOT NULL, media_id INTEGER, post_id INTEGER NOT NULL, link VARCHAR, directory VARCHAR, filename VARCHAR, size INTEGER, api_type VARCHAR, media_type VARCHAR, preview INTEGER, linked VARCHAR, downloaded INTEGER, created_at TIMESTAMP, record_created_at TIMESTAMP, PRIMARY KEY(id), UNIQUE(media_id));", connection))
@ -139,11 +133,9 @@ namespace OF_DL.Helpers
{
try
{
using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
Log.Debug("Database data source: " + connection.DataSource);
connection.Open();
using (SqliteCommand cmd = new("CREATE TABLE IF NOT EXISTS users (id INTEGER NOT NULL, user_id INTEGER NOT NULL, username VARCHAR NOT NULL, PRIMARY KEY(id), UNIQUE(username));", connection))
{
await cmd.ExecuteNonQueryAsync();
@ -194,9 +186,7 @@ namespace OF_DL.Helpers
{
try
{
using SqliteConnection connection = new($"Data Source={Directory.GetCurrentDirectory()}/users.db");
connection.Open();
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={Directory.GetCurrentDirectory()}/users.db");
using (SqliteCommand checkCmd = new($"SELECT user_id, username FROM users WHERE user_id = @userId;", connection))
{
@ -247,8 +237,8 @@ namespace OF_DL.Helpers
{
try
{
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
await EnsureCreatedAtColumnExists(connection, "messages");
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM messages WHERE post_id=@post_id", connection);
cmd.Parameters.AddWithValue("@post_id", post_id);
@ -286,8 +276,8 @@ namespace OF_DL.Helpers
{
try
{
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
await EnsureCreatedAtColumnExists(connection, "posts");
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM posts WHERE post_id=@post_id", connection);
cmd.Parameters.AddWithValue("@post_id", post_id);
@ -324,8 +314,8 @@ namespace OF_DL.Helpers
{
try
{
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
await EnsureCreatedAtColumnExists(connection, "stories");
using SqliteCommand cmd = new($"SELECT COUNT(*) FROM stories WHERE post_id=@post_id", connection);
cmd.Parameters.AddWithValue("@post_id", post_id);
@ -362,8 +352,8 @@ namespace OF_DL.Helpers
{
try
{
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
await EnsureCreatedAtColumnExists(connection, "medias");
StringBuilder sql = new StringBuilder("SELECT COUNT(*) FROM medias WHERE media_id=@media_id");
if (downloadConfig.DownloadDuplicatedMedia)
@ -400,22 +390,21 @@ namespace OF_DL.Helpers
{
try
{
bool downloaded = false;
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id");
if (downloadConfig.DownloadDuplicatedMedia)
{
StringBuilder sql = new StringBuilder("SELECT downloaded FROM medias WHERE media_id=@media_id");
if(downloadConfig.DownloadDuplicatedMedia)
{
sql.Append(" and api_type=@api_type");
}
connection.Open();
using SqliteCommand cmd = new (sql.ToString(), connection);
cmd.Parameters.AddWithValue("@media_id", media_id);
cmd.Parameters.AddWithValue("@api_type", api_type);
downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
sql.Append(" and api_type=@api_type");
}
connection.Open();
using SqliteCommand cmd = new(sql.ToString(), connection);
cmd.Parameters.AddWithValue("@media_id", media_id);
cmd.Parameters.AddWithValue("@api_type", api_type);
bool downloaded = Convert.ToBoolean(await cmd.ExecuteScalarAsync());
return downloaded;
}
catch (Exception ex)
@ -435,8 +424,7 @@ namespace OF_DL.Helpers
public async Task UpdateMedia(string folder, long media_id, string api_type, string directory, string filename, long size, bool downloaded, DateTime created_at)
{
using SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db");
connection.Open();
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
// Construct the update command
StringBuilder sql = new StringBuilder("UPDATE medias SET directory=@directory, filename=@filename, size=@size, downloaded=@downloaded, created_at=@created_at WHERE media_id=@media_id");
@ -463,25 +451,21 @@ namespace OF_DL.Helpers
public async Task<long> GetStoredFileSize(string folder, long media_id, string api_type)
{
long size;
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
{
connection.Open();
using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection);
cmd.Parameters.AddWithValue("@media_id", media_id);
cmd.Parameters.AddWithValue("@api_type", api_type);
size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
}
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
using SqliteCommand cmd = new($"SELECT size FROM medias WHERE media_id=@media_id and api_type=@api_type", connection);
cmd.Parameters.AddWithValue("@media_id", media_id);
cmd.Parameters.AddWithValue("@api_type", api_type);
long size = Convert.ToInt64(await cmd.ExecuteScalarAsync());
return size;
}
public async Task<DateTime?> GetMostRecentPostDate(string folder)
{
DateTime? mostRecentDate = null;
using (SqliteConnection connection = new($"Data Source={folder}/Metadata/user_data.db"))
{
connection.Open();
using SqliteCommand cmd = new(@"
SqliteConnection connection = await GetAndOpenConnectionAsync($"Data Source={folder}/Metadata/user_data.db");
using SqliteCommand cmd = new(@"
SELECT
MIN(created_at) AS created_at
FROM (
@ -497,13 +481,14 @@ namespace OF_DL.Helpers
ON P.post_id = m.post_id
WHERE m.downloaded = 0
)", connection);
var scalarValue = await cmd.ExecuteScalarAsync();
if(scalarValue != null && scalarValue != DBNull.Value)
{
mostRecentDate = Convert.ToDateTime(scalarValue);
}
var scalarValue = await cmd.ExecuteScalarAsync();
if (scalarValue != null && scalarValue != DBNull.Value)
{
return Convert.ToDateTime(scalarValue);
}
return mostRecentDate;
return null;
}
private async Task EnsureCreatedAtColumnExists(SqliteConnection connection, string tableName)
@ -527,5 +512,35 @@ namespace OF_DL.Helpers
await alterCmd.ExecuteNonQueryAsync();
}
}
public static void CloseAllConnections()
{
foreach (SqliteConnection cn in _connections.Values)
{
cn?.Close();
cn?.Dispose();
}
_connections.Clear();
}
private static async Task<SqliteConnection> GetAndOpenConnectionAsync(string connectionString, int numberOfRetries = 2)
{
try
{
SqliteConnection connection = new(connectionString);
connection.Open();
return connection;
}
catch (Exception)
{
if (--numberOfRetries <= 0)
throw;
await Task.Delay(300);
return await GetAndOpenConnectionAsync(connectionString, numberOfRetries);
}
}
}
}

10
OF DL/Program.cs
View File

@ -806,7 +806,15 @@ public class Program
}
AnsiConsole.Markup($"[green]Logged In successfully as {validate.name} {validate.username}\n[/]");
await DownloadAllData(apiHelper, auth, config);
try
{
await DownloadAllData(apiHelper, auth, config);
}
finally
{
DBHelper.CloseAllConnections();
}
}
catch (Exception ex)
{