diff --git a/.gitignore b/.gitignore index dec4dc1..06c7a2b 100644 --- a/.gitignore +++ b/.gitignore @@ -307,10 +307,6 @@ node_modules/ *.dsw *.dsp -# Visual Studio 6 technical files -*.ncb -*.aps - # Visual Studio LightSwitch build output **/*.HTMLClient/GeneratedArtifacts **/*.DesktopClient/GeneratedArtifacts @@ -404,14 +400,8 @@ FodyWeavers.xsd *.sln.iml .idea -## -## Visual studio for Mac -## - - # globs Makefile.in -*.userprefs *.usertasks config.make config.status @@ -470,16 +460,12 @@ ehthumbs_vista.db # Recycle Bin used on file shares $RECYCLE.BIN/ -# Windows Installer files -*.cab -*.msi -*.msix -*.msm -*.msp - # Windows shortcuts *.lnk # Vim temporary swap files *.swp -/Manager.App/Library/ + +# Manager.App +[Ll]ibrary/ +[Cc]ache/ diff --git a/Manager.App/Components/Dialogs/AccountDialog.razor b/Manager.App/Components/Dialogs/AccountDialog.razor index dbabe9b..5134465 100644 --- a/Manager.App/Components/Dialogs/AccountDialog.razor +++ b/Manager.App/Components/Dialogs/AccountDialog.razor @@ -1,4 +1,6 @@ +@using Manager.App.Services.System @inject ISnackbar SnackbarService +@inject CacheService Cache @@ -19,16 +21,15 @@ case AccountImportSteps.Authenticate: - Anonymous client + HelperText="Use an WEB user agent."/> Import cookies @($"{ImportCookies.Count} cookie(s) imported") - + @if (banner != null) { - + } else { @@ -85,7 +86,7 @@ @if (avatar != null) { - + } else { diff --git a/Manager.App/Components/Dialogs/AccountDialog.razor.cs b/Manager.App/Components/Dialogs/AccountDialog.razor.cs index f2809a8..8f1897b 100644 --- a/Manager.App/Components/Dialogs/AccountDialog.razor.cs +++ b/Manager.App/Components/Dialogs/AccountDialog.razor.cs @@ -10,7 +10,6 @@ namespace Manager.App.Components.Dialogs { [CascadingParameter] private IMudDialogInstance? MudDialog { get; set; } [Parameter] public string DefaultUserAgent { get; set; } = ""; - private bool IsAnonymous { get; set; } private ClientPrep? PreparingClient { get; set; } private CookieCollection ImportCookies { get; set; } = []; private bool _isLoading; @@ -22,12 +21,7 @@ namespace Manager.App.Components.Dialogs private bool CanSave() { - if (IsAnonymous || PreparingClient?.YouTubeClient?.State?.LoggedIn == true) - { - return true; - } - - return false; + return PreparingClient?.YouTubeClient?.State?.LoggedIn == true; } private bool CanContinue() @@ -35,13 +29,13 @@ namespace Manager.App.Components.Dialogs switch (_steps) { case AccountImportSteps.Authenticate: - if (IsAnonymous || ImportCookies.Count != 0) + if (ImportCookies.Count != 0) { return true; } break; case AccountImportSteps.Validate: - if (IsAnonymous || PreparingClient?.YouTubeClient?.State?.LoggedIn == true) + if (PreparingClient?.YouTubeClient?.State?.LoggedIn == true) { return true; } @@ -96,7 +90,6 @@ namespace Manager.App.Components.Dialogs { PreparingClient?.YouTubeClient?.Dispose(); PreparingClient = null; - IsAnonymous = false; ImportCookies.Clear(); _steps = AccountImportSteps.Authenticate; StateHasChanged(); @@ -150,10 +143,6 @@ namespace Manager.App.Components.Dialogs { _isLoading = true; PreparingClient = new ClientPrep(); - if (IsAnonymous) - { - ImportCookies.Clear(); - } var clientResult = await YouTubeClient.CreateAsync(ImportCookies, DefaultUserAgent); if (clientResult.IsSuccess) { diff --git a/Manager.App/Components/Pages/Channels.razor.cs b/Manager.App/Components/Pages/Channels.razor.cs index eb9d7f8..7ac4ec9 100644 --- a/Manager.App/Components/Pages/Channels.razor.cs +++ b/Manager.App/Components/Pages/Channels.razor.cs @@ -13,7 +13,7 @@ public partial class Channels : ComponentBase private async Task> ServerReload(TableState state, CancellationToken token) { - var results = await LibraryService.GetChannelAccountsAsync(state.PageSize, state.Page * state.PageSize, token); + var results = await LibraryService.GetChannelsAsync(state.PageSize, state.Page * state.PageSize, token); return !results.IsSuccess ? new TableData() : new TableData { Items = results.Value, TotalItems = results.Total }; } @@ -42,12 +42,11 @@ public partial class Channels : ComponentBase } else { + if (_table != null) + { + await _table.ReloadServerData(); + } Snackbar.Add($"Client {clientPrep.Channel?.Handle ?? clientPrep.YouTubeClient.Id} saved!", Severity.Success); } - - if (_table != null) - { - await _table.ReloadServerData(); - } } } \ No newline at end of file diff --git a/Manager.App/Components/Pages/Services.razor.cs b/Manager.App/Components/Pages/Services.razor.cs index aedcf9b..40e4f23 100644 --- a/Manager.App/Components/Pages/Services.razor.cs +++ b/Manager.App/Components/Pages/Services.razor.cs @@ -27,7 +27,9 @@ public partial class Services : ComponentBase private List GetInitialEvents() { var totalToGet = 1000 / _backgroundServices.Count; - var initial = _backgroundServices.SelectMany(x => x.ProgressEvents.Items.TakeLast(totalToGet)); + var initial = _backgroundServices + .SelectMany(x => x.ProgressEvents.Items.TakeLast(totalToGet)) + .OrderBy(x => x.DateUtc); return initial.ToList(); } diff --git a/Manager.App/Controllers/CacheController.cs b/Manager.App/Controllers/CacheController.cs new file mode 100644 index 0000000..f77fde4 --- /dev/null +++ b/Manager.App/Controllers/CacheController.cs @@ -0,0 +1,27 @@ +using Manager.App.Services.System; +using Microsoft.AspNetCore.Mvc; + +namespace Manager.App.Controllers; + +[ApiController] +[Route("api/v1/[controller]")] +public class CacheController(ILogger logger, CacheService cacheService) : ControllerBase +{ + [HttpGet] + public async Task Cache([FromQuery(Name = "url")] string url) + { + if (string.IsNullOrWhiteSpace(url)) + { + return BadRequest("No url given."); + } + + var cacheResult = await cacheService.CacheFromUrl(url); + if (!cacheResult.IsSuccess) + { + logger.LogError("Cache request failed. {ErrorMessage}", cacheResult.Error?.Description); + return StatusCode(500, cacheResult.Error?.Description); + } + + return File(cacheResult.Value.Data, cacheResult.Value.ContentType ?? string.Empty, cacheResult.Value.OriginalFileName); + } +} \ No newline at end of file diff --git a/Manager.App/Manager.App.csproj b/Manager.App/Manager.App.csproj index 6a36124..397483b 100644 --- a/Manager.App/Manager.App.csproj +++ b/Manager.App/Manager.App.csproj @@ -32,6 +32,7 @@ + diff --git a/Manager.App/Program.cs b/Manager.App/Program.cs index 0f530d6..db83305 100644 --- a/Manager.App/Program.cs +++ b/Manager.App/Program.cs @@ -10,6 +10,8 @@ builder.Services.AddRazorComponents() AppContext.SetSwitch("System.Net.Http.EnableActivityPropagation", false); +builder.Services.AddControllers(); + /* Manager */ builder.SetupLogging(); builder.SetupSettings(); @@ -32,6 +34,7 @@ app.UseHttpsRedirection(); app.UseStaticFiles(); app.UseAntiforgery(); +app.MapControllers(); app.MapRazorComponents() .AddInteractiveServerRenderMode(); diff --git a/Manager.App/Services/ExtendedBackgroundService.cs b/Manager.App/Services/ExtendedBackgroundService.cs index 1a0e742..ac96673 100644 --- a/Manager.App/Services/ExtendedBackgroundService.cs +++ b/Manager.App/Services/ExtendedBackgroundService.cs @@ -29,9 +29,10 @@ public abstract class ExtendedBackgroundService(string name, string description, _resumeSignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); await _resumeSignal.Task.WaitAsync(stoppingToken); } + + await ExecuteServiceAsync(stoppingToken); await Task.Delay(ExecuteInterval, stoppingToken); - await ExecuteServiceAsync(stoppingToken); } } catch (Exception e) diff --git a/Manager.App/Services/ILibraryService.cs b/Manager.App/Services/ILibraryService.cs index 42c16a0..ee4f5ff 100644 --- a/Manager.App/Services/ILibraryService.cs +++ b/Manager.App/Services/ILibraryService.cs @@ -13,5 +13,5 @@ public interface ILibraryService public Task SaveChannelAsync(ChannelEntity channel, CancellationToken cancellationToken = default); public Task> GetLibraryInfoAsync(CancellationToken cancellationToken = default); - public Task> GetChannelAccountsAsync(int total = 20, int offset = 0, CancellationToken cancellationToken = default); + public Task> GetChannelsAsync(int total = 20, int offset = 0, CancellationToken cancellationToken = default); } \ No newline at end of file diff --git a/Manager.App/Services/LibraryService.cs b/Manager.App/Services/LibraryService.cs index 409c3a3..f2004a5 100644 --- a/Manager.App/Services/LibraryService.cs +++ b/Manager.App/Services/LibraryService.cs @@ -168,7 +168,7 @@ public class LibraryService : ILibraryService } } - public async Task> GetChannelAccountsAsync(int total = 20, int offset = 0, CancellationToken cancellationToken = default) + public async Task> GetChannelsAsync(int total = 20, int offset = 0, CancellationToken cancellationToken = default) { try { diff --git a/Manager.App/Services/System/CacheService.cs b/Manager.App/Services/System/CacheService.cs new file mode 100644 index 0000000..7e8f423 --- /dev/null +++ b/Manager.App/Services/System/CacheService.cs @@ -0,0 +1,151 @@ +using System.Security.Cryptography; +using System.Text; +using DotBased.Logging; +using DotBased.Monads; +using Manager.Data.Contexts; +using Manager.Data.Entities.Cache; +using Manager.YouTube; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace Manager.App.Services.System; + +public class CacheService(ILogger logger, IHostEnvironment environment) : ExtendedBackgroundService(nameof(CacheService), "Manages caching.", logger, TimeSpan.FromDays(1)) +{ + private DirectoryInfo? _cacheDirectory; + private PooledDbContextFactory? _dbContextFactory; + private const string DataSubDir = "data"; + + protected override Task InitializeAsync(CancellationToken stoppingToken) + { + _cacheDirectory = new DirectoryInfo(Path.Combine(Directory.GetCurrentDirectory(), "cache")); + _cacheDirectory.Create(); + Directory.CreateDirectory(Path.Combine(_cacheDirectory.FullName, DataSubDir)); + LogEvent($"Cache directory: {_cacheDirectory.FullName}"); + + var dbContextOptionsBuilder = new DbContextOptionsBuilder(); + dbContextOptionsBuilder.UseSqlite($"Data Source={Path.Combine(_cacheDirectory.FullName, "cache_index.db")}"); + _dbContextFactory = new PooledDbContextFactory(dbContextOptionsBuilder.Options); + + return Task.CompletedTask; + } + + protected override async Task ExecuteServiceAsync(CancellationToken stoppingToken) + { + if (environment.IsDevelopment()) + { + LogEvent("Development mode detected, skipping cache cleaning..."); + return; + } + + try + { + await ClearCacheAsync(stoppingToken); + } + catch (Exception e) + { + logger.LogError(e, "Error in execution of service."); + LogEvent($"Service execution failed. {e.Message}", LogSeverity.Error); + throw; + } + } + + public string CreateCacheUrl(string originalUrl) => $"/api/v1/cache?url={originalUrl}"; + + public async Task> CacheFromUrl(string url, CancellationToken cancellationToken = default) + { + try + { + if (string.IsNullOrWhiteSpace(url)) + { + return ResultError.Fail("Url is empty."); + } + + if (_cacheDirectory == null) + { + return ResultError.Fail("Cache directory is not initialized."); + } + + if (_dbContextFactory == null) + { + return ResultError.Fail("Context factory is not initialized."); + } + + await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken); + + var urlKeyBytes = SHA1.HashData(Encoding.UTF8.GetBytes(url)); + var urlKey = Convert.ToHexString(urlKeyBytes); + var cacheEntity = await context.Cache.FirstOrDefaultAsync(c => c.Id == urlKey, cancellationToken: cancellationToken); + if (cacheEntity == null) + { + var downloadResult = await NetworkService.DownloadBytesAsync(new HttpRequestMessage(HttpMethod.Get, url)); + if (!downloadResult.IsSuccess) + { + LogEvent($"Failed to download from url: {url}"); + return ResultError.Fail("Download failed."); + } + var download = downloadResult.Value; + await using var downloadFile = File.Create(Path.Combine(_cacheDirectory.FullName, DataSubDir, $"{urlKey}.cache")); + await downloadFile.WriteAsync(download.Data.AsMemory(0, download.Data.Length), cancellationToken); + + cacheEntity = new CacheEntity + { + Id = urlKey, + CachedAtUtc = DateTime.UtcNow, + ContentLength = download.ContentLength, + ContentType = download.ContentType, + OriginalFileName = download.FileName, + }; + + context.Cache.Add(cacheEntity); + var saved = await context.SaveChangesAsync(cancellationToken); + if (saved <= 0) + { + LogEvent($"Cache entity {cacheEntity.Id} could not be saved.", LogSeverity.Error); + return ResultError.Fail("Failed to save to cache db."); + } + + return new CacheFile(download.Data, download.ContentType, download.FileName); + } + + var filePath = Path.Combine(_cacheDirectory.FullName, DataSubDir, $"{urlKey}.cache"); + var buffer = await File.ReadAllBytesAsync(filePath, cancellationToken); + if (buffer.Length == 0) + { + LogEvent($"Failed to read data from disk. File: {filePath}", LogSeverity.Error); + return ResultError.Fail($"Error reading data from disk. File: {filePath}"); + } + + return new CacheFile(buffer.ToArray(), cacheEntity.ContentType, cacheEntity.OriginalFileName); + } + catch (Exception e) + { + return ResultError.Error(e, "Cache error."); + } + } + + private async Task ClearCacheAsync(CancellationToken cancellationToken) + { + if (_dbContextFactory == null) + { + throw new InvalidOperationException("No DbContext factory configured."); + } + + await using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken); + + var toRemove = dbContext.Cache.Where(c => c.CachedAtUtc >= DateTime.UtcNow.AddDays(-1)); + if (!toRemove.Any()) + { + LogEvent("No items found to purge from cache."); + return; + } + + var totalToRemove = toRemove.Count(); + LogEvent($"Found {totalToRemove} cache items that are older than 1 day(s)"); + dbContext.RemoveRange(toRemove); + var deleted = await dbContext.SaveChangesAsync(cancellationToken); + LogEvent($"Removed {deleted}/{totalToRemove} items"); + } +} + +public record CacheFile(byte[] Data, string? ContentType, string? OriginalFileName); \ No newline at end of file diff --git a/Manager.App/Services/System/ClientService.cs b/Manager.App/Services/System/ClientService.cs index 0661b2b..f91b83f 100644 --- a/Manager.App/Services/System/ClientService.cs +++ b/Manager.App/Services/System/ClientService.cs @@ -10,7 +10,7 @@ namespace Manager.App.Services.System; public class ClientService(IServiceScopeFactory scopeFactory, ILogger logger) : ExtendedBackgroundService(nameof(ClientService), "Managing YouTube clients", logger, TimeSpan.FromMinutes(10)) { - private readonly List _clients = []; + private readonly List _loadedClients = []; private CancellationToken _cancellationToken; private ILibraryService? _libraryService; diff --git a/Manager.Data/Contexts/CacheDbContext.cs b/Manager.Data/Contexts/CacheDbContext.cs new file mode 100644 index 0000000..c8f76ad --- /dev/null +++ b/Manager.Data/Contexts/CacheDbContext.cs @@ -0,0 +1,25 @@ +using Manager.Data.Entities.Cache; +using Microsoft.EntityFrameworkCore; + +namespace Manager.Data.Contexts; + +public sealed class CacheDbContext : DbContext +{ + public CacheDbContext(DbContextOptions options) : base(options) + { + Database.EnsureCreated(); + } + + public DbSet Cache { get; set; } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(ce => + { + ce.ToTable("cache"); + ce.HasKey(x => x.Id); + }); + + base.OnModelCreating(modelBuilder); + } +} \ No newline at end of file diff --git a/Manager.YouTube/NetworkService.cs b/Manager.YouTube/NetworkService.cs index 18944d2..2353946 100644 --- a/Manager.YouTube/NetworkService.cs +++ b/Manager.YouTube/NetworkService.cs @@ -6,7 +6,7 @@ namespace Manager.YouTube; public static class NetworkService { public const string Origin = "https://www.youtube.com"; - private static readonly HttpClient HttpClient = new HttpClient(); + private static readonly HttpClient HttpClient = new(); public static async Task> MakeRequestAsync(HttpRequestMessage request, YouTubeClient client, bool skipAuthenticationHeader = false) { @@ -43,7 +43,7 @@ public static class NetworkService return ResultError.Fail($"Failed to get file to download, response code: {response.StatusCode}."); } - var data = await response.Content.ReadAsByteArrayAsync();; + var data = await response.Content.ReadAsByteArrayAsync(); return new DownloadResult(data, response.Content.Headers.ContentType?.MediaType, response.Content.Headers.ContentDisposition?.FileName?.Trim('"'), response.Content.Headers.ContentLength ?? 0); }