Compare commits

...

5 Commits

Author SHA1 Message Date
max
79ea2badf1 [FIX] EventConsole batch fix 2025-09-18 02:10:13 +02:00
max
5250b9f3f9 [REWORK] Changes saving client and channel info 2025-09-18 02:01:45 +02:00
max
9e173258ed [CHANGES] 2025-09-18 00:42:17 +02:00
max
ab532ac6dc [CHANGE] Fixed cache service && Download channel images from cache 2025-09-18 00:30:35 +02:00
max
8a64d6fc64 [CHANGE] Remove anon accounts && added simple caching for urls 2025-09-17 23:44:02 +02:00
23 changed files with 490 additions and 231 deletions

22
.gitignore vendored
View File

@@ -307,10 +307,6 @@ node_modules/
*.dsw
*.dsp
# Visual Studio 6 technical files
*.ncb
*.aps
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
@@ -404,14 +400,8 @@ FodyWeavers.xsd
*.sln.iml
.idea
##
## Visual studio for Mac
##
# globs
Makefile.in
*.userprefs
*.usertasks
config.make
config.status
@@ -470,16 +460,12 @@ ehthumbs_vista.db
# Recycle Bin used on file shares
$RECYCLE.BIN/
# Windows Installer files
*.cab
*.msi
*.msix
*.msm
*.msp
# Windows shortcuts
*.lnk
# Vim temporary swap files
*.swp
/Manager.App/Library/
# Manager.App
[Ll]ibrary/
[Cc]ache/

View File

@@ -107,7 +107,7 @@ public partial class EventConsole : ComponentBase
_batchLock.Release();
}
foreach (var serviceEvent in _batchBuffer.Where(serviceEvent => !_serviceEvents.Contains(serviceEvent)))
foreach (var serviceEvent in batch.Where(serviceEvent => !_serviceEvents.Contains(serviceEvent)))
{
_serviceEvents.Add(serviceEvent);
}

View File

@@ -1,11 +1,13 @@
@using Manager.App.Services.System
@inject ISnackbar SnackbarService
@inject CacheService Cache
<ForcedLoadingOverlay Visible="_isLoading"/>
@{
var client = PreparingClient?.YouTubeClient;
var client = ClientChannel?.YouTubeClient;
var clientState = client?.State;
var channel = PreparingClient?.Channel;
var channel = ClientChannel?.Channel;
var avatar = channel?.AvatarImages.FirstOrDefault();
var banner = channel?.BannerImages.FirstOrDefault();
}
@@ -19,16 +21,15 @@
case AccountImportSteps.Authenticate:
<MudStack Spacing="2">
<MudPaper Elevation="0" Outlined Class="pa-2">
<MudSwitch @bind-Value="@IsAnonymous" Color="Color.Info">Anonymous client</MudSwitch>
<MudTextField @bind-Value="@DefaultUserAgent" Required Label="User agent"
HelperText="Use an WEB client user agent."/>
HelperText="Use an WEB user agent."/>
</MudPaper>
<MudStack Row Spacing="2" Style="height: 100%">
<MudPaper Elevation="0" Outlined Class="pa-2" Style="width: 50%;">
<MudText>Import cookies</MudText>
<MudText Typo="Typo.caption">@($"{ImportCookies.Count} cookie(s) imported")</MudText>
<MudForm @bind-IsValid="@_cookieImportTextValid" Disabled="@(IsAnonymous)">
<MudForm @bind-IsValid="@_cookieImportTextValid">
<MudTextField @bind-Value="@_cookieDomain" Immediate Required Label="Domain"
RequiredError="Domain is required."/>
<MudTextField Class="my-2" Lines="4" AutoGrow @bind-Value="@_cookieText" Immediate
@@ -76,7 +77,7 @@
<MudPaper Elevation="0">
@if (banner != null)
{
<MudImage Src="@banner.Url" Height="250" Style="width: 100%;"/>
<MudImage Src="@Cache.CreateCacheUrl(banner.Url)" Height="250" Style="width: 100%;"/>
}
else
{
@@ -85,7 +86,7 @@
<MudStack Row Spacing="3" Class="px-4">
@if (avatar != null)
{
<MudImage Src="@avatar.Url" Class="mt-n5" Height="100" Width="100"/>
<MudImage Src="@Cache.CreateCacheUrl(avatar.Url)" Class="mt-n5" Height="100" Width="100"/>
}
else
{

View File

@@ -10,8 +10,7 @@ namespace Manager.App.Components.Dialogs
{
[CascadingParameter] private IMudDialogInstance? MudDialog { get; set; }
[Parameter] public string DefaultUserAgent { get; set; } = "";
private bool IsAnonymous { get; set; }
private ClientPrep? PreparingClient { get; set; }
private ClientChannel? ClientChannel { get; set; }
private CookieCollection ImportCookies { get; set; } = [];
private bool _isLoading;
private AccountImportSteps _steps = AccountImportSteps.Authenticate;
@@ -22,12 +21,7 @@ namespace Manager.App.Components.Dialogs
private bool CanSave()
{
if (IsAnonymous || PreparingClient?.YouTubeClient?.State?.LoggedIn == true)
{
return true;
}
return false;
return ClientChannel?.YouTubeClient?.State?.LoggedIn == true;
}
private bool CanContinue()
@@ -35,13 +29,13 @@ namespace Manager.App.Components.Dialogs
switch (_steps)
{
case AccountImportSteps.Authenticate:
if (IsAnonymous || ImportCookies.Count != 0)
if (ImportCookies.Count != 0)
{
return true;
}
break;
case AccountImportSteps.Validate:
if (IsAnonymous || PreparingClient?.YouTubeClient?.State?.LoggedIn == true)
if (ClientChannel?.YouTubeClient?.State?.LoggedIn == true)
{
return true;
}
@@ -67,7 +61,7 @@ namespace Manager.App.Components.Dialogs
case AccountImportSteps.Validate:
if (CanSave())
{
MudDialog?.Close(DialogResult.Ok(PreparingClient));
MudDialog?.Close(DialogResult.Ok(ClientChannel));
await InvokeAsync(StateHasChanged);
return;
}
@@ -94,9 +88,8 @@ namespace Manager.App.Components.Dialogs
private void ClearPreparedClient()
{
PreparingClient?.YouTubeClient?.Dispose();
PreparingClient = null;
IsAnonymous = false;
ClientChannel?.YouTubeClient?.Dispose();
ClientChannel = null;
ImportCookies.Clear();
_steps = AccountImportSteps.Authenticate;
StateHasChanged();
@@ -149,28 +142,24 @@ namespace Manager.App.Components.Dialogs
private async Task BuildClient()
{
_isLoading = true;
PreparingClient = new ClientPrep();
if (IsAnonymous)
{
ImportCookies.Clear();
}
ClientChannel = new ClientChannel();
var clientResult = await YouTubeClient.CreateAsync(ImportCookies, DefaultUserAgent);
if (clientResult.IsSuccess)
{
PreparingClient.YouTubeClient = clientResult.Value;
ClientChannel.YouTubeClient = clientResult.Value;
}
if (PreparingClient.YouTubeClient == null)
if (ClientChannel.YouTubeClient == null)
{
SnackbarService.Add("Failed to get client!", Severity.Error);
_isLoading = false;
return;
}
var accountResult = await PreparingClient.YouTubeClient.GetChannelByIdAsync(PreparingClient.YouTubeClient.Id);
var accountResult = await ClientChannel.YouTubeClient.GetChannelByIdAsync(ClientChannel.YouTubeClient.Id);
if (accountResult.IsSuccess)
{
PreparingClient.Channel = accountResult.Value;
ClientChannel.Channel = accountResult.Value;
}
_isLoading = false;
await InvokeAsync(StateHasChanged);

View File

@@ -13,7 +13,7 @@ public partial class Channels : ComponentBase
private async Task<TableData<ChannelEntity>> ServerReload(TableState state, CancellationToken token)
{
var results = await LibraryService.GetChannelAccountsAsync(state.PageSize, state.Page * state.PageSize, token);
var results = await LibraryService.GetChannelsAsync(state.PageSize, state.Page * state.PageSize, token);
return !results.IsSuccess ? new TableData<ChannelEntity>() : new TableData<ChannelEntity> { Items = results.Value, TotalItems = results.Total };
}
@@ -29,25 +29,39 @@ public partial class Channels : ComponentBase
return;
}
var clientPrep = (ClientPrep)result.Data;
if (clientPrep?.YouTubeClient == null)
var clientChannel = (ClientChannel)result.Data;
if (clientChannel?.YouTubeClient == null)
{
Snackbar.Add("No YouTube client received.", Severity.Error);
return;
}
var savedResult = await ClientService.SaveClientAsync(clientPrep.YouTubeClient, clientPrep.Channel);
if (!savedResult.IsSuccess)
var savedClientResult = await ClientService.SaveClientAsync(clientChannel.YouTubeClient);
if (savedClientResult.IsSuccess)
{
Snackbar.Add($"Failed to store client: {savedResult.Error?.Description ?? "Unknown!"}", Severity.Error);
if (_table != null)
{
await _table.ReloadServerData();
}
Snackbar.Add($"Client {clientChannel.Channel?.Handle ?? clientChannel.YouTubeClient.Id} saved!", Severity.Success);
ClientService.AddClient(clientChannel.YouTubeClient);
}
else
{
Snackbar.Add($"Client {clientPrep.Channel?.Handle ?? clientPrep.YouTubeClient.Id} saved!", Severity.Success);
Snackbar.Add($"Failed to store client: {savedClientResult.Error?.Description ?? "Unknown!"}", Severity.Error);
}
if (_table != null)
if (clientChannel.Channel == null)
{
await _table.ReloadServerData();
Snackbar.Add("No channel information received!", Severity.Warning);
}
else
{
var saveChannelResult = await LibraryService.SaveChannelAsync(clientChannel.Channel);
if (!saveChannelResult.IsSuccess)
{
Snackbar.Add("Failed to save channel information", Severity.Warning);
}
}
}
}

View File

@@ -27,7 +27,9 @@ public partial class Services : ComponentBase
private List<ServiceEvent> GetInitialEvents()
{
var totalToGet = 1000 / _backgroundServices.Count;
var initial = _backgroundServices.SelectMany(x => x.ProgressEvents.Items.TakeLast(totalToGet));
var initial = _backgroundServices
.SelectMany(x => x.ProgressEvents.Items.TakeLast(totalToGet))
.OrderBy(x => x.DateUtc);
return initial.ToList();
}

View File

@@ -0,0 +1,27 @@
using Manager.App.Services.System;
using Microsoft.AspNetCore.Mvc;
namespace Manager.App.Controllers;
[ApiController]
[Route("api/v1/[controller]")]
public class CacheController(ILogger<CacheController> logger, CacheService cacheService) : ControllerBase
{
[HttpGet]
public async Task<IActionResult> Cache([FromQuery(Name = "url")] string url, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(url))
{
return BadRequest("No url given.");
}
var cacheResult = await cacheService.CacheFromUrl(url, cancellationToken);
if (!cacheResult.IsSuccess)
{
logger.LogError("Cache request failed. {ErrorMessage}", cacheResult.Error?.Description);
return StatusCode(500, cacheResult.Error?.Description);
}
return File(cacheResult.Value.Data, cacheResult.Value.ContentType ?? string.Empty, cacheResult.Value.OriginalFileName);
}
}

View File

@@ -32,6 +32,7 @@
</ItemGroup>
<ItemGroup>
<Folder Include="cache\" />
<Folder Include="Library\" />
<Folder Include="Logs\Debug\" />
</ItemGroup>

View File

@@ -1,10 +0,0 @@
using Manager.YouTube;
using Manager.YouTube.Models.Innertube;
namespace Manager.App.Models.Library;
public class ClientPrep
{
public YouTubeClient? YouTubeClient { get; set; }
public Channel? Channel { get; set; }
}

View File

@@ -10,6 +10,8 @@ builder.Services.AddRazorComponents()
AppContext.SetSwitch("System.Net.Http.EnableActivityPropagation", false);
builder.Services.AddControllers();
/* Manager */
builder.SetupLogging();
builder.SetupSettings();
@@ -32,6 +34,7 @@ app.UseHttpsRedirection();
app.UseStaticFiles();
app.UseAntiforgery();
app.MapControllers();
app.MapRazorComponents<App>()
.AddInteractiveServerRenderMode();

View File

@@ -29,9 +29,10 @@ public abstract class ExtendedBackgroundService(string name, string description,
_resumeSignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
await _resumeSignal.Task.WaitAsync(stoppingToken);
}
await ExecuteServiceAsync(stoppingToken);
await Task.Delay(ExecuteInterval, stoppingToken);
await ExecuteServiceAsync(stoppingToken);
}
}
catch (Exception e)
@@ -57,7 +58,7 @@ public abstract class ExtendedBackgroundService(string name, string description,
if (State == ServiceState.Running)
{
State = ServiceState.Paused;
logger.LogInformation("Pauses service: {ServiceName}", Name);
LogEvent("Service paused.");
}
}
@@ -67,7 +68,7 @@ public abstract class ExtendedBackgroundService(string name, string description,
{
State = ServiceState.Running;
_resumeSignal.TrySetResult();
logger.LogInformation("Resumed service: {ServiceName}", Name);
LogEvent("Service resumed.");
}
}

View File

@@ -8,10 +8,12 @@ namespace Manager.App.Services;
public interface ILibraryService
{
public Task<Result> FetchChannelImagesAsync(Channel channel);
public Task<Result> FetchChannelImagesAsync(InnertubeChannel innertubeChannel);
public Task<Result> SaveClientAsync(ClientAccountEntity client, CancellationToken cancellationToken = default);
public Task<Result<ChannelEntity>> GetChannelByIdAsync(string id, CancellationToken cancellationToken = default);
public Task<Result> SaveChannelAsync(ChannelEntity channel, CancellationToken cancellationToken = default);
public Task<Result> SaveChannelAsync(InnertubeChannel innertubeChannel, CancellationToken cancellationToken = default);
public Task<Result<LibraryInformation>> GetLibraryInfoAsync(CancellationToken cancellationToken = default);
public Task<ListResult<ChannelEntity>> GetChannelAccountsAsync(int total = 20, int offset = 0, CancellationToken cancellationToken = default);
public Task<ListResult<ChannelEntity>> GetChannelsAsync(int total = 20, int offset = 0, CancellationToken cancellationToken = default);
}

View File

@@ -3,9 +3,9 @@ using Manager.App.Constants;
using Manager.App.Models.Library;
using Manager.App.Models.Settings;
using Manager.App.Models.System;
using Manager.App.Services.System;
using Manager.Data.Contexts;
using Manager.Data.Entities.LibraryContext;
using Manager.YouTube;
using Manager.YouTube.Models.Innertube;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
@@ -18,27 +18,28 @@ public class LibraryService : ILibraryService
private readonly LibrarySettings _librarySettings;
private readonly IDbContextFactory<LibraryDbContext> _dbContextFactory;
private readonly DirectoryInfo _libraryDirectory;
private readonly CacheService _cacheService;
public LibraryService(ILogger<LibraryService> logger, IOptions<LibrarySettings> librarySettings, IDbContextFactory<LibraryDbContext> contextFactory)
public LibraryService(ILogger<LibraryService> logger, IOptions<LibrarySettings> librarySettings, IDbContextFactory<LibraryDbContext> contextFactory, CacheService cacheService)
{
_logger = logger;
_librarySettings = librarySettings.Value;
_dbContextFactory = contextFactory;
_cacheService = cacheService;
_libraryDirectory = Directory.CreateDirectory(_librarySettings.Path);
logger.LogDebug("Working dir for library: {LibraryWorkingDir}", _libraryDirectory.FullName);
logger.LogDebug("Library directory: {LibraryWorkingDir}", _libraryDirectory.FullName);
Directory.CreateDirectory(Path.Combine(_librarySettings.Path, LibraryConstants.Directories.SubDirMedia));
Directory.CreateDirectory(Path.Combine(_librarySettings.Path, LibraryConstants.Directories.SubDirChannels));
}
public async Task<Result> FetchChannelImagesAsync(Channel channel)
public async Task<Result> FetchChannelImagesAsync(InnertubeChannel innertubeChannel)
{
try
{
await using var context = await _dbContextFactory.CreateDbContextAsync();
await AddWebImagesAsync(context, channel.AvatarImages, channel.Id, "avatars", LibraryConstants.FileTypes.ChannelAvatar, LibraryConstants.Directories.SubDirChannels);
await AddWebImagesAsync(context, channel.BannerImages, channel.Id, "banners", LibraryConstants.FileTypes.ChannelBanner, LibraryConstants.Directories.SubDirChannels);
await AddWebImagesAsync(context, innertubeChannel.AvatarImages, innertubeChannel.Id, "avatars", LibraryConstants.FileTypes.ChannelAvatar, LibraryConstants.Directories.SubDirChannels);
await AddWebImagesAsync(context, innertubeChannel.BannerImages, innertubeChannel.Id, "banners", LibraryConstants.FileTypes.ChannelBanner, LibraryConstants.Directories.SubDirChannels);
if (!context.ChangeTracker.HasChanges())
{
@@ -65,32 +66,33 @@ public class LibraryService : ILibraryService
continue;
}
var downloadResult = await NetworkService.DownloadBytesAsync(new HttpRequestMessage(HttpMethod.Get, image.Url));
if (!downloadResult.IsSuccess)
var cacheResult = await _cacheService.CacheFromUrl(image.Url);
if (!cacheResult.IsSuccess)
{
_logger.LogWarning("Failed to get image {ImageUrl}", image.Url);
continue;
}
var download = downloadResult.Value;
var cachedFile = cacheResult.Value;
var fileId = Guid.NewGuid();
var fileName = download.FileName ?? $"{fileId}.{download.ContentType?.Split('/').Last() ?? "unknown"}";
var fileName = cachedFile.OriginalFileName ?? $"{fileId}.{cachedFile.ContentType?.Split('/').Last() ?? "unknown"}";
var relativePath = Path.Combine(foreignKey, libSubDir, $"{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}_{fileName}");
var savePath = Path.Combine(_libraryDirectory.FullName, subDir, relativePath);
Directory.CreateDirectory(Path.GetDirectoryName(savePath) ?? savePath);
await using var fileStream = File.Create(savePath);
await fileStream.WriteAsync(download.Data.AsMemory(0, download.Data.Length));
await fileStream.WriteAsync(cachedFile.Data.AsMemory(0, cachedFile.Data.Length));
var file = new FileEntity
{
Id = fileId,
OriginalUrl = image.Url,
OriginalFileName = download.FileName,
OriginalFileName = cachedFile.OriginalFileName,
ForeignKey = foreignKey,
FileType = fileType,
RelativePath = relativePath.Replace('\\', '/'),
MimeType = download.ContentType,
SizeBytes = download.ContentLength,
MimeType = cachedFile.ContentType,
SizeBytes = cachedFile.Data.Length,
Height = image.Height,
Width = image.Width
};
@@ -99,6 +101,43 @@ public class LibraryService : ILibraryService
}
}
public async Task<Result> SaveClientAsync(ClientAccountEntity client, CancellationToken cancellationToken = default)
{
try
{
await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var updateEntity = false;
var dbClient = context.ClientAccounts.FirstOrDefault(c => c.Id == client.Id);
if (dbClient == null)
{
dbClient = client;
}
else
{
updateEntity = true;
dbClient.HttpCookies = client.HttpCookies;
dbClient.UserAgent = client.UserAgent;
}
if (updateEntity)
{
context.ClientAccounts.Update(dbClient);
}
else
{
context.ClientAccounts.Add(dbClient);
}
var savedResult= await context.SaveChangesAsync(cancellationToken);
return savedResult <= 0 ? ResultError.Fail("Could not save changes!") : Result.Success();
}
catch (Exception e)
{
return ResultError.Error(e);
}
}
public async Task<Result<ChannelEntity>> GetChannelByIdAsync(string id, CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(id))
@@ -109,7 +148,11 @@ public class LibraryService : ILibraryService
try
{
await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var channel = await context.Channels.Include(c => c.ClientAccount).FirstOrDefaultAsync(c => c.Id == id, cancellationToken);
var channel = await context.Channels
.Include(c => c.ClientAccount)
.ThenInclude(p => p!.HttpCookies)
.FirstOrDefaultAsync(c => c.Id == id, cancellationToken);
if (channel == null)
{
return ResultError.Fail("Channel not found!");
@@ -123,18 +166,53 @@ public class LibraryService : ILibraryService
}
}
public async Task<Result> SaveChannelAsync(ChannelEntity channel, CancellationToken cancellationToken = default)
public async Task<Result> SaveChannelAsync(InnertubeChannel innertubeChannel, CancellationToken cancellationToken = default)
{
try
{
await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
if (context.Channels.Any(c => c.Id == channel.Id))
var imagesResult = await FetchChannelImagesAsync(innertubeChannel);
if (!imagesResult.IsSuccess)
{
context.Channels.Update(channel);
return ResultError.Fail("Failed to fetch channel images!");
}
await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var channelResult = await GetChannelByIdAsync(innertubeChannel.Id, cancellationToken);
ChannelEntity? channelEntity;
try
{
if (channelResult.IsSuccess)
{
channelEntity = channelResult.Value;
channelEntity.Name = innertubeChannel.ChannelName;
channelEntity.Handle = innertubeChannel.Handle;
channelEntity.Description = innertubeChannel.Description;
}
else
{
channelEntity = new ChannelEntity
{
Id = innertubeChannel.Id,
Name = innertubeChannel.ChannelName,
Handle = innertubeChannel.Handle,
Description = innertubeChannel.Description
};
}
}
catch (Exception e)
{
return ResultError.Error(e);
}
if (context.Channels.Any(c => c.Id == innertubeChannel.Id))
{
context.Channels.Update(channelEntity);
}
else
{
context.Channels.Add(channel);
context.Channels.Add(channelEntity);
}
var changed = await context.SaveChangesAsync(cancellationToken);
@@ -142,7 +220,7 @@ public class LibraryService : ILibraryService
}
catch (Exception e)
{
return ResultError.Error(e);
return HandleException(e);
}
}
@@ -168,7 +246,7 @@ public class LibraryService : ILibraryService
}
}
public async Task<ListResult<ChannelEntity>> GetChannelAccountsAsync(int total = 20, int offset = 0, CancellationToken cancellationToken = default)
public async Task<ListResult<ChannelEntity>> GetChannelsAsync(int total = 20, int offset = 0, CancellationToken cancellationToken = default)
{
try
{

View File

@@ -0,0 +1,184 @@
using System.Security.Cryptography;
using System.Text;
using DotBased.Logging;
using DotBased.Monads;
using Manager.Data.Contexts;
using Manager.Data.Entities.Cache;
using Manager.YouTube;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
namespace Manager.App.Services.System;
public class CacheService(ILogger<CacheService> logger, IHostEnvironment environment) : ExtendedBackgroundService(nameof(CacheService), "Manages caching.", logger, TimeSpan.FromHours(5))
{
private DirectoryInfo? _cacheDirectory;
private PooledDbContextFactory<CacheDbContext>? _dbContextFactory;
private const string DataSubDir = "data";
protected override Task InitializeAsync(CancellationToken stoppingToken)
{
_cacheDirectory = new DirectoryInfo(Path.Combine(Directory.GetCurrentDirectory(), "cache"));
_cacheDirectory.Create();
Directory.CreateDirectory(Path.Combine(_cacheDirectory.FullName, DataSubDir));
LogEvent($"Cache directory: {_cacheDirectory.FullName}");
var dbContextOptionsBuilder = new DbContextOptionsBuilder<CacheDbContext>();
dbContextOptionsBuilder.UseSqlite($"Data Source={Path.Combine(_cacheDirectory.FullName, "cache_index.db")}");
_dbContextFactory = new PooledDbContextFactory<CacheDbContext>(dbContextOptionsBuilder.Options);
return Task.CompletedTask;
}
protected override async Task ExecuteServiceAsync(CancellationToken stoppingToken)
{
if (environment.IsDevelopment())
{
LogEvent("Development mode detected, skipping cache cleaning...");
return;
}
try
{
await ClearCacheAsync(stoppingToken);
}
catch (Exception e)
{
logger.LogError(e, "Error in execution of service.");
LogEvent($"Service execution failed. {e.Message}", LogSeverity.Error);
throw;
}
}
public string CreateCacheUrl(string originalUrl) => $"/api/v1/cache?url={originalUrl}";
public async Task<Result<CacheFile>> CacheFromUrl(string url, CancellationToken cancellationToken = default)
{
try
{
if (string.IsNullOrWhiteSpace(url))
{
return ResultError.Fail("Url is empty.");
}
if (_cacheDirectory == null)
{
return ResultError.Fail("Cache directory is not initialized.");
}
if (_dbContextFactory == null)
{
return ResultError.Fail("Context factory is not initialized.");
}
await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var urlKeyBytes = SHA1.HashData(Encoding.UTF8.GetBytes(url));
var urlKey = Convert.ToHexString(urlKeyBytes);
var cacheEntity = await context.Cache.FirstOrDefaultAsync(c => c.Id == urlKey, cancellationToken: cancellationToken);
if (cacheEntity == null)
{
var downloadResult = await NetworkService.DownloadBytesAsync(new HttpRequestMessage(HttpMethod.Get, url));
if (!downloadResult.IsSuccess)
{
LogEvent($"Failed to download from url: {url}");
return ResultError.Fail("Download failed.");
}
var download = downloadResult.Value;
await using var downloadFile = File.Create(Path.Combine(_cacheDirectory.FullName, DataSubDir, $"{urlKey}.cache"));
await downloadFile.WriteAsync(download.Data.AsMemory(0, download.Data.Length), cancellationToken);
cacheEntity = new CacheEntity
{
Id = urlKey,
CachedAtUtc = DateTime.UtcNow,
ContentLength = download.ContentLength,
ContentType = download.ContentType,
OriginalFileName = download.FileName,
};
context.Cache.Add(cacheEntity);
var saved = await context.SaveChangesAsync(cancellationToken);
if (saved <= 0)
{
LogEvent($"Cache entity {cacheEntity.Id} could not be saved.", LogSeverity.Error);
return ResultError.Fail("Failed to save to cache db.");
}
return new CacheFile(download.Data, download.ContentType, download.FileName);
}
var filePath = Path.Combine(_cacheDirectory.FullName, DataSubDir, $"{urlKey}.cache");
var buffer = await File.ReadAllBytesAsync(filePath, cancellationToken);
if (buffer.Length == 0)
{
LogEvent($"Failed to read data from disk. File: {filePath}", LogSeverity.Error);
return ResultError.Fail($"Error reading data from disk. File: {filePath}");
}
return new CacheFile(buffer.ToArray(), cacheEntity.ContentType, cacheEntity.OriginalFileName);
}
catch (Exception e)
{
return ResultError.Error(e, "Cache error.");
}
}
private async Task ClearCacheAsync(CancellationToken cancellationToken)
{
if (_dbContextFactory == null)
{
throw new InvalidOperationException("No DbContext factory configured.");
}
if (_cacheDirectory == null)
{
throw new InvalidOperationException("No cache directory configured.");
}
await using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var toRemove = dbContext.Cache.Where(c => c.CachedAtUtc < DateTime.UtcNow.AddDays(-1));
if (!toRemove.Any())
{
LogEvent("No items found to purge from cache.");
return;
}
var totalToRemove = toRemove.Count();
LogEvent($"Found {totalToRemove} cache items that are older than 1 day(s)");
var deleted = new List<CacheEntity>();
foreach (var entity in toRemove)
{
var pathToFile = Path.Combine(_cacheDirectory.FullName, DataSubDir, $"{entity.Id}.cache");
if (!File.Exists(pathToFile))
{
deleted.Add(entity);
continue;
}
try
{
File.Delete(pathToFile);
}
catch (Exception e)
{
logger.LogError(e, "Failed to delete cache entity with id: {EntityId}. Skipping cache entity...", entity.Id);
continue;
}
deleted.Add(entity);
}
dbContext.RemoveRange(deleted);
var dbDeleted = await dbContext.SaveChangesAsync(cancellationToken);
if (dbDeleted < deleted.Count)
{
LogEvent("Could not delete all files from cache.", LogSeverity.Warning);
}
LogEvent($"Removed {dbDeleted}/{totalToRemove} items");
}
}
public record CacheFile(byte[] Data, string? ContentType, string? OriginalFileName);

View File

@@ -3,20 +3,17 @@ using DotBased.Logging;
using DotBased.Monads;
using Manager.Data.Entities.LibraryContext;
using Manager.YouTube;
using Manager.YouTube.Models.Innertube;
namespace Manager.App.Services.System;
public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientService> logger)
: ExtendedBackgroundService(nameof(ClientService), "Managing YouTube clients", logger, TimeSpan.FromMinutes(10))
{
private readonly List<YouTubeClient> _clients = [];
private CancellationToken _cancellationToken;
private readonly YouTubeClientCollection _loadedClients = [];
private ILibraryService? _libraryService;
protected override Task InitializeAsync(CancellationToken stoppingToken)
{
_cancellationToken = stoppingToken;
stoppingToken.Register(CancellationRequested);
using var scope = scopeFactory.CreateScope();
_libraryService = scope.ServiceProvider.GetRequiredService<ILibraryService>();
@@ -24,17 +21,79 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
return Task.CompletedTask;
}
protected override Task ExecuteServiceAsync(CancellationToken stoppingToken)
protected override async Task ExecuteServiceAsync(CancellationToken stoppingToken)
{
return Task.CompletedTask;
LogEvent($"Saving {_loadedClients.Count} loaded client(s)");
foreach (var client in _loadedClients)
{
await SaveClientAsync(client, cancellationToken: stoppingToken);
}
}
private void CancellationRequested()
private async void CancellationRequested()
{
// Clear up
foreach (var client in _loadedClients)
{
await SaveClientAsync(client);
client.Dispose();
}
}
public async Task<Result> SaveClientAsync(YouTubeClient client, Channel? channelInfo = null, CancellationToken cancellationToken = default)
public async Task<Result> AddClientByIdAsync(string id, CancellationToken stoppingToken = default)
{
if (_libraryService == null)
{
return ResultError.Fail("Library service is not initialized!.");
}
var clientResult = await _libraryService.GetChannelByIdAsync(id, stoppingToken);
if (!clientResult.IsSuccess)
{
return clientResult;
}
var clientAcc = clientResult.Value.ClientAccount;
if (clientAcc == null)
{
return ResultError.Fail("Client account is not initialized!.");
}
var cookieCollection = new CookieCollection();
foreach (var httpCookie in clientAcc.HttpCookies)
{
var cookie = new Cookie
{
Name = httpCookie.Name,
Value = httpCookie.Value,
Domain = httpCookie.Domain,
Path = httpCookie.Path,
Secure = httpCookie.Secure,
HttpOnly = httpCookie.HttpOnly,
Expires = httpCookie.ExpiresUtc ?? DateTime.MinValue
};
cookieCollection.Add(cookie);
}
var ytClientResult = await YouTubeClient.CreateAsync(cookieCollection, clientAcc.UserAgent ?? "");
if (!ytClientResult.IsSuccess)
{
return ytClientResult;
}
AddClient(ytClientResult.Value);
return Result.Success();
}
public void AddClient(YouTubeClient client)
{
if (_loadedClients.Contains(client))
{
return;
}
_loadedClients.Add(client);
}
public async Task<Result> SaveClientAsync(YouTubeClient client, CancellationToken cancellationToken = default)
{
if (_libraryService == null)
{
@@ -47,121 +106,7 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
return ResultError.Fail("Client does not have an ID, cannot save to library database!");
}
if (channelInfo != null)
{
var imagesResult = await _libraryService.FetchChannelImagesAsync(channelInfo);
if (!imagesResult.IsSuccess)
{
logger.LogWarning("Failed to fetch channel images!");
}
}
var channelResult = await _libraryService.GetChannelByIdAsync(client.Id, cancellationToken);
ChannelEntity? channelEntity;
try
{
if (channelResult.IsSuccess)
{
channelEntity = channelResult.Value;
UpdateChannelEntity(client, channelEntity, channelInfo);
}
else
{
channelEntity = CreateNewChannelFromClient(client, channelInfo);
}
}
catch (Exception e)
{
LogEvent("Failed to save client: " + e.Message, LogSeverity.Warning);
return ResultError.Error(e);
}
var saveResult = await _libraryService.SaveChannelAsync(channelEntity, cancellationToken);
var saveResult = await _libraryService.SaveClientAsync(new ClientAccountEntity { Id = client.Id, UserAgent = client.UserAgent }, cancellationToken);
return saveResult;
}
private void UpdateChannelEntity(YouTubeClient client, ChannelEntity entity, Channel? channelInfo)
{
if (channelInfo != null)
{
entity.Name = channelInfo.ChannelName;
entity.Handle = channelInfo.Handle;
entity.Description = channelInfo.Description;
}
var clientAcc = entity.ClientAccount;
if (clientAcc != null)
{
clientAcc.UserAgent = clientAcc.UserAgent;
var currentCookies = client.CookieContainer.GetAllCookies();
foreach (var cookieEntity in clientAcc.HttpCookies.ToList())
{
var cookie = currentCookies[cookieEntity.Name];
if (cookie == null)
{
clientAcc.HttpCookies.Remove(cookieEntity);
continue;
}
if (!cookie.Domain.Equals(cookieEntity.Domain, StringComparison.InvariantCultureIgnoreCase))
{
continue;
}
cookieEntity.Value = cookie.Value;
cookieEntity.Path = cookie.Path;
cookieEntity.Secure = cookie.Secure;
cookieEntity.HttpOnly = cookie.HttpOnly;
cookieEntity.ExpiresUtc = cookie.Expires == DateTime.MinValue ? null : cookie.Expires;
}
}
}
private ChannelEntity CreateNewChannelFromClient(YouTubeClient client, Channel? channelInfo)
{
if (channelInfo == null)
{
throw new ArgumentNullException(nameof(channelInfo), "Channel information is required to store new client/account.");
}
var cookies = new List<HttpCookieEntity>();
foreach (var cookieObj in client.CookieContainer.GetAllCookies())
{
if (cookieObj is not Cookie cookie)
{
continue;
}
var cookieEntity = new HttpCookieEntity
{
ClientId = client.Id,
Name = cookie.Name,
Value = cookie.Value,
Domain = cookie.Domain,
Path = cookie.Path,
Secure = cookie.Secure,
HttpOnly = cookie.HttpOnly,
ExpiresUtc = cookie.Expires == DateTime.MinValue ? null : cookie.Expires
};
cookies.Add(cookieEntity);
}
var clientAcc = new ClientAccountEntity
{
Id = client.Id,
UserAgent = client.UserAgent,
HttpCookies = cookies
};
var channel = new ChannelEntity
{
Id = channelInfo.Id,
Name = channelInfo.ChannelName,
Handle = channelInfo.Handle,
Description = channelInfo.Description,
ClientAccount = clientAcc
};
return channel;
}
}

View File

@@ -0,0 +1,25 @@
using Manager.Data.Entities.Cache;
using Microsoft.EntityFrameworkCore;
namespace Manager.Data.Contexts;
public sealed class CacheDbContext : DbContext
{
public CacheDbContext(DbContextOptions<CacheDbContext> options) : base(options)
{
Database.EnsureCreated();
}
public DbSet<CacheEntity> Cache { get; set; }
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<CacheEntity>(ce =>
{
ce.ToTable("cache");
ce.HasKey(x => x.Id);
});
base.OnModelCreating(modelBuilder);
}
}

View File

@@ -18,7 +18,7 @@ public sealed class LibraryDbContext : DbContext
public DbSet<CaptionEntity> Captions { get; set; }
public DbSet<ChannelEntity> Channels { get; set; }
public DbSet<ClientAccountEntity> Accounts { get; set; }
public DbSet<ClientAccountEntity> ClientAccounts { get; set; }
public DbSet<HttpCookieEntity> HttpCookies { get; set; }
public DbSet<MediaEntity> Media { get; set; }
public DbSet<MediaFormatEntity> MediaFormats { get; set; }

View File

@@ -16,7 +16,7 @@ public class HttpCookieEntity : DateTimeBase
public string? Domain { get; set; }
[MaxLength(DataConstants.DbContext.DefaultDbStringSize)]
public string? Path { get; set; }
public DateTimeOffset? ExpiresUtc { get; set; }
public DateTime? ExpiresUtc { get; set; }
public bool Secure { get; set; }
public bool HttpOnly { get; set; }
}

View File

@@ -1,6 +1,6 @@
namespace Manager.YouTube.Models.Innertube;
public class Channel
public class InnertubeChannel
{
public required string Id { get; set; }
public bool NoIndex { get; set; }

View File

@@ -6,7 +6,7 @@ namespace Manager.YouTube;
public static class NetworkService
{
public const string Origin = "https://www.youtube.com";
private static readonly HttpClient HttpClient = new HttpClient();
private static readonly HttpClient HttpClient = new();
public static async Task<Result<string>> MakeRequestAsync(HttpRequestMessage request, YouTubeClient client, bool skipAuthenticationHeader = false)
{
@@ -43,7 +43,7 @@ public static class NetworkService
return ResultError.Fail($"Failed to get file to download, response code: {response.StatusCode}.");
}
var data = await response.Content.ReadAsByteArrayAsync();;
var data = await response.Content.ReadAsByteArrayAsync();
return new DownloadResult(data, response.Content.Headers.ContentType?.MediaType, response.Content.Headers.ContentDisposition?.FileName?.Trim('"'), response.Content.Headers.ContentLength ?? 0);
}

View File

@@ -6,7 +6,7 @@ namespace Manager.YouTube.Parsers.Json;
public static class ChannelJsonParser
{
public static Result<Channel> ParseJsonToChannelData(string json)
public static Result<InnertubeChannel> ParseJsonToChannelData(string json)
{
try
{
@@ -23,7 +23,7 @@ public static class ChannelJsonParser
throw new InvalidOperationException("No channel id found.");
}
var channel = new Channel
var channel = new InnertubeChannel
{
Id = channelId,
ChannelName = channelMetadata.GetProperty("title").ToString(),

View File

@@ -148,7 +148,7 @@ public sealed class YouTubeClient : IDisposable
return Result.Success();
}
public async Task<Result<Channel>> GetChannelByIdAsync(string channelId)
public async Task<Result<InnertubeChannel>> GetChannelByIdAsync(string channelId)
{
if (State == null)
{

View File

@@ -0,0 +1,11 @@
using System.Collections.ObjectModel;
namespace Manager.YouTube;
public class YouTubeClientCollection : KeyedCollection<string, YouTubeClient>
{
protected override string GetKeyForItem(YouTubeClient item)
{
return item.Id;
}
}