From 0056a14f79f588fee63f1d42f641f76ff434ad10 Mon Sep 17 00:00:00 2001 From: max Date: Mon, 15 Sep 2025 00:23:57 +0200 Subject: [PATCH] [CHANGE] Fixed auditing, storing images from account import --- Manager.App/Components/Pages/Channels.razor | 2 +- .../Components/Pages/Channels.razor.cs | 8 +- Manager.App/Constants/LibraryConstants.cs | 18 +++++ Manager.App/Services/ILibraryService.cs | 2 + Manager.App/Services/LibraryService.cs | 80 +++++++++++++++++-- Manager.App/Services/System/ClientService.cs | 33 ++++---- Manager.Data/Contexts/AuditInterceptor.cs | 5 +- Manager.Data/Contexts/LibraryDbContext.cs | 13 ++- Manager.Data/DataConstants.cs | 5 +- Manager.Data/Entities/Audit/EntityAudit.cs | 1 + Manager.Data/Entities/DateTimeBase.cs | 4 +- .../Entities/LibraryContext/FileEntity.cs | 25 ++++++ .../LibraryContext/HttpCookieEntity.cs | 2 +- Manager.YouTube/Models/Innertube/Channel.cs | 1 + Manager.YouTube/NetworkService.cs | 24 +++++- .../Parsers/Json/ChannelJsonParser.cs | 25 +++--- 16 files changed, 201 insertions(+), 47 deletions(-) create mode 100644 Manager.App/Constants/LibraryConstants.cs create mode 100644 Manager.Data/Entities/LibraryContext/FileEntity.cs diff --git a/Manager.App/Components/Pages/Channels.razor b/Manager.App/Components/Pages/Channels.razor index 18010dd..ac9afce 100644 --- a/Manager.App/Components/Pages/Channels.razor +++ b/Manager.App/Components/Pages/Channels.razor @@ -19,7 +19,7 @@ - + Channels diff --git a/Manager.App/Components/Pages/Channels.razor.cs b/Manager.App/Components/Pages/Channels.razor.cs index a66bc62..eb9d7f8 100644 --- a/Manager.App/Components/Pages/Channels.razor.cs +++ b/Manager.App/Components/Pages/Channels.razor.cs @@ -9,10 +9,11 @@ namespace Manager.App.Components.Pages; public partial class Channels : ComponentBase { private readonly DialogOptions _dialogOptions = new() { BackdropClick = false, CloseButton = true, FullWidth = true, MaxWidth = MaxWidth.ExtraLarge }; + private MudTable? _table; private async Task> ServerReload(TableState state, CancellationToken token) { - var results = await LibraryService.GetChannelAccountsAsync(state.Page * state.PageSize, state.PageSize, token); + var results = await LibraryService.GetChannelAccountsAsync(state.PageSize, state.Page * state.PageSize, token); return !results.IsSuccess ? new TableData() : new TableData { Items = results.Value, TotalItems = results.Total }; } @@ -44,6 +45,9 @@ public partial class Channels : ComponentBase Snackbar.Add($"Client {clientPrep.Channel?.Handle ?? clientPrep.YouTubeClient.Id} saved!", Severity.Success); } - await InvokeAsync(StateHasChanged); + if (_table != null) + { + await _table.ReloadServerData(); + } } } \ No newline at end of file diff --git a/Manager.App/Constants/LibraryConstants.cs b/Manager.App/Constants/LibraryConstants.cs new file mode 100644 index 0000000..dfe7f3c --- /dev/null +++ b/Manager.App/Constants/LibraryConstants.cs @@ -0,0 +1,18 @@ +namespace Manager.App.Constants; + +public static class LibraryConstants +{ + public static class Directories + { + public const string SubDirMedia = "Media"; + public const string SubDirChannels = "Channels"; + } + + public static class FileTypes + { + public const string ChannelAvatar = "channel/avatar"; + public const string ChannelBanner = "channel/banner"; + public const string VideoThumbnail = "video/thumbnail"; + public const string VideoCaption = "video/caption"; + } +} \ No newline at end of file diff --git a/Manager.App/Services/ILibraryService.cs b/Manager.App/Services/ILibraryService.cs index b41a4fe..42c16a0 100644 --- a/Manager.App/Services/ILibraryService.cs +++ b/Manager.App/Services/ILibraryService.cs @@ -2,11 +2,13 @@ using DotBased.Monads; using Manager.App.Models.Library; using Manager.App.Models.System; using Manager.Data.Entities.LibraryContext; +using Manager.YouTube.Models.Innertube; namespace Manager.App.Services; public interface ILibraryService { + public Task FetchChannelImagesAsync(Channel channel); public Task> GetChannelByIdAsync(string id, CancellationToken cancellationToken = default); public Task SaveChannelAsync(ChannelEntity channel, CancellationToken cancellationToken = default); public Task> GetLibraryInfoAsync(CancellationToken cancellationToken = default); diff --git a/Manager.App/Services/LibraryService.cs b/Manager.App/Services/LibraryService.cs index 6b4e9d5..409c3a3 100644 --- a/Manager.App/Services/LibraryService.cs +++ b/Manager.App/Services/LibraryService.cs @@ -1,9 +1,12 @@ using DotBased.Monads; +using Manager.App.Constants; using Manager.App.Models.Library; using Manager.App.Models.Settings; using Manager.App.Models.System; using Manager.Data.Contexts; using Manager.Data.Entities.LibraryContext; +using Manager.YouTube; +using Manager.YouTube.Models.Innertube; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; @@ -15,8 +18,7 @@ public class LibraryService : ILibraryService private readonly LibrarySettings _librarySettings; private readonly IDbContextFactory _dbContextFactory; private readonly DirectoryInfo _libraryDirectory; - private const string SubDirMedia = "Media"; - private const string SubDirChannels = "Channels"; + public LibraryService(ILogger logger, IOptions librarySettings, IDbContextFactory contextFactory) { @@ -25,8 +27,76 @@ public class LibraryService : ILibraryService _dbContextFactory = contextFactory; _libraryDirectory = Directory.CreateDirectory(_librarySettings.Path); logger.LogDebug("Working dir for library: {LibraryWorkingDir}", _libraryDirectory.FullName); - Directory.CreateDirectory(Path.Combine(_librarySettings.Path, SubDirMedia)); - Directory.CreateDirectory(Path.Combine(_librarySettings.Path, SubDirChannels)); + Directory.CreateDirectory(Path.Combine(_librarySettings.Path, LibraryConstants.Directories.SubDirMedia)); + Directory.CreateDirectory(Path.Combine(_librarySettings.Path, LibraryConstants.Directories.SubDirChannels)); + } + + public async Task FetchChannelImagesAsync(Channel channel) + { + try + { + await using var context = await _dbContextFactory.CreateDbContextAsync(); + + await AddWebImagesAsync(context, channel.AvatarImages, channel.Id, "avatars", LibraryConstants.FileTypes.ChannelAvatar, LibraryConstants.Directories.SubDirChannels); + await AddWebImagesAsync(context, channel.BannerImages, channel.Id, "banners", LibraryConstants.FileTypes.ChannelBanner, LibraryConstants.Directories.SubDirChannels); + + if (!context.ChangeTracker.HasChanges()) + { + _logger.LogInformation("No changes detected. Skipping."); + return Result.Success(); + } + + await context.SaveChangesAsync(); + } + catch (Exception e) + { + return ResultError.Error(e); + } + + return Result.Success(); + } + + private async Task AddWebImagesAsync(LibraryDbContext context, List images, string foreignKey, string libSubDir, string fileType, string subDir) + { + foreach (var image in images) + { + if (context.Files.Any(f => image.Url.Equals(f.OriginalUrl, StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + var downloadResult = await NetworkService.DownloadBytesAsync(new HttpRequestMessage(HttpMethod.Get, image.Url)); + if (!downloadResult.IsSuccess) + { + continue; + } + + var download = downloadResult.Value; + + var fileId = Guid.NewGuid(); + var fileName = download.FileName ?? $"{fileId}.{download.ContentType?.Split('/').Last() ?? "unknown"}"; + var relativePath = Path.Combine(foreignKey, libSubDir, $"{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}_{fileName}"); + var savePath = Path.Combine(_libraryDirectory.FullName, subDir, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(savePath) ?? savePath); + await using var fileStream = File.Create(savePath); + await fileStream.WriteAsync(download.Data.AsMemory(0, download.Data.Length)); + + var file = new FileEntity + { + Id = fileId, + OriginalUrl = image.Url, + OriginalFileName = download.FileName, + ForeignKey = foreignKey, + FileType = fileType, + RelativePath = relativePath.Replace('\\', '/'), + MimeType = download.ContentType, + SizeBytes = download.ContentLength, + Height = image.Height, + Width = image.Width + }; + + await context.Files.AddAsync(file); + } } public async Task> GetChannelByIdAsync(string id, CancellationToken cancellationToken = default) @@ -68,7 +138,7 @@ public class LibraryService : ILibraryService } var changed = await context.SaveChangesAsync(cancellationToken); - return changed <= 0 ? Result.Success() : ResultError.Fail("Failed to save channel!"); + return changed <= 0 ? ResultError.Fail("Failed to save channel!") : Result.Success(); } catch (Exception e) { diff --git a/Manager.App/Services/System/ClientService.cs b/Manager.App/Services/System/ClientService.cs index ac2bbf8..0661b2b 100644 --- a/Manager.App/Services/System/ClientService.cs +++ b/Manager.App/Services/System/ClientService.cs @@ -46,20 +46,29 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger(); @@ -147,7 +156,7 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger> LoadClientByIdAsync(string id) - { - if (string.IsNullOrWhiteSpace(id)) - { - return ResultError.Fail("Client ID is empty!"); - } - - return ResultError.Fail("Not implemented"); - }*/ } \ No newline at end of file diff --git a/Manager.Data/Contexts/AuditInterceptor.cs b/Manager.Data/Contexts/AuditInterceptor.cs index 45f3274..063976c 100644 --- a/Manager.Data/Contexts/AuditInterceptor.cs +++ b/Manager.Data/Contexts/AuditInterceptor.cs @@ -36,11 +36,11 @@ public class AuditInterceptor : SaveChangesInterceptor foreach (var entry in entries) { - var primaryKey = entry.Properties.First(p => p.Metadata.IsPrimaryKey()).CurrentValue?.ToString() ?? "Unknown"; + var primaryKey = entry.Properties.First(p => p.Metadata.IsPrimaryKey()).CurrentValue?.ToString(); var declaredProperties = entry.Entity.GetType() .GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance) - .Where(p => !Attribute.IsDefined(p.DeclaringType!, typeof(NoAuditAttribute))) + .Where(p => !Attribute.IsDefined(p.DeclaringType!, typeof(NoAuditAttribute), false)) .Select(p => p.Name) .ToHashSet(); @@ -78,6 +78,7 @@ public class AuditInterceptor : SaveChangesInterceptor { return new EntityAudit { + Id = Guid.NewGuid(), EntityName = entry.Entity.GetType().Name, EntityId = primaryKey ?? "Unknown", PropertyName = prop.Metadata.Name, diff --git a/Manager.Data/Contexts/LibraryDbContext.cs b/Manager.Data/Contexts/LibraryDbContext.cs index 3aff6ab..4c18984 100644 --- a/Manager.Data/Contexts/LibraryDbContext.cs +++ b/Manager.Data/Contexts/LibraryDbContext.cs @@ -23,7 +23,7 @@ public sealed class LibraryDbContext : DbContext public DbSet Media { get; set; } public DbSet MediaFormats { get; set; } public DbSet Playlists { get; set; } - // Other media (images)? + public DbSet Files { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { @@ -32,9 +32,10 @@ public sealed class LibraryDbContext : DbContext protected override void OnModelCreating(ModelBuilder modelBuilder) { - modelBuilder.Entity(eh => + modelBuilder.Entity(ea => { - eh.ToTable("audits"); + ea.HasKey(a => a.Id); + ea.ToTable("audits"); }); modelBuilder.Entity(ce => @@ -96,6 +97,12 @@ public sealed class LibraryDbContext : DbContext ple.ToTable("playlists"); ple.HasKey(x => x.Id); }); + + modelBuilder.Entity(file => + { + file.ToTable("files"); + file.HasKey(x => x.Id); + }); /* Join tables */ diff --git a/Manager.Data/DataConstants.cs b/Manager.Data/DataConstants.cs index e4ba6a9..ea703f5 100644 --- a/Manager.Data/DataConstants.cs +++ b/Manager.Data/DataConstants.cs @@ -4,7 +4,8 @@ public static class DataConstants { public static class DbContext { - public const int DefaultDbStringSize = 100; - public const int DefaultDbDescriptionStringSize = 500; + public const int DefaultDbStringSize = 500; + public const int DefaultDbDescriptionStringSize = 5500; + public const int DefaultDbUrlSize = 10000; } } \ No newline at end of file diff --git a/Manager.Data/Entities/Audit/EntityAudit.cs b/Manager.Data/Entities/Audit/EntityAudit.cs index 0468aba..b9191cd 100644 --- a/Manager.Data/Entities/Audit/EntityAudit.cs +++ b/Manager.Data/Entities/Audit/EntityAudit.cs @@ -5,6 +5,7 @@ namespace Manager.Data.Entities.Audit; public class EntityAudit { + public required Guid Id { get; set; } [MaxLength(200)] public required string EntityName { get; set; } [MaxLength(200)] diff --git a/Manager.Data/Entities/DateTimeBase.cs b/Manager.Data/Entities/DateTimeBase.cs index cc09714..4b910c6 100644 --- a/Manager.Data/Entities/DateTimeBase.cs +++ b/Manager.Data/Entities/DateTimeBase.cs @@ -5,6 +5,6 @@ namespace Manager.Data.Entities; [NoAudit] public abstract class DateTimeBase { - public DateTime CreatedAtUtc { get; set; } = DateTime.UtcNow; - public DateTime LastModifiedUtc { get; set; } = DateTime.UtcNow; + public DateTime CreatedAtUtc { get; set; } + public DateTime LastModifiedUtc { get; set; } } \ No newline at end of file diff --git a/Manager.Data/Entities/LibraryContext/FileEntity.cs b/Manager.Data/Entities/LibraryContext/FileEntity.cs new file mode 100644 index 0000000..985b737 --- /dev/null +++ b/Manager.Data/Entities/LibraryContext/FileEntity.cs @@ -0,0 +1,25 @@ +using System.ComponentModel.DataAnnotations; + +namespace Manager.Data.Entities.LibraryContext; + +public class FileEntity : DateTimeBase +{ + public required Guid Id { get; set; } + [MaxLength(DataConstants.DbContext.DefaultDbStringSize)] + public required string ForeignKey { get; set; } + [MaxLength(DataConstants.DbContext.DefaultDbStringSize)] + public required string FileType { get; set; } + [MaxLength(DataConstants.DbContext.DefaultDbStringSize)] + public required string RelativePath { get; set; } + [MaxLength(DataConstants.DbContext.DefaultDbStringSize)] + public string? MimeType { get; set; } + public long SizeBytes { get; set; } + [MaxLength(DataConstants.DbContext.DefaultDbUrlSize)] + public string? OriginalUrl { get; set; } + [MaxLength(DataConstants.DbContext.DefaultDbStringSize)] + public string? OriginalFileName { get; set; } + + public int? Width { get; set; } + public int? Height { get; set; } + public long? LenghtMilliseconds { get; set; } +} \ No newline at end of file diff --git a/Manager.Data/Entities/LibraryContext/HttpCookieEntity.cs b/Manager.Data/Entities/LibraryContext/HttpCookieEntity.cs index 0b76d80..417dcf0 100644 --- a/Manager.Data/Entities/LibraryContext/HttpCookieEntity.cs +++ b/Manager.Data/Entities/LibraryContext/HttpCookieEntity.cs @@ -3,7 +3,7 @@ using Manager.Data.Entities.Audit; namespace Manager.Data.Entities.LibraryContext; -[Auditable] +[NoAudit] public class HttpCookieEntity : DateTimeBase { [MaxLength(DataConstants.DbContext.DefaultDbStringSize)] diff --git a/Manager.YouTube/Models/Innertube/Channel.cs b/Manager.YouTube/Models/Innertube/Channel.cs index ed65f70..ceac828 100644 --- a/Manager.YouTube/Models/Innertube/Channel.cs +++ b/Manager.YouTube/Models/Innertube/Channel.cs @@ -2,6 +2,7 @@ namespace Manager.YouTube.Models.Innertube; public class Channel { + public required string Id { get; set; } public bool NoIndex { get; set; } public bool Unlisted { get; set; } public bool FamilySafe { get; set; } diff --git a/Manager.YouTube/NetworkService.cs b/Manager.YouTube/NetworkService.cs index 4eff9eb..18944d2 100644 --- a/Manager.YouTube/NetworkService.cs +++ b/Manager.YouTube/NetworkService.cs @@ -6,6 +6,7 @@ namespace Manager.YouTube; public static class NetworkService { public const string Origin = "https://www.youtube.com"; + private static readonly HttpClient HttpClient = new HttpClient(); public static async Task> MakeRequestAsync(HttpRequestMessage request, YouTubeClient client, bool skipAuthenticationHeader = false) { @@ -32,8 +33,25 @@ public static class NetworkService } } - public static async Task> DownloadBytesAsync(HttpRequestMessage request, YouTubeClient client) + public static async Task> DownloadBytesAsync(HttpRequestMessage request, YouTubeClient? client = null) { - return ResultError.Fail("Not implemented"); + try + { + var response = client != null ? await client.HttpClient.SendAsync(request) : await HttpClient.SendAsync(request); + if (!response.IsSuccessStatusCode) + { + return ResultError.Fail($"Failed to get file to download, response code: {response.StatusCode}."); + } + + var data = await response.Content.ReadAsByteArrayAsync();; + + return new DownloadResult(data, response.Content.Headers.ContentType?.MediaType, response.Content.Headers.ContentDisposition?.FileName?.Trim('"'), response.Content.Headers.ContentLength ?? 0); + } + catch (Exception e) + { + return ResultError.Error(e); + } } -} \ No newline at end of file +} + +public record DownloadResult(byte[] Data, string? ContentType, string? FileName, long ContentLength); \ No newline at end of file diff --git a/Manager.YouTube/Parsers/Json/ChannelJsonParser.cs b/Manager.YouTube/Parsers/Json/ChannelJsonParser.cs index aceafa1..6e11ab5 100644 --- a/Manager.YouTube/Parsers/Json/ChannelJsonParser.cs +++ b/Manager.YouTube/Parsers/Json/ChannelJsonParser.cs @@ -4,18 +4,30 @@ using Manager.YouTube.Models.Innertube; namespace Manager.YouTube.Parsers.Json; -/// -/// Parsing functionality for the response from the innertube browse endpoint. -/// public static class ChannelJsonParser { public static Result ParseJsonToChannelData(string json) { try { - var channel = new Channel(); var doc = JsonDocument.Parse(json); var rootDoc = doc.RootElement; + + var channelMetadata = rootDoc + .GetProperty("metadata") + .GetProperty("channelMetadataRenderer"); + + var channelId = channelMetadata.GetProperty("externalId").GetString(); + if (channelId == null) + { + throw new InvalidOperationException("No channel id found."); + } + + var channel = new Channel + { + Id = channelId, + ChannelName = channelMetadata.GetProperty("title").ToString(), + }; var microformat = rootDoc.GetProperty("microformat").GetProperty("microformatDataRenderer"); @@ -29,11 +41,6 @@ public static class ChannelJsonParser channel.Unlisted = microformat.GetProperty("unlisted").GetBoolean(); channel.FamilySafe = microformat.GetProperty("familySafe").GetBoolean(); - var channelMetadata = rootDoc - .GetProperty("metadata") - .GetProperty("channelMetadataRenderer"); - channel.ChannelName = channelMetadata.GetProperty("title").GetString(); - var avatarThumbnails = channelMetadata.GetProperty("avatar") .GetProperty("thumbnails") .EnumerateArray();