max/YouTube-Manager
1
0
Fork 0
Code Issues 4 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

[CHANGE] Fixed auditing, storing images from account import

Browse Source
This commit is contained in:
max
2025-09-15 00:23:57 +02:00
parent e82736a45f
commit 0056a14f79
16 changed files with 201 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Manager.App/Components/Pages/Channels.razor
View File

@@ -19,7 +19,7 @@
</MudStack>
</MudPaper>
<MudTable ServerData="ServerReload">
<MudTable @ref="@_table" ServerData="ServerReload">
<ToolBarContent>
<MudText Typo="Typo.h6">Channels</MudText>
</ToolBarContent>

8
Manager.App/Components/Pages/Channels.razor.cs
View File

@@ -9,10 +9,11 @@ namespace Manager.App.Components.Pages;
public partial class Channels : ComponentBase
{
private readonly DialogOptions _dialogOptions = new() { BackdropClick = false, CloseButton = true, FullWidth = true, MaxWidth = MaxWidth.ExtraLarge };
private MudTable<ChannelEntity>? _table;
private async Task<TableData<ChannelEntity>> ServerReload(TableState state, CancellationToken token)
{
var results = await LibraryService.GetChannelAccountsAsync(state.Page * state.PageSize, state.PageSize, token);
var results = await LibraryService.GetChannelAccountsAsync(state.PageSize, state.Page * state.PageSize, token);
return !results.IsSuccess ? new TableData<ChannelEntity>() : new TableData<ChannelEntity> { Items = results.Value, TotalItems = results.Total };
}
@@ -44,6 +45,9 @@ public partial class Channels : ComponentBase
Snackbar.Add($"Client {clientPrep.Channel?.Handle ?? clientPrep.YouTubeClient.Id} saved!", Severity.Success);
}
await InvokeAsync(StateHasChanged);
if (_table != null)
{
await _table.ReloadServerData();
}
}
}

18
Manager.App/Constants/LibraryConstants.cs Normal file
View File

@@ -0,0 +1,18 @@
namespace Manager.App.Constants;
public static class LibraryConstants
{
public static class Directories
{
public const string SubDirMedia = "Media";
public const string SubDirChannels = "Channels";
}
public static class FileTypes
{
public const string ChannelAvatar = "channel/avatar";
public const string ChannelBanner = "channel/banner";
public const string VideoThumbnail = "video/thumbnail";
public const string VideoCaption = "video/caption";
}
}

2
Manager.App/Services/ILibraryService.cs
View File

@@ -2,11 +2,13 @@ using DotBased.Monads;
using Manager.App.Models.Library;
using Manager.App.Models.System;
using Manager.Data.Entities.LibraryContext;
using Manager.YouTube.Models.Innertube;
namespace Manager.App.Services;
public interface ILibraryService
{
public Task<Result> FetchChannelImagesAsync(Channel channel);
public Task<Result<ChannelEntity>> GetChannelByIdAsync(string id, CancellationToken cancellationToken = default);
public Task<Result> SaveChannelAsync(ChannelEntity channel, CancellationToken cancellationToken = default);
public Task<Result<LibraryInformation>> GetLibraryInfoAsync(CancellationToken cancellationToken = default);

80
Manager.App/Services/LibraryService.cs
View File

@@ -1,9 +1,12 @@
using DotBased.Monads;
using Manager.App.Constants;
using Manager.App.Models.Library;
using Manager.App.Models.Settings;
using Manager.App.Models.System;
using Manager.Data.Contexts;
using Manager.Data.Entities.LibraryContext;
using Manager.YouTube;
using Manager.YouTube.Models.Innertube;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Options;
@@ -15,8 +18,7 @@ public class LibraryService : ILibraryService
private readonly LibrarySettings _librarySettings;
private readonly IDbContextFactory<LibraryDbContext> _dbContextFactory;
private readonly DirectoryInfo _libraryDirectory;
private const string SubDirMedia = "Media";
private const string SubDirChannels = "Channels";
public LibraryService(ILogger<LibraryService> logger, IOptions<LibrarySettings> librarySettings, IDbContextFactory<LibraryDbContext> contextFactory)
{
@@ -25,8 +27,76 @@ public class LibraryService : ILibraryService
_dbContextFactory = contextFactory;
_libraryDirectory = Directory.CreateDirectory(_librarySettings.Path);
logger.LogDebug("Working dir for library: {LibraryWorkingDir}", _libraryDirectory.FullName);
Directory.CreateDirectory(Path.Combine(_librarySettings.Path, SubDirMedia));
Directory.CreateDirectory(Path.Combine(_librarySettings.Path, SubDirChannels));
Directory.CreateDirectory(Path.Combine(_librarySettings.Path, LibraryConstants.Directories.SubDirMedia));
Directory.CreateDirectory(Path.Combine(_librarySettings.Path, LibraryConstants.Directories.SubDirChannels));
}
public async Task<Result> FetchChannelImagesAsync(Channel channel)
{
try
{
await using var context = await _dbContextFactory.CreateDbContextAsync();
await AddWebImagesAsync(context, channel.AvatarImages, channel.Id, "avatars", LibraryConstants.FileTypes.ChannelAvatar, LibraryConstants.Directories.SubDirChannels);
await AddWebImagesAsync(context, channel.BannerImages, channel.Id, "banners", LibraryConstants.FileTypes.ChannelBanner, LibraryConstants.Directories.SubDirChannels);
if (!context.ChangeTracker.HasChanges())
{
_logger.LogInformation("No changes detected. Skipping.");
return Result.Success();
}
await context.SaveChangesAsync();
}
catch (Exception e)
{
return ResultError.Error(e);
}
return Result.Success();
}
private async Task AddWebImagesAsync(LibraryDbContext context, List<WebImage> images, string foreignKey, string libSubDir, string fileType, string subDir)
{
foreach (var image in images)
{
if (context.Files.Any(f => image.Url.Equals(f.OriginalUrl, StringComparison.OrdinalIgnoreCase)))
{
continue;
}
var downloadResult = await NetworkService.DownloadBytesAsync(new HttpRequestMessage(HttpMethod.Get, image.Url));
if (!downloadResult.IsSuccess)
{
continue;
}
var download = downloadResult.Value;
var fileId = Guid.NewGuid();
var fileName = download.FileName ?? $"{fileId}.{download.ContentType?.Split('/').Last() ?? "unknown"}";
var relativePath = Path.Combine(foreignKey, libSubDir, $"{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}_{fileName}");
var savePath = Path.Combine(_libraryDirectory.FullName, subDir, relativePath);
Directory.CreateDirectory(Path.GetDirectoryName(savePath) ?? savePath);
await using var fileStream = File.Create(savePath);
await fileStream.WriteAsync(download.Data.AsMemory(0, download.Data.Length));
var file = new FileEntity
{
Id = fileId,
OriginalUrl = image.Url,
OriginalFileName = download.FileName,
ForeignKey = foreignKey,
FileType = fileType,
RelativePath = relativePath.Replace('\\', '/'),
MimeType = download.ContentType,
SizeBytes = download.ContentLength,
Height = image.Height,
Width = image.Width
};
await context.Files.AddAsync(file);
}
}
public async Task<Result<ChannelEntity>> GetChannelByIdAsync(string id, CancellationToken cancellationToken = default)
@@ -68,7 +138,7 @@ public class LibraryService : ILibraryService
}
var changed = await context.SaveChangesAsync(cancellationToken);
return changed <= 0 ? Result.Success() : ResultError.Fail("Failed to save channel!");
return changed <= 0 ? ResultError.Fail("Failed to save channel!") : Result.Success();
}
catch (Exception e)
{

33
Manager.App/Services/System/ClientService.cs
View File

@@ -47,19 +47,28 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
return ResultError.Fail("Client does not have an ID, cannot save to library database!");
}
if (channelInfo != null)
{
var imagesResult = await _libraryService.FetchChannelImagesAsync(channelInfo);
if (!imagesResult.IsSuccess)
{
logger.LogWarning("Failed to fetch channel images!");
}
}
var channelResult = await _libraryService.GetChannelByIdAsync(client.Id, cancellationToken);
ChannelEntity? channel;
ChannelEntity? channelEntity;
try
{
if (channelResult.IsSuccess)
{
channel = channelResult.Value;
UpdateChannelEntity(client, channel, channelInfo);
channelEntity = channelResult.Value;
UpdateChannelEntity(client, channelEntity, channelInfo);
}
else
{
channel = CreateNewChannelFromClient(client, channelInfo);
channelEntity = CreateNewChannelFromClient(client, channelInfo);
}
}
catch (Exception e)
@@ -68,7 +77,7 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
return ResultError.Error(e);
}
var saveResult = await _libraryService.SaveChannelAsync(channel, cancellationToken);
var saveResult = await _libraryService.SaveChannelAsync(channelEntity, cancellationToken);
return saveResult;
}
@@ -113,7 +122,7 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
{
if (channelInfo == null)
{
throw new ArgumentNullException(nameof(channelInfo), "Channel information required to store new client/account.");
throw new ArgumentNullException(nameof(channelInfo), "Channel information is required to store new client/account.");
}
var cookies = new List<HttpCookieEntity>();
@@ -147,7 +156,7 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
var channel = new ChannelEntity
{
Id = client.Id,
Id = channelInfo.Id,
Name = channelInfo.ChannelName,
Handle = channelInfo.Handle,
Description = channelInfo.Description,
@@ -155,14 +164,4 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
};
return channel;
}
/*public async Task<Result<YouTubeClient>> LoadClientByIdAsync(string id)
{
if (string.IsNullOrWhiteSpace(id))
{
return ResultError.Fail("Client ID is empty!");
}
return ResultError.Fail("Not implemented");
}*/
}

5
Manager.Data/Contexts/AuditInterceptor.cs
View File

@@ -36,11 +36,11 @@ public class AuditInterceptor : SaveChangesInterceptor
foreach (var entry in entries)
{
var primaryKey = entry.Properties.First(p => p.Metadata.IsPrimaryKey()).CurrentValue?.ToString() ?? "Unknown";
var primaryKey = entry.Properties.First(p => p.Metadata.IsPrimaryKey()).CurrentValue?.ToString();
var declaredProperties = entry.Entity.GetType()
.GetProperties(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.Instance)
.Where(p => !Attribute.IsDefined(p.DeclaringType!, typeof(NoAuditAttribute)))
.Where(p => !Attribute.IsDefined(p.DeclaringType!, typeof(NoAuditAttribute), false))
.Select(p => p.Name)
.ToHashSet();
@@ -78,6 +78,7 @@ public class AuditInterceptor : SaveChangesInterceptor
{
return new EntityAudit
{
Id = Guid.NewGuid(),
EntityName = entry.Entity.GetType().Name,
EntityId = primaryKey ?? "Unknown",
PropertyName = prop.Metadata.Name,

13
Manager.Data/Contexts/LibraryDbContext.cs
View File

@@ -23,7 +23,7 @@ public sealed class LibraryDbContext : DbContext
public DbSet<MediaEntity> Media { get; set; }
public DbSet<MediaFormatEntity> MediaFormats { get; set; }
public DbSet<PlaylistEntity> Playlists { get; set; }
// Other media (images)?
public DbSet<FileEntity> Files { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
@@ -32,9 +32,10 @@ public sealed class LibraryDbContext : DbContext
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<EntityAudit>(eh =>
modelBuilder.Entity<EntityAudit>(ea =>
{
eh.ToTable("audits");
ea.HasKey(a => a.Id);
ea.ToTable("audits");
});
modelBuilder.Entity<CaptionEntity>(ce =>
@@ -97,6 +98,12 @@ public sealed class LibraryDbContext : DbContext
ple.HasKey(x => x.Id);
});
modelBuilder.Entity<FileEntity>(file =>
{
file.ToTable("files");
file.HasKey(x => x.Id);
});
/* Join tables */
modelBuilder.Entity<PlaylistMedia>(pmj =>

5
Manager.Data/DataConstants.cs
View File

@@ -4,7 +4,8 @@ public static class DataConstants
{
public static class DbContext
{
public const int DefaultDbStringSize = 100;
public const int DefaultDbDescriptionStringSize = 500;
public const int DefaultDbStringSize = 500;
public const int DefaultDbDescriptionStringSize = 5500;
public const int DefaultDbUrlSize = 10000;
}
}

1
Manager.Data/Entities/Audit/EntityAudit.cs
View File

@@ -5,6 +5,7 @@ namespace Manager.Data.Entities.Audit;
public class EntityAudit
{
public required Guid Id { get; set; }
[MaxLength(200)]
public required string EntityName { get; set; }
[MaxLength(200)]

4
Manager.Data/Entities/DateTimeBase.cs
View File

@@ -5,6 +5,6 @@ namespace Manager.Data.Entities;
[NoAudit]
public abstract class DateTimeBase
{
public DateTime CreatedAtUtc { get; set; } = DateTime.UtcNow;
public DateTime LastModifiedUtc { get; set; } = DateTime.UtcNow;
public DateTime CreatedAtUtc { get; set; }
public DateTime LastModifiedUtc { get; set; }
}

25
Manager.Data/Entities/LibraryContext/FileEntity.cs Normal file
View File

@@ -0,0 +1,25 @@
using System.ComponentModel.DataAnnotations;
namespace Manager.Data.Entities.LibraryContext;
public class FileEntity : DateTimeBase
{
public required Guid Id { get; set; }
[MaxLength(DataConstants.DbContext.DefaultDbStringSize)]
public required string ForeignKey { get; set; }
[MaxLength(DataConstants.DbContext.DefaultDbStringSize)]
public required string FileType { get; set; }
[MaxLength(DataConstants.DbContext.DefaultDbStringSize)]
public required string RelativePath { get; set; }
[MaxLength(DataConstants.DbContext.DefaultDbStringSize)]
public string? MimeType { get; set; }
public long SizeBytes { get; set; }
[MaxLength(DataConstants.DbContext.DefaultDbUrlSize)]
public string? OriginalUrl { get; set; }
[MaxLength(DataConstants.DbContext.DefaultDbStringSize)]
public string? OriginalFileName { get; set; }
public int? Width { get; set; }
public int? Height { get; set; }
public long? LenghtMilliseconds { get; set; }
}

2
Manager.Data/Entities/LibraryContext/HttpCookieEntity.cs
View File

@@ -3,7 +3,7 @@ using Manager.Data.Entities.Audit;
namespace Manager.Data.Entities.LibraryContext;
[Auditable]
[NoAudit]
public class HttpCookieEntity : DateTimeBase
{
[MaxLength(DataConstants.DbContext.DefaultDbStringSize)]

1
Manager.YouTube/Models/Innertube/Channel.cs
View File

@@ -2,6 +2,7 @@ namespace Manager.YouTube.Models.Innertube;
public class Channel
{
public required string Id { get; set; }
public bool NoIndex { get; set; }
public bool Unlisted { get; set; }
public bool FamilySafe { get; set; }

22
Manager.YouTube/NetworkService.cs
View File

@@ -6,6 +6,7 @@ namespace Manager.YouTube;
public static class NetworkService
{
public const string Origin = "https://www.youtube.com";
private static readonly HttpClient HttpClient = new HttpClient();
public static async Task<Result<string>> MakeRequestAsync(HttpRequestMessage request, YouTubeClient client, bool skipAuthenticationHeader = false)
{
@@ -32,8 +33,25 @@ public static class NetworkService
}
}
public static async Task<Result<byte[]>> DownloadBytesAsync(HttpRequestMessage request, YouTubeClient client)
public static async Task<Result<DownloadResult>> DownloadBytesAsync(HttpRequestMessage request, YouTubeClient? client = null)
{
return ResultError.Fail("Not implemented");
try
{
var response = client != null ? await client.HttpClient.SendAsync(request) : await HttpClient.SendAsync(request);
if (!response.IsSuccessStatusCode)
{
return ResultError.Fail($"Failed to get file to download, response code: {response.StatusCode}.");
}
var data = await response.Content.ReadAsByteArrayAsync();;
return new DownloadResult(data, response.Content.Headers.ContentType?.MediaType, response.Content.Headers.ContentDisposition?.FileName?.Trim('"'), response.Content.Headers.ContentLength ?? 0);
}
catch (Exception e)
{
return ResultError.Error(e);
}
}
}
public record DownloadResult(byte[] Data, string? ContentType, string? FileName, long ContentLength);

25
Manager.YouTube/Parsers/Json/ChannelJsonParser.cs
View File

@@ -4,19 +4,31 @@ using Manager.YouTube.Models.Innertube;
namespace Manager.YouTube.Parsers.Json;
/// <summary>
/// Parsing functionality for the response from the innertube browse endpoint.
/// </summary>
public static class ChannelJsonParser
{
public static Result<Channel> ParseJsonToChannelData(string json)
{
try
{
var channel = new Channel();
var doc = JsonDocument.Parse(json);
var rootDoc = doc.RootElement;
var channelMetadata = rootDoc
.GetProperty("metadata")
.GetProperty("channelMetadataRenderer");
var channelId = channelMetadata.GetProperty("externalId").GetString();
if (channelId == null)
{
throw new InvalidOperationException("No channel id found.");
}
var channel = new Channel
{
Id = channelId,
ChannelName = channelMetadata.GetProperty("title").ToString(),
};
var microformat = rootDoc.GetProperty("microformat").GetProperty("microformatDataRenderer");
channel.AvailableCountries = microformat
@@ -29,11 +41,6 @@ public static class ChannelJsonParser
channel.Unlisted = microformat.GetProperty("unlisted").GetBoolean();
channel.FamilySafe = microformat.GetProperty("familySafe").GetBoolean();
var channelMetadata = rootDoc
.GetProperty("metadata")
.GetProperty("channelMetadataRenderer");
channel.ChannelName = channelMetadata.GetProperty("title").GetString();
var avatarThumbnails = channelMetadata.GetProperty("avatar")
.GetProperty("thumbnails")
.EnumerateArray();