Compare commits

..

26 Commits

Author SHA1 Message Date
max
3d20c116da [CHANGE] Adding playlist functionality 2025-11-02 21:43:06 +01:00
max
a849b7524d [CHANGE] Added video info page 2025-11-02 21:22:56 +01:00
max
bf957436f0 [CHANGE] Disabled signature challenge. Need to run js. 2025-11-02 19:45:41 +01:00
max
16343c9a56 [CHANGE] Testing js runtime script 2025-11-02 00:02:09 +01:00
max
4c04378080 [CHANGE] Added jint. Reworking decipher functionality 2025-10-28 19:08:59 +01:00
max
b5c701b971 [CHANGE] Reworked cipher stuff 2025-10-24 21:16:08 +02:00
max
a84195aefa [CHANGE] Reworked parsers/converters. Decipher operation from script do not work! 2025-10-24 00:23:09 +02:00
max
de28591d24 [REFACTOR] 2025-10-23 21:51:31 +02:00
max
264be8d529 [REFACTOR] Rename response 2025-10-23 21:36:16 +02:00
max
25589d18d8 [CHANGE] Update client. Reworked small things 2025-10-23 21:28:08 +02:00
max
972af513f0 [CHANGE] Linked decipher to video implementation 2025-10-23 20:08:02 +02:00
max
e87e1c57f9 [CHANGE] Updated cipher implementation and added decipher function 2025-10-23 19:45:36 +02:00
max
41f880cfef [CHANGE] Added old cipher implementation 2025-10-23 19:27:07 +02:00
max
9fdde5e756 [CHANGE] Added logger to client 2025-10-23 18:15:05 +02:00
max
ed9cb7eff1 [CHANGE] Check for video id else throw exception. 2025-10-23 18:06:26 +02:00
max
97f7f5dcf6 [CHANGE] Implementation video data 2025-10-21 20:46:31 +02:00
max
ed1b7406a6 [CHANGE] Json video parser && video model 2025-10-21 16:47:24 +02:00
max
2b5e93ff8a [CHANGE] Implementing video fetching and deciphering 2025-10-20 13:57:55 +02:00
max
1555ae9f3d [CHANGE] No need to reload client from db if already loaded && can be loaded without checking library service 2025-10-19 23:32:05 +02:00
max
34e029ec79 [CHANGE] Client handling 2025-10-19 23:21:07 +02:00
max
e4596df392 [CHANGE] ApplicationContext removed and related classes. Added Drive info to library info page. 2025-10-19 22:47:48 +02:00
max
2f19d60be0 [CHANGE] Reworked ExtendedBackgroundService.cs with State logic and actions 2025-10-08 18:22:16 +02:00
max
b8d2573d78 [CHANGE] Fixes library db 2025-09-28 18:59:39 +02:00
max
a2a420d596 [CHANGE] Small optimizations 2025-09-28 18:34:45 +02:00
max
c170b8db1f [CHANGE] Added cookie rotation with state. Updated cookie table 2025-09-28 18:05:52 +02:00
max
abc1505b6e [CHANGE] Cookie import by netscape cookie txt format 2025-09-28 17:47:34 +02:00
56 changed files with 1974 additions and 311 deletions

View File

@@ -0,0 +1,13 @@
@using Manager.App.Models.System
@using Manager.App.Services.System
@inject ISnackbar Snackbar
@inject ClientService ClientService
<MudText>Cipher manager</MudText>
<MudStack Row Spacing="2">
<MudAutocomplete T="YouTubeClientItem" Label="Client" @bind-Value="@_selectedClient" SearchFunc="SearchClientsAsync" ToStringFunc="@(i => i == null ? "null?" : $"{i.Name} ({i.Handle})")"
Variant="Variant.Outlined" ShowProgressIndicator ProgressIndicatorColor="Color.Primary">
</MudAutocomplete>
<MudButton OnClick="ExecCipher">Exec</MudButton>
</MudStack>

View File

@@ -0,0 +1,43 @@
using Manager.App.Models.System;
using Manager.YouTube.Util.Cipher;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
using MudBlazor;
namespace Manager.App.Components.Application.Dev;
public partial class CipherDev : ComponentBase
{
private YouTubeClientItem? _selectedClient;
private async Task ExecCipher(MouseEventArgs obj)
{
if (_selectedClient == null)
{
Snackbar.Add("No client selected", Severity.Warning);
return;
}
var ytClientResult = await ClientService.LoadClientByIdAsync(_selectedClient.Id);
if (!ytClientResult.IsSuccess)
{
Snackbar.Add(ytClientResult.Error?.Description ?? "Failed to get the client!", Severity.Error);
return;
}
var ytClient = ytClientResult.Value;
if (ytClient.State == null)
{
Snackbar.Add("Client state is null!", Severity.Warning);
return;
}
var decoder = await CipherManager.GetDecoderAsync(ytClient.State, ytClient);
}
private async Task<IEnumerable<YouTubeClientItem>> SearchClientsAsync(string? search, CancellationToken cancellationToken)
{
var searchResults = await ClientService.GetClientsAsync(search, cancellationToken: cancellationToken);
return !searchResults.IsSuccess ? [] : searchResults.Value;
}
}

View File

@@ -0,0 +1,17 @@
@using Manager.App.Models.System
@using Manager.App.Services.System
@inject ISnackbar Snackbar
@inject ClientService ClientService
@inject NavigationManager NavigationManager
<MudText>Video data</MudText>
<MudStack Row Spacing="2">
<MudAutocomplete T="YouTubeClientItem" Label="Client" @bind-Value="@_selectedClient" SearchFunc="SearchClientsAsync" ToStringFunc="@(i => i == null ? "null?" : $"{i.Name} ({i.Handle})")"
Variant="Variant.Outlined" ShowProgressIndicator ProgressIndicatorColor="Color.Primary">
</MudAutocomplete>
<MudTextField Label="Video id" @bind-Value="@_videoId"/>
</MudStack>
<MudStack>
<MudButton OnClick="NavigateToVideo">Get data</MudButton>
</MudStack>

View File

@@ -0,0 +1,40 @@
using Manager.App.Models.System;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
using MudBlazor;
namespace Manager.App.Components.Application.Dev;
public partial class DevelopmentVideo : ComponentBase
{
private YouTubeClientItem? _selectedClient;
private string _videoId = "";
private async Task<IEnumerable<YouTubeClientItem>> SearchClientsAsync(string? search, CancellationToken cancellationToken)
{
var searchResults = await ClientService.GetClientsAsync(search, cancellationToken: cancellationToken);
return !searchResults.IsSuccess ? [] : searchResults.Value;
}
private void NavigateToVideo(MouseEventArgs obj)
{
if (_selectedClient == null)
{
Snackbar.Add("No client selected!", Severity.Warning);
return;
}
if (string.IsNullOrWhiteSpace(_videoId))
{
Snackbar.Add("No video ID set!", Severity.Warning);
return;
}
if (_videoId.Length != 11)
{
Snackbar.Add("Video ID needs to have an length of 11 chars!", Severity.Warning);
}
NavigationManager.NavigateTo($"/video/{_videoId}?clientId={_selectedClient.Id}");
}
}

View File

@@ -27,44 +27,66 @@
<MudStack Row Spacing="2" Style="height: 100%"> <MudStack Row Spacing="2" Style="height: 100%">
<MudPaper Elevation="0" Outlined Class="pa-2" Style="width: 50%;"> <MudPaper Elevation="0" Outlined Class="pa-2" Style="width: 50%;">
<MudText>Import cookies</MudText> <MudText>Import cookies (Netscape Cookie format)</MudText>
<MudText Typo="Typo.caption">@($"{ImportCookies.Count} cookie(s) imported")</MudText> <MudStack Spacing="2">
<MudForm @bind-IsValid="@_cookieImportTextValid"> <MudStack Row Spacing="2">
<MudTextField @bind-Value="@_cookieDomain" Immediate Required Label="Domain" <MudFileUpload T="IBrowserFile" Accept=".txt" FilesChanged="UploadFiles">
RequiredError="Domain is required."/> <ActivatorContent>
<MudButton Variant="Variant.Filled"
Color="Color.Primary"
StartIcon="@Icons.Material.Filled.CloudUpload">
Upload cookie txt
</MudButton>
</ActivatorContent>
</MudFileUpload>
<MudButton Variant="Variant.Outlined"
OnClick="ParseCookies" Disabled="@(string.IsNullOrWhiteSpace(_cookieText))">Import
</MudButton>
</MudStack>
@if (MissingCookies.Any())
{
<MudPaper Class="pa-2" Elevation="0" Outlined>
<MudAlert Severity="Severity.Warning" Square Class="mb-2 mt-3">Some required cookies are not found, add the following cookie(s) to continue.</MudAlert>
<MudChipSet T="string" ReadOnly>
@foreach (var missingCookieName in MissingCookies)
{
<MudChip Variant="Variant.Text" Color="Color.Info">@missingCookieName</MudChip>
}
</MudChipSet>
</MudPaper>
}
<MudTextField Class="my-2" Lines="4" AutoGrow @bind-Value="@_cookieText" Immediate <MudTextField Class="my-2" Lines="4" AutoGrow @bind-Value="@_cookieText" Immediate
Required Label="Cookies" Variant="Variant.Outlined" Required Label="Cookies" Variant="Variant.Outlined"/>
Placeholder="EXAMPLE: Cookie1=Value1; Cookie2=Value2;" </MudStack>
Validation="@(new Func<string, string?>(ValidateCookieText))"/>
<MudButton Variant="Variant.Outlined" Disabled="@(!_cookieImportTextValid)"
OnClick="ParseCookies">Import
</MudButton>
</MudForm>
</MudPaper> </MudPaper>
<MudDataGrid Items="ImportCookies" Dense Elevation="0" Outlined Style="width: 50%;"> <MudDataGrid Items="ImportCookies" Dense Elevation="0" Outlined Style="width: 50%;">
<Header> <ToolBarContent>
<MudStack Class="ma-2"> <MudText>Cookies</MudText>
<MudText>Cookies</MudText> <MudSpacer />
</MudStack> <MudText Typo="Typo.caption">@($"{ImportCookies.Count} cookie(s)")</MudText>
</Header> </ToolBarContent>
<Columns> <Columns>
<TemplateColumn Title="Name"> <TemplateColumn Title="Name">
<CellTemplate> <CellTemplate>
<MudTextField Variant="Variant.Text" @bind-Value="@context.Item.Name" <MudText>@context.Item.Name</MudText>
Immediate/>
</CellTemplate> </CellTemplate>
</TemplateColumn> </TemplateColumn>
<TemplateColumn Title="Domain"> <TemplateColumn Title="Domain">
<CellTemplate> <CellTemplate>
<MudTextField Variant="Variant.Text" @bind-Value="@context.Item.Domain" <MudText>@context.Item.Domain</MudText>
Immediate/> </CellTemplate>
</TemplateColumn>
<TemplateColumn Title="Expires">
<CellTemplate>
<MudText>@context.Item.Expires</MudText>
</CellTemplate> </CellTemplate>
</TemplateColumn> </TemplateColumn>
<TemplateColumn Title="Value"> <TemplateColumn Title="Value">
<CellTemplate> <CellTemplate>
<MudTextField Variant="Variant.Text" @bind-Value="@context.Item.Value" <MudTooltip Text="@context.Item.Value">
Immediate/> <MudText Style="white-space: nowrap; overflow: hidden; text-overflow: ellipsis; max-width: 250px;">@context.Item.Value</MudText>
</MudTooltip>
</CellTemplate> </CellTemplate>
</TemplateColumn> </TemplateColumn>
</Columns> </Columns>

View File

@@ -1,7 +1,12 @@
using System.Net; using System.Net;
using System.Net.Mime;
using System.Text;
using Manager.App.Models.Library; using Manager.App.Models.Library;
using Manager.YouTube; using Manager.YouTube;
using Manager.YouTube.Constants;
using Manager.YouTube.Parsers;
using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Forms;
using MudBlazor; using MudBlazor;
namespace Manager.App.Components.Dialogs namespace Manager.App.Components.Dialogs
@@ -12,12 +17,11 @@ namespace Manager.App.Components.Dialogs
[Parameter] public string DefaultUserAgent { get; set; } = ""; [Parameter] public string DefaultUserAgent { get; set; } = "";
private ClientChannel? ClientChannel { get; set; } private ClientChannel? ClientChannel { get; set; }
private CookieCollection ImportCookies { get; set; } = []; private CookieCollection ImportCookies { get; set; } = [];
private IEnumerable<string> MissingCookies => CookieConstants.RequiredCookiesNames.Where(req => !ImportCookies.Select(c => c.Name).ToHashSet().Contains(req)).ToList();
private bool _isLoading; private bool _isLoading;
private AccountImportSteps _steps = AccountImportSteps.Authenticate; private AccountImportSteps _steps = AccountImportSteps.Authenticate;
private bool _cookieImportTextValid;
private string _cookieText = ""; private string _cookieText = "";
private string _cookieDomain = ".youtube.com";
private bool CanSave() private bool CanSave()
{ {
@@ -70,13 +74,40 @@ namespace Manager.App.Components.Dialogs
} }
await InvokeAsync(StateHasChanged); await InvokeAsync(StateHasChanged);
} }
private void ParseCookies() private async Task UploadFiles(IBrowserFile? file)
{ {
if (file == null)
{
SnackbarService.Add("File is null!", Severity.Error);
return;
}
if (file.ContentType != MediaTypeNames.Text.Plain)
{
SnackbarService.Add($"File uploaded with unsupported content type: {file.ContentType}", Severity.Warning);
return;
}
_isLoading = true;
var streamReader = new StreamReader(file.OpenReadStream(), Encoding.UTF8);
_cookieText = await streamReader.ReadToEndAsync();
_isLoading = false;
await InvokeAsync(StateHasChanged);
}
private async Task ParseCookies()
{
if (string.IsNullOrEmpty(_cookieText))
{
return;
}
try try
{ {
ImportCookies.Clear(); ImportCookies.Clear();
ImportCookies.Add(ParseCookieHeader(_cookieText, _cookieDomain)); var parsedCookies = await CookieTxtParser.ParseAsync(new MemoryStream(Encoding.UTF8.GetBytes(_cookieText)), CookieConstants.RequiredCookiesNames.ToHashSet());
ImportCookies.Add(parsedCookies);
_cookieText = string.Empty; _cookieText = string.Empty;
} }
catch (Exception e) catch (Exception e)
@@ -94,50 +125,6 @@ namespace Manager.App.Components.Dialogs
_steps = AccountImportSteps.Authenticate; _steps = AccountImportSteps.Authenticate;
StateHasChanged(); StateHasChanged();
} }
private static string? ValidateCookieText(string text)
{
if (string.IsNullOrWhiteSpace(text))
return "Cookies are required";
var pairs = text.Split(';', StringSplitOptions.RemoveEmptyEntries);
foreach (var pair in pairs)
{
if (!pair.Contains('=')) return "Invalid.";
var key = pair[..pair.IndexOf('=')].Trim();
if (string.IsNullOrEmpty(key)) return "Invalid.";
}
return null;
}
public static CookieCollection ParseCookieHeader(string cookieHeader, string domain = "")
{
var collection = new CookieCollection();
if (string.IsNullOrWhiteSpace(cookieHeader))
return collection;
var cookies = cookieHeader.Split(';', StringSplitOptions.RemoveEmptyEntries);
foreach (var cookieStr in cookies)
{
var parts = cookieStr.Split('=', 2);
if (parts.Length != 2) continue;
var name = parts[0].Trim();
var value = parts[1].Trim();
var cookie = new Cookie(name, value)
{
Path = "/",
Domain = domain
};
collection.Add(cookie);
}
return collection;
}
private async Task BuildClient() private async Task BuildClient()
{ {

View File

@@ -2,9 +2,10 @@
<MudNavMenu> <MudNavMenu>
<MudNavLink Href="/" Icon="@Icons.Material.Filled.Home" Match="NavLinkMatch.All">Home</MudNavLink> <MudNavLink Href="/" Icon="@Icons.Material.Filled.Home" Match="NavLinkMatch.All">Home</MudNavLink>
<MudNavGroup Title="Library" Expanded Icon="@Icons.Custom.Brands.YouTube" IconColor="Color.Error"> <MudNavGroup Title="Library" Expanded Icon="@Icons.Custom.Brands.YouTube" IconColor="Color.Error">
<MudNavLink Href="/Search" Icon="@Icons.Material.Filled.Search" Match="NavLinkMatch.All" Disabled>Search</MudNavLink>
<MudNavLink Href="/Accounts" Icon="@Icons.Material.Filled.AccountBox" Match="NavLinkMatch.All">Accounts</MudNavLink> <MudNavLink Href="/Accounts" Icon="@Icons.Material.Filled.AccountBox" Match="NavLinkMatch.All">Accounts</MudNavLink>
<MudNavLink Href="/Channels" Icon="@Icons.Material.Filled.AccountCircle" Match="NavLinkMatch.All">Channels</MudNavLink> <MudNavLink Href="/Channels" Icon="@Icons.Material.Filled.AccountCircle" Match="NavLinkMatch.All">Channels</MudNavLink>
<MudNavLink Href="/Playlists" Icon="@Icons.Material.Filled.ViewList" Match="NavLinkMatch.All">Playlists</MudNavLink> <MudNavLink Href="/Playlists" Icon="@Icons.Material.Filled.ViewList" Match="NavLinkMatch.All" Disabled>Playlists</MudNavLink>
<MudNavLink Href="/Library" Icon="@Icons.Material.Filled.Info" Match="NavLinkMatch.All" IconColor="Color.Info">Info</MudNavLink> <MudNavLink Href="/Library" Icon="@Icons.Material.Filled.Info" Match="NavLinkMatch.All" IconColor="Color.Info">Info</MudNavLink>
</MudNavGroup> </MudNavGroup>
<MudNavGroup Title="Application" Expanded Icon="@Icons.Material.Filled.SettingsSystemDaydream" IconColor="Color.Primary"> <MudNavGroup Title="Application" Expanded Icon="@Icons.Material.Filled.SettingsSystemDaydream" IconColor="Color.Primary">

View File

@@ -50,7 +50,6 @@ public partial class Accounts : ComponentBase
await _table.ReloadServerData(); await _table.ReloadServerData();
} }
Snackbar.Add($"Client {clientChannel.Channel?.Handle ?? clientChannel.YouTubeClient.Id} saved!", Severity.Success); Snackbar.Add($"Client {clientChannel.Channel?.Handle ?? clientChannel.YouTubeClient.Id} saved!", Severity.Success);
ClientService.AddClient(clientChannel.YouTubeClient);
} }
else else
{ {

View File

@@ -8,7 +8,7 @@
<MudStack Spacing="2"> <MudStack Spacing="2">
<MudTable @ref="@_table" ServerData="ServerReload"> <MudTable @ref="@_table" ServerData="ServerReload">
<ToolBarContent> <ToolBarContent>
<MudText Typo="Typo.h6">Channels stored in the library</MudText> <MudText Typo="Typo.h6">Channels</MudText>
<MudSpacer /> <MudSpacer />
<MudTextField T="string" ValueChanged="@(s=>OnSearch(s))" Placeholder="Search" Adornment="Adornment.Start" DebounceInterval="300" <MudTextField T="string" ValueChanged="@(s=>OnSearch(s))" Placeholder="Search" Adornment="Adornment.Start" DebounceInterval="300"
AdornmentIcon="@Icons.Material.Filled.Search" IconSize="Size.Medium" Class="mt-0"></MudTextField> AdornmentIcon="@Icons.Material.Filled.Search" IconSize="Size.Medium" Class="mt-0"></MudTextField>

View File

@@ -6,4 +6,10 @@
<MudTabPanel Text="Authentication"> <MudTabPanel Text="Authentication">
<AuthenticationHasher /> <AuthenticationHasher />
</MudTabPanel> </MudTabPanel>
<MudTabPanel Text="Video">
<DevelopmentVideo />
</MudTabPanel>
<MudTabPanel Text="Cipher">
<CipherDev />
</MudTabPanel>
</MudTabs> </MudTabs>

View File

@@ -26,6 +26,18 @@
<td>Library size:</td> <td>Library size:</td>
<td>@($"{Suffix.BytesToSizeSuffix(_libraryInformation.TotalSizeBytes)} ({_libraryInformation.TotalSizeBytes} bytes)")</td> <td>@($"{Suffix.BytesToSizeSuffix(_libraryInformation.TotalSizeBytes)} ({_libraryInformation.TotalSizeBytes} bytes)")</td>
</tr> </tr>
<tr>
<td>Drive total size:</td>
<td>@($"{Suffix.BytesToSizeSuffix(_libraryInformation.DriveTotalSpaceBytes)} ({_libraryInformation.DriveTotalSpaceBytes} bytes)")</td>
</tr>
<tr>
<td>Drive used space:</td>
<td>@($"{Suffix.BytesToSizeSuffix(_libraryInformation.DriveUsedSpaceBytes)} ({_libraryInformation.DriveUsedSpaceBytes} bytes)")</td>
</tr>
<tr>
<td>Drive free space available:</td>
<td>@($"{Suffix.BytesToSizeSuffix(_libraryInformation.DriveFreeSpaceBytes)} ({_libraryInformation.DriveFreeSpaceBytes} bytes)")</td>
</tr>
<tr> <tr>
<td>Total media:</td> <td>Total media:</td>

View File

@@ -7,7 +7,7 @@
<PageTitle>Services</PageTitle> <PageTitle>Services</PageTitle>
<MudDataGrid T="ExtendedBackgroundService" Items="@_backgroundServices" Filterable QuickFilter="@QuickFilter"> <MudDataGrid T="ExtendedBackgroundService" Items="@_backgroundServices" Filterable QuickFilter="@QuickFilter" Dense>
<ToolBarContent> <ToolBarContent>
<MudText Typo="Typo.h6">Services</MudText> <MudText Typo="Typo.h6">Services</MudText>
<MudSpacer/> <MudSpacer/>
@@ -20,16 +20,19 @@
<PropertyColumn Property="x => x.Description" Title="Description"/> <PropertyColumn Property="x => x.Description" Title="Description"/>
<PropertyColumn Property="x => x.State" Title="Status"/> <PropertyColumn Property="x => x.State" Title="Status"/>
<PropertyColumn Property="x => x.ExecuteInterval" Title="Execute interval"/> <PropertyColumn Property="x => x.ExecuteInterval" Title="Execute interval"/>
<TemplateColumn> <TemplateColumn Title="Actions">
<CellTemplate> <CellTemplate>
<MudStack Row Spacing="2"> <MudMenu Icon="@Icons.Material.Filled.MoreVert"
<MudButton Disabled="@(context.Item?.State == ServiceState.Paused)" AriaLabel="Actions">
OnClick="@(() => { context.Item?.Pause(); })" Variant="Variant.Outlined">Pause @foreach (var action in context.Item?.Actions ?? [])
</MudButton> {
<MudButton Disabled="@(context.Item?.State == ServiceState.Running)" <MudMenuItem OnClick="@action.Action" Disabled="@(!action.IsEnabled())">
OnClick="@(() => { context.Item?.Resume(); })" Variant="Variant.Outlined">Resume <MudTooltip Text="@action.Description">
</MudButton> <span>@action.Id</span>
</MudStack> </MudTooltip>
</MudMenuItem>
}
</MudMenu>
</CellTemplate> </CellTemplate>
</TemplateColumn> </TemplateColumn>
</Columns> </Columns>

View File

@@ -0,0 +1,244 @@
@page "/Video/{VideoId}"
@using Manager.App.Services.System
@inject ISnackbar Snackbar
@inject ClientService ClientService
@inject CacheService Cache
<ForcedLoadingOverlay Visible="_loading"/>
@if (!_loading && _video != null)
{
<MudStack Spacing="2">
<MudCard>
@{
var thumbnailUrl = _video.Thumbnails.OrderByDescending(t => t.Width).FirstOrDefault()?.Url;
}
@if (!string.IsNullOrWhiteSpace(thumbnailUrl))
{
<MudCardMedia Image="@Cache.CreateCacheUrl(thumbnailUrl)" Height="500"/>
}
<MudCardContent>
<MudText Typo="Typo.h5">@_video.Title</MudText>
<MudText Typo="Typo.body2">@_video.Description</MudText>
</MudCardContent>
</MudCard>
<MudExpansionPanels MultiExpansion>
<MudExpansionPanel Text="Info" Expanded>
<MudStack Spacing="2" Row Wrap="Wrap.Wrap">
@* Info *@
<MudSimpleTable Bordered Dense Elevation="0" Outlined Square Hover>
<tbody>
<tr>
<td>Video ID:</td>
<td>@_video.VideoId</td>
</tr>
<tr>
<td>Title:</td>
<td>@_video.Title</td>
</tr>
<tr>
<td>Description:</td>
<td>@_video.Description</td>
</tr>
<tr>
<td>HashTags:</td>
<td>@foreach (var hashtag in _video.HashTags)
{
<MudChip T="string" Variant="Variant.Text" Color="Color.Info">@hashtag</MudChip>
}
</td>
</tr>
<tr>
<td>View count:</td>
<td>@_video.ViewCount</td>
</tr>
<tr>
<td>Like count:</td>
<td>@_video.LikeCount</td>
</tr>
<tr>
<td>Channel ID:</td>
<td>@_video.ChannelId</td>
</tr>
<tr>
<td>Author:</td>
<td>@_video.Author</td>
</tr>
<tr>
<td>Playability status:</td>
<td>@_video.PlayabilityStatus</td>
</tr>
<tr>
<td>Length seconds:</td>
<td>@_video.LengthSeconds</td>
</tr>
<tr>
<td>Keywords:</td>
<td>@foreach (var keyword in _video.Keywords)
{
<MudChip T="string" Variant="Variant.Text">@keyword</MudChip>
}
</td>
</tr>
<tr>
<td>Publish date:</td>
<td>@_video.PublishDate</td>
</tr>
<tr>
<td>Upload date:</td>
<td>@_video.UploadDate</td>
</tr>
<tr>
<td>Category:</td>
<td>@_video.Category</td>
</tr>
</tbody>
</MudSimpleTable>
@* Boolean values *@
<MudSimpleTable Bordered Dense Elevation="0" Outlined Square Hover>
<tbody>
<tr>
<td>Is owner viewing:</td>
<td>@_video.IsOwnerViewing</td>
</tr>
<tr>
<td>Allow rating:</td>
<td>@_video.AllowRating</td>
</tr>
<tr>
<td>Is crawlable:</td>
<td>@_video.IsCrawlable</td>
</tr>
<tr>
<td>Is private:</td>
<td>@_video.IsPrivate</td>
</tr>
<tr>
<td>Is unplugged corpus:</td>
<td>@_video.IsUnpluggedCorpus</td>
</tr>
<tr>
<td>Is live:</td>
<td>@_video.IsLive</td>
</tr>
<tr>
<td>Is family save:</td>
<td>@_video.IsFamilySave</td>
</tr>
<tr>
<td>Is unlisted:</td>
<td>@_video.IsUnlisted</td>
</tr>
<tr>
<td>Has Ypc metadata:</td>
<td>@_video.HasYpcMetadata</td>
</tr>
<tr>
<td>Is shorts eligible:</td>
<td>@_video.IsShortsEligible</td>
</tr>
</tbody>
</MudSimpleTable>
</MudStack>
</MudExpansionPanel>
<MudExpansionPanel Text="Streaming data">
@if (_video.StreamingData == null)
{
<MudAlert Severity="Severity.Info">No streaming data available!</MudAlert>
}
else
{
<MudStack Spacing="2" Row Wrap="Wrap.Wrap">
<MudStack>
<MudText Typo="Typo.h5">Adaptive Formats</MudText>
<MudTable Items="@_video.StreamingData.AdaptiveFormats">
<HeaderContent>
<MudTh>Id</MudTh>
<MudTh>Mime type</MudTh>
<MudTh>Bitrate</MudTh>
<MudTh>Resolution</MudTh>
<MudTh>Last modified (UNIX epoch)</MudTh>
<MudTh>Quality</MudTh>
<MudTh>FPS</MudTh>
</HeaderContent>
<RowTemplate>
<MudTd>@context.Itag</MudTd>
<MudTd>@context.MimeType</MudTd>
<MudTd>@context.Bitrate</MudTd>
<MudTd>@($"{context.Width}x{context.Height}")</MudTd>
<MudTd>@context.LastModified</MudTd>
<MudTd>@context.Quality</MudTd>
<MudTd>@context.Fps</MudTd>
</RowTemplate>
</MudTable>
</MudStack>
<MudStack>
<MudText Typo="Typo.h5">Formats</MudText>
<MudTable Items="@_video.StreamingData.Formats">
<HeaderContent>
<MudTh>Id</MudTh>
<MudTh>Mime type</MudTh>
<MudTh>Bitrate</MudTh>
<MudTh>Resolution</MudTh>
<MudTh>Last modified (UNIX epoch)</MudTh>
<MudTh>Quality</MudTh>
<MudTh>FPS</MudTh>
</HeaderContent>
<RowTemplate>
<MudTd>@context.Itag</MudTd>
<MudTd>@context.MimeType</MudTd>
<MudTd>@context.Bitrate</MudTd>
<MudTd>@($"{context.Width}x{context.Height}")</MudTd>
<MudTd>@context.LastModified</MudTd>
<MudTd>@context.Quality</MudTd>
<MudTd>@context.Fps</MudTd>
</RowTemplate>
</MudTable>
</MudStack>
</MudStack>
}
</MudExpansionPanel>
<MudExpansionPanel Text="Player config">
@if (_video.PlayerConfig == null)
{
<MudAlert Severity="Severity.Info">No player config available!</MudAlert>
}
else
{
<MudSimpleTable Bordered Dense Elevation="0" Outlined Square Hover>
<tbody>
<tr>
<td>Audio loudness DB:</td>
<td>@_video.PlayerConfig.AudioLoudnessDb</td>
</tr>
<tr>
<td>Audio perceptual loudness DB:</td>
<td>@_video.PlayerConfig.AudioPerceptualLoudnessDb</td>
</tr>
<tr>
<td>Audio enable per format loudness:</td>
<td>@_video.PlayerConfig.AudioLoudnessDb</td>
</tr>
<tr>
<td>Max bitrate:</td>
<td>@_video.PlayerConfig.MaxBitrate</td>
</tr>
<tr>
<td>Max read ahead time MS:</td>
<td>@_video.PlayerConfig.MaxReadAheadMediaTimeMs</td>
</tr>
<tr>
<td>Min read ahead time MS:</td>
<td>@_video.PlayerConfig.MinReadAheadMediaTimeMs</td>
</tr>
<tr>
<td>Read ahead growth rate MS:</td>
<td>@_video.PlayerConfig.ReadAheadGrowthRateMs</td>
</tr>
</tbody>
</MudSimpleTable>
}
</MudExpansionPanel>
</MudExpansionPanels>
</MudStack>
}

View File

@@ -0,0 +1,48 @@
using Manager.YouTube.Models;
using Microsoft.AspNetCore.Components;
using MudBlazor;
namespace Manager.App.Components.Pages;
public partial class Video : ComponentBase
{
[Parameter]
public required string VideoId { get; set; }
[SupplyParameterFromQuery(Name = "clientId")]
public string ClientId { get; set; } = "";
private bool _loading = true;
private YouTubeVideo? _video;
protected override async Task OnInitializedAsync()
{
if (string.IsNullOrEmpty(VideoId))
{
Snackbar.Add("Video id is null or empty!", Severity.Error);
_loading = false;
return;
}
var clientResult = await ClientService.LoadClientByIdAsync(ClientId);
if (!clientResult.IsSuccess)
{
Snackbar.Add(clientResult.Error?.Description ?? "Failed to load client!", Severity.Error);
_loading = false;
return;
}
var client = clientResult.Value;
var videoResult = await client.GetVideoByIdAsync(VideoId);
if (!videoResult.IsSuccess)
{
Snackbar.Add(videoResult.Error?.Description ?? "Failed to get video.", Severity.Error);
_loading = false;
return;
}
_video = videoResult.Value;
_loading = false;
}
}

View File

@@ -8,4 +8,7 @@ public record LibraryInformation
public long TotalMedia { get; set; } public long TotalMedia { get; set; }
public long TotalChannels { get; set; } public long TotalChannels { get; set; }
public long TotalSizeBytes { get; set; } public long TotalSizeBytes { get; set; }
public long DriveTotalSpaceBytes { get; set; }
public long DriveFreeSpaceBytes { get; set; }
public long DriveUsedSpaceBytes { get; set; }
} }

View File

@@ -0,0 +1,16 @@
using Manager.App.Models.Library;
namespace Manager.App.Models.System;
public class YouTubeClientItem : AccountListView
{
public YouTubeClientItem(AccountListView accountListView)
{
Id = accountListView.Id;
Name = accountListView.Name;
Handle = accountListView.Handle;
HasCookies = accountListView.HasCookies;
AvatarFileId = accountListView.AvatarFileId;
}
public bool IsLoaded { get; set; }
}

View File

@@ -7,52 +7,91 @@ public abstract class ExtendedBackgroundService(string name, string description,
: BackgroundService : BackgroundService
{ {
private TaskCompletionSource _resumeSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); private TaskCompletionSource _resumeSignal = new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly List<ServiceAction> _actions = [];
private TaskCompletionSource? _manualContinue;
public ServiceState State { get; private set; } = ServiceState.Stopped; public ServiceState State { get; private set; } = ServiceState.Stopped;
public CircularBuffer<ServiceEvent> ProgressEvents { get; } = new(500); public CircularBuffer<ServiceEvent> ProgressEvents { get; } = new(500);
public string Name { get; } = name; public string Name { get; } = name;
public string Description { get; set; } = description; public string Description { get; } = description;
public TimeSpan ExecuteInterval { get; set; } = executeInterval ?? TimeSpan.FromMinutes(1); public TimeSpan ExecuteInterval { get; } = executeInterval ?? TimeSpan.FromSeconds(5);
public IReadOnlyList<ServiceAction> Actions => _actions;
protected void AddActions(IEnumerable<ServiceAction> actions)
{
_actions.AddRange(actions);
}
protected sealed override async Task ExecuteAsync(CancellationToken stoppingToken) protected sealed override async Task ExecuteAsync(CancellationToken stoppingToken)
{ {
State = ServiceState.Running; State = ServiceState.Running;
logger.LogInformation("Initializing background service: {ServiceName}", Name); logger.LogInformation("Initializing background service: {ServiceName}", Name);
_actions.AddRange(
[
new ServiceAction("Start", "Start the service (after the service is stopped of faulted.)", Start, () => State is ServiceState.Stopped or ServiceState.Faulted),
new ServiceAction("Pause", "Pause the service", Pause, () => State != ServiceState.Paused),
new ServiceAction("Resume", "Resume the service", Resume, () => State != ServiceState.Running)
]);
await InitializeAsync(stoppingToken); await InitializeAsync(stoppingToken);
try while (!stoppingToken.IsCancellationRequested)
{ {
logger.LogInformation("Running background service: {ServiceName}", Name); if (State == ServiceState.Running)
while (!stoppingToken.IsCancellationRequested)
{ {
if (State == ServiceState.Paused) try
{ {
_resumeSignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); logger.LogInformation("Started running background service: {ServiceName}", Name);
await _resumeSignal.Task.WaitAsync(stoppingToken); while (!stoppingToken.IsCancellationRequested)
} {
if (State == ServiceState.Paused)
{
_resumeSignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
await _resumeSignal.Task.WaitAsync(stoppingToken);
}
await ExecuteServiceAsync(stoppingToken); await ExecuteServiceAsync(stoppingToken);
await Task.Delay(ExecuteInterval, stoppingToken); await Task.Delay(ExecuteInterval, stoppingToken);
}
}
catch (OperationCanceledException e)
{
logger.LogInformation(e, "Service {ServiceName} received cancellation", Name);
}
catch (Exception e)
{
State = ServiceState.Faulted;
logger.LogError(e, "Background service {ServiceName} faulted!", Name);
LogEvent("Error executing background service.", LogSeverity.Error);
}
finally
{
State = ServiceState.Stopped;
}
} }
}
catch (Exception e) _manualContinue = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
{ var delayTask = Task.Delay(TimeSpan.FromMinutes(5), stoppingToken);
if (e is not OperationCanceledException) await Task.WhenAny(delayTask, _manualContinue.Task);
{ _manualContinue = null;
State = ServiceState.Faulted;
logger.LogError(e,"Background service {ServiceName} faulted!", Name);
throw;
}
logger.LogInformation(e,"Service {ServiceName} received cancellation", Name);
}
finally
{
State = ServiceState.Stopped;
} }
} }
protected void LogEvent(string message, LogSeverity severity = LogSeverity.Info) => ProgressEvents.Add(new ServiceEvent(string.Intern(Name), message, DateTime.UtcNow, severity)); protected void LogEvent(string message, LogSeverity severity = LogSeverity.Info) => ProgressEvents.Add(new ServiceEvent(string.Intern(Name), message, DateTime.UtcNow, severity));
public void Start()
{
if (State is ServiceState.Stopped or ServiceState.Faulted)
{
State = ServiceState.Running;
_manualContinue?.TrySetResult();
LogEvent("Started service.");
}
}
public void Pause() public void Pause()
{ {
if (State == ServiceState.Running) if (State == ServiceState.Running)
@@ -94,4 +133,6 @@ public enum ServiceState
Paused Paused
} }
public record struct ServiceEvent(string Source, string Message, DateTime DateUtc, LogSeverity Severity); public record struct ServiceEvent(string Source, string Message, DateTime DateUtc, LogSeverity Severity);
public record ServiceAction(string Id, string Description, Action Action, Func<bool> IsEnabled);

View File

@@ -8,7 +8,6 @@ namespace Manager.App.Services;
public interface ILibraryService public interface ILibraryService
{ {
public Task<Result> FetchChannelImagesAsync(InnertubeChannel innertubeChannel);
public Task<Result> SaveClientAsync(ClientAccountEntity client, CancellationToken cancellationToken = default); public Task<Result> SaveClientAsync(ClientAccountEntity client, CancellationToken cancellationToken = default);
public Task<Result<LibraryFile>> GetFileByIdAsync(Guid id, CancellationToken cancellationToken = default); public Task<Result<LibraryFile>> GetFileByIdAsync(Guid id, CancellationToken cancellationToken = default);
public Task<Result<ChannelEntity>> GetChannelByIdAsync(string id, CancellationToken cancellationToken = default); public Task<Result<ChannelEntity>> GetChannelByIdAsync(string id, CancellationToken cancellationToken = default);

View File

@@ -16,7 +16,6 @@ namespace Manager.App.Services;
public class LibraryService : ILibraryService public class LibraryService : ILibraryService
{ {
private readonly ILogger<LibraryService> _logger; private readonly ILogger<LibraryService> _logger;
private readonly LibrarySettings _librarySettings;
private readonly IDbContextFactory<LibraryDbContext> _dbContextFactory; private readonly IDbContextFactory<LibraryDbContext> _dbContextFactory;
private readonly DirectoryInfo _libraryDirectory; private readonly DirectoryInfo _libraryDirectory;
private readonly CacheService _cacheService; private readonly CacheService _cacheService;
@@ -24,38 +23,13 @@ public class LibraryService : ILibraryService
public LibraryService(ILogger<LibraryService> logger, IOptions<LibrarySettings> librarySettings, IDbContextFactory<LibraryDbContext> contextFactory, CacheService cacheService) public LibraryService(ILogger<LibraryService> logger, IOptions<LibrarySettings> librarySettings, IDbContextFactory<LibraryDbContext> contextFactory, CacheService cacheService)
{ {
_logger = logger; _logger = logger;
_librarySettings = librarySettings.Value; var librarySettings1 = librarySettings.Value;
_dbContextFactory = contextFactory; _dbContextFactory = contextFactory;
_cacheService = cacheService; _cacheService = cacheService;
_libraryDirectory = Directory.CreateDirectory(_librarySettings.Path); _libraryDirectory = Directory.CreateDirectory(librarySettings1.Path);
logger.LogDebug("Library directory: {LibraryWorkingDir}", _libraryDirectory.FullName); logger.LogDebug("Library directory: {LibraryWorkingDir}", _libraryDirectory.FullName);
Directory.CreateDirectory(Path.Combine(_librarySettings.Path, LibraryConstants.Directories.SubDirMedia)); Directory.CreateDirectory(Path.Combine(librarySettings1.Path, LibraryConstants.Directories.SubDirMedia));
Directory.CreateDirectory(Path.Combine(_librarySettings.Path, LibraryConstants.Directories.SubDirChannels)); Directory.CreateDirectory(Path.Combine(librarySettings1.Path, LibraryConstants.Directories.SubDirChannels));
}
public async Task<Result> FetchChannelImagesAsync(InnertubeChannel innertubeChannel)
{
try
{
await using var context = await _dbContextFactory.CreateDbContextAsync();
await AddWebImagesAsync(context, innertubeChannel.AvatarImages, innertubeChannel.Id, "avatars", LibraryConstants.FileTypes.ChannelAvatar, LibraryConstants.Directories.SubDirChannels);
await AddWebImagesAsync(context, innertubeChannel.BannerImages, innertubeChannel.Id, "banners", LibraryConstants.FileTypes.ChannelBanner, LibraryConstants.Directories.SubDirChannels);
if (!context.ChangeTracker.HasChanges())
{
_logger.LogInformation("No changes detected. Skipping.");
return Result.Success();
}
await context.SaveChangesAsync();
}
catch (Exception e)
{
return HandleException(e);
}
return Result.Success();
} }
private async Task AddWebImagesAsync(LibraryDbContext context, List<WebImage> images, string foreignKey, string libSubDir, string fileType, string subDir) private async Task AddWebImagesAsync(LibraryDbContext context, List<WebImage> images, string foreignKey, string libSubDir, string fileType, string subDir)
@@ -127,6 +101,7 @@ public class LibraryService : ILibraryService
} }
else else
{ {
context.HttpCookies.RemoveRange(context.HttpCookies.Where(x => x.ClientId == client.Id));
context.ClientAccounts.Add(dbClient); context.ClientAccounts.Add(dbClient);
} }
@@ -169,9 +144,9 @@ public class LibraryService : ILibraryService
try try
{ {
await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken); await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var channel = await context.Channels var channel = await context.Channels.AsSplitQuery()
.Include(c => c.ClientAccount) .Include(c => c.ClientAccount)
.ThenInclude(p => p!.HttpCookies) .ThenInclude(p => p!.HttpCookies)
.Include(f => f.Files) .Include(f => f.Files)
.FirstOrDefaultAsync(c => c.Id == id, cancellationToken); .FirstOrDefaultAsync(c => c.Id == id, cancellationToken);
@@ -191,13 +166,7 @@ public class LibraryService : ILibraryService
public async Task<Result> SaveChannelAsync(InnertubeChannel innertubeChannel, CancellationToken cancellationToken = default) public async Task<Result> SaveChannelAsync(InnertubeChannel innertubeChannel, CancellationToken cancellationToken = default)
{ {
try try
{ {
var imagesResult = await FetchChannelImagesAsync(innertubeChannel);
if (!imagesResult.IsSuccess)
{
return ResultError.Fail("Failed to fetch channel images!");
}
await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken); await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var channelResult = await GetChannelByIdAsync(innertubeChannel.Id, cancellationToken); var channelResult = await GetChannelByIdAsync(innertubeChannel.Id, cancellationToken);
@@ -228,7 +197,7 @@ public class LibraryService : ILibraryService
return ResultError.Error(e); return ResultError.Error(e);
} }
if (context.Channels.Any(c => c.Id == innertubeChannel.Id)) if (channelResult.IsSuccess)
{ {
context.Channels.Update(channelEntity); context.Channels.Update(channelEntity);
} }
@@ -237,6 +206,9 @@ public class LibraryService : ILibraryService
context.Channels.Add(channelEntity); context.Channels.Add(channelEntity);
} }
await AddWebImagesAsync(context, innertubeChannel.AvatarImages, innertubeChannel.Id, "avatars", LibraryConstants.FileTypes.ChannelAvatar, LibraryConstants.Directories.SubDirChannels);
await AddWebImagesAsync(context, innertubeChannel.BannerImages, innertubeChannel.Id, "banners", LibraryConstants.FileTypes.ChannelBanner, LibraryConstants.Directories.SubDirChannels);
var changed = await context.SaveChangesAsync(cancellationToken); var changed = await context.SaveChangesAsync(cancellationToken);
return changed <= 0 ? ResultError.Fail("Failed to save channel!") : Result.Success(); return changed <= 0 ? ResultError.Fail("Failed to save channel!") : Result.Success();
} }
@@ -251,6 +223,7 @@ public class LibraryService : ILibraryService
try try
{ {
await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken); await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var libraryDriveInfo = GetLibraryDriveInfo(_libraryDirectory);
var libInfo = new LibraryInformation var libInfo = new LibraryInformation
{ {
LibraryPath = _libraryDirectory.FullName, LibraryPath = _libraryDirectory.FullName,
@@ -258,7 +231,10 @@ public class LibraryService : ILibraryService
LastModifiedUtc = _libraryDirectory.LastWriteTimeUtc, LastModifiedUtc = _libraryDirectory.LastWriteTimeUtc,
TotalChannels = await context.Channels.CountAsync(cancellationToken: cancellationToken), TotalChannels = await context.Channels.CountAsync(cancellationToken: cancellationToken),
TotalMedia = await context.Media.CountAsync(cancellationToken: cancellationToken), TotalMedia = await context.Media.CountAsync(cancellationToken: cancellationToken),
TotalSizeBytes = GetDirectorySize(_libraryDirectory) TotalSizeBytes = GetDirectorySize(_libraryDirectory),
DriveTotalSpaceBytes = libraryDriveInfo.totalSpace,
DriveFreeSpaceBytes = libraryDriveInfo.freeSpace,
DriveUsedSpaceBytes = libraryDriveInfo.usedSpace
}; };
return libInfo; return libInfo;
} }
@@ -380,6 +356,20 @@ public class LibraryService : ILibraryService
} }
} }
private (long totalSpace, long freeSpace, long usedSpace) GetLibraryDriveInfo(DirectoryInfo dir)
{
try
{
var drive = new DriveInfo(dir.FullName);
return (drive.TotalSize, drive.AvailableFreeSpace, drive.TotalSize - drive.AvailableFreeSpace);
}
catch (Exception e)
{
_logger.LogError(e, "Error while getting directory free space.");
throw;
}
}
private ResultError HandleException(Exception exception) private ResultError HandleException(Exception exception)
{ {
if (exception is OperationCanceledException) if (exception is OperationCanceledException)

View File

@@ -2,6 +2,7 @@ using System.Security.Cryptography;
using System.Text; using System.Text;
using DotBased.Logging; using DotBased.Logging;
using DotBased.Monads; using DotBased.Monads;
using DotBased.Utilities;
using Manager.Data.Contexts; using Manager.Data.Contexts;
using Manager.Data.Entities.Cache; using Manager.Data.Entities.Cache;
using Manager.YouTube; using Manager.YouTube;
@@ -10,23 +11,45 @@ using Microsoft.EntityFrameworkCore.Infrastructure;
namespace Manager.App.Services.System; namespace Manager.App.Services.System;
public class CacheService(ILogger<CacheService> logger, IHostEnvironment environment) : ExtendedBackgroundService(nameof(CacheService), "Manages caching.", logger, TimeSpan.FromHours(5)) public class CacheService(ILogger<CacheService> logger, IHostEnvironment environment)
: ExtendedBackgroundService(nameof(CacheService), "Manages caching.", logger, TimeSpan.FromHours(5))
{ {
private DirectoryInfo? _cacheDirectory; private DirectoryInfo? _cacheDirectory;
private PooledDbContextFactory<CacheDbContext>? _dbContextFactory; private PooledDbContextFactory<CacheDbContext>? _dbContextFactory;
private const string DataSubDir = "data"; private const string DataSubDir = "data";
private const int CacheMaxAgeDays = 1;
private readonly SemaphoreSlim _cacheSemaphoreSlim = new(1, 1);
protected override Task InitializeAsync(CancellationToken stoppingToken) protected override Task InitializeAsync(CancellationToken stoppingToken)
{ {
_cacheDirectory = new DirectoryInfo(Path.Combine(Directory.GetCurrentDirectory(), "cache")); _cacheDirectory = new DirectoryInfo(Path.Combine(Directory.GetCurrentDirectory(), "cache"));
_cacheDirectory.Create(); _cacheDirectory.Create();
Directory.CreateDirectory(Path.Combine(_cacheDirectory.FullName, DataSubDir)); Directory.CreateDirectory(Path.Combine(_cacheDirectory.FullName, DataSubDir));
LogEvent($"Cache directory: {_cacheDirectory.FullName}"); LogEvent($"Cache directory: {_cacheDirectory.FullName}");
AddActions([
new ServiceAction("Clear cache", "Manually clear cache", () =>
{
LogEvent("Manual cache clear requested.");
_ = Task.Run(async () =>
{
try
{
await ClearCacheAsync(stoppingToken);
}
catch (Exception e)
{
logger.LogError(e, "Error clearing cache manually!");
LogEvent("Error manually clearing cache.", LogSeverity.Error);
}
}, stoppingToken);
}, () => true)
]);
var dbContextOptionsBuilder = new DbContextOptionsBuilder<CacheDbContext>(); var dbContextOptionsBuilder = new DbContextOptionsBuilder<CacheDbContext>();
dbContextOptionsBuilder.UseSqlite($"Data Source={Path.Combine(_cacheDirectory.FullName, "cache_index.db")}"); dbContextOptionsBuilder.UseSqlite($"Data Source={Path.Combine(_cacheDirectory.FullName, "cache_index.db")}");
_dbContextFactory = new PooledDbContextFactory<CacheDbContext>(dbContextOptionsBuilder.Options); _dbContextFactory = new PooledDbContextFactory<CacheDbContext>(dbContextOptionsBuilder.Options);
return Task.CompletedTask; return Task.CompletedTask;
} }
@@ -72,20 +95,24 @@ public class CacheService(ILogger<CacheService> logger, IHostEnvironment environ
} }
await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken); await using var context = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var urlKeyBytes = SHA1.HashData(Encoding.UTF8.GetBytes(url)); var urlKeyBytes = SHA1.HashData(Encoding.UTF8.GetBytes(url));
var urlKey = Convert.ToHexString(urlKeyBytes); var urlKey = Convert.ToHexString(urlKeyBytes);
var cacheEntity = await context.Cache.FirstOrDefaultAsync(c => c.Id == urlKey, cancellationToken: cancellationToken); var cacheEntity =
await context.Cache.FirstOrDefaultAsync(c => c.Id == urlKey, cancellationToken: cancellationToken);
if (cacheEntity == null) if (cacheEntity == null)
{ {
var downloadResult = await NetworkService.DownloadBytesAsync(new HttpRequestMessage(HttpMethod.Get, url)); var downloadResult =
await NetworkService.DownloadBytesAsync(new HttpRequestMessage(HttpMethod.Get, url));
if (!downloadResult.IsSuccess) if (!downloadResult.IsSuccess)
{ {
LogEvent($"Failed to download from url: {url}"); LogEvent($"Failed to download from url: {url}");
return ResultError.Fail("Download failed."); return ResultError.Fail("Download failed.");
} }
var download = downloadResult.Value; var download = downloadResult.Value;
await using var downloadFile = File.Create(Path.Combine(_cacheDirectory.FullName, DataSubDir, $"{urlKey}.cache")); await using var downloadFile =
File.Create(Path.Combine(_cacheDirectory.FullName, DataSubDir, $"{urlKey}.cache"));
await downloadFile.WriteAsync(download.Data.AsMemory(0, download.Data.Length), cancellationToken); await downloadFile.WriteAsync(download.Data.AsMemory(0, download.Data.Length), cancellationToken);
cacheEntity = new CacheEntity cacheEntity = new CacheEntity
@@ -96,7 +123,7 @@ public class CacheService(ILogger<CacheService> logger, IHostEnvironment environ
ContentType = download.ContentType, ContentType = download.ContentType,
OriginalFileName = download.FileName, OriginalFileName = download.FileName,
}; };
context.Cache.Add(cacheEntity); context.Cache.Add(cacheEntity);
var saved = await context.SaveChangesAsync(cancellationToken); var saved = await context.SaveChangesAsync(cancellationToken);
if (saved <= 0) if (saved <= 0)
@@ -126,58 +153,71 @@ public class CacheService(ILogger<CacheService> logger, IHostEnvironment environ
private async Task ClearCacheAsync(CancellationToken cancellationToken) private async Task ClearCacheAsync(CancellationToken cancellationToken)
{ {
if (_dbContextFactory == null) if (!await _cacheSemaphoreSlim.WaitAsync(0, cancellationToken))
{ {
throw new InvalidOperationException("No DbContext factory configured."); LogEvent("The cache cleaning task is already running. Skipping this call.", LogSeverity.Warning);
}
if (_cacheDirectory == null)
{
throw new InvalidOperationException("No cache directory configured.");
}
await using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var toRemove = dbContext.Cache.Where(c => c.CachedAtUtc < DateTime.UtcNow.AddDays(-1));
if (!toRemove.Any())
{
LogEvent("No items found to purge from cache.");
return; return;
} }
var totalToRemove = toRemove.Count(); try
LogEvent($"Found {totalToRemove} cache items that are older than 1 day(s)");
var deleted = new List<CacheEntity>();
foreach (var entity in toRemove)
{ {
var pathToFile = Path.Combine(_cacheDirectory.FullName, DataSubDir, $"{entity.Id}.cache"); if (_dbContextFactory == null)
if (!File.Exists(pathToFile)) throw new InvalidOperationException("No DbContext factory configured.");
if (_cacheDirectory == null)
throw new InvalidOperationException("No cache directory configured.");
await using var dbContext = await _dbContextFactory.CreateDbContextAsync(cancellationToken);
var toRemove = dbContext.Cache.Where(c => c.CachedAtUtc < DateTime.UtcNow.AddDays(-CacheMaxAgeDays));
if (!toRemove.Any())
{ {
LogEvent($"No items older than {CacheMaxAgeDays} day(s) found to clear from cache.");
return;
}
var totalToRemove = toRemove.Count();
LogEvent($"Found {totalToRemove} cache items that are older than 1 day(s)");
var deleted = new List<CacheEntity>();
long totalBytesRemoved = 0;
foreach (var entity in toRemove)
{
var pathToFile = Path.Combine(_cacheDirectory.FullName, DataSubDir, $"{entity.Id}.cache");
if (!File.Exists(pathToFile))
{
deleted.Add(entity);
totalBytesRemoved += entity.ContentLength;
continue;
}
try
{
File.Delete(pathToFile);
}
catch (Exception e)
{
logger.LogError(e, "Failed to delete cache entity with id: {EntityId}. Skipping cache entity...",
entity.Id);
continue;
}
totalBytesRemoved += entity.ContentLength;
deleted.Add(entity); deleted.Add(entity);
continue;
} }
try dbContext.RemoveRange(deleted);
{ var dbDeleted = await dbContext.SaveChangesAsync(cancellationToken);
File.Delete(pathToFile);
} if (dbDeleted < deleted.Count)
catch (Exception e) LogEvent("Could not delete all files from cache.", LogSeverity.Warning);
{
logger.LogError(e, "Failed to delete cache entity with id: {EntityId}. Skipping cache entity...", entity.Id); LogEvent($"Removed {dbDeleted}/{totalToRemove} items from cache. Total of {Suffix.BytesToSizeSuffix(totalBytesRemoved)} removed from disk.");
continue;
}
deleted.Add(entity);
} }
finally
dbContext.RemoveRange(deleted);
var dbDeleted = await dbContext.SaveChangesAsync(cancellationToken);
if (dbDeleted < deleted.Count)
{ {
LogEvent("Could not delete all files from cache.", LogSeverity.Warning); _cacheSemaphoreSlim.Release();
} }
LogEvent($"Removed {dbDeleted}/{totalToRemove} items");
} }
} }

View File

@@ -1,6 +1,7 @@
using System.Net; using System.Net;
using DotBased.Logging; using DotBased.Logging;
using DotBased.Monads; using DotBased.Monads;
using Manager.App.Models.System;
using Manager.Data.Entities.LibraryContext; using Manager.Data.Entities.LibraryContext;
using Manager.YouTube; using Manager.YouTube;
@@ -30,26 +31,48 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
} }
} }
private async void CancellationRequested() private void CancellationRequested()
{ {
foreach (var client in _loadedClients) foreach (var client in _loadedClients)
{ {
await SaveClientAsync(client);
client.Dispose(); client.Dispose();
} }
} }
public async Task<Result> AddClientByIdAsync(string id, CancellationToken stoppingToken = default) public async Task<ListResult<YouTubeClientItem>> GetClientsAsync(string? search, int offset = 0, int limit = 10, CancellationToken cancellationToken = default)
{ {
if (_libraryService == null) if (_libraryService == null)
{ {
return ResultError.Fail("Library service is not initialized!."); return ResultError.Fail("Library service is not initialized!.");
} }
var accountsResult = await _libraryService.GetAccountsAsync(search, offset, limit, cancellationToken);
if (!accountsResult.IsSuccess)
{
return accountsResult.Error ?? ResultError.Fail("Failed to get accounts!");
}
var comparedClients = accountsResult.Value.Select(x => new YouTubeClientItem(x)
{ Id = x.Id, IsLoaded = _loadedClients.Contains(x.Id) }).ToList();
return new ListResultReturn<YouTubeClientItem>(comparedClients, accountsResult.Total);
}
public async Task<Result<YouTubeClient>> LoadClientByIdAsync(string id, CancellationToken cancellationToken = default)
{
if (_loadedClients.TryGetValue(id, out var client))
{
return client;
}
var clientResult = await _libraryService.GetChannelByIdAsync(id, stoppingToken); if (_libraryService == null)
{
return ResultError.Fail("Library service is not initialized!.");
}
var clientResult = await _libraryService.GetChannelByIdAsync(id, cancellationToken);
if (!clientResult.IsSuccess) if (!clientResult.IsSuccess)
{ {
return clientResult; return clientResult.Error ?? ResultError.Fail("Failed to load channel from database!");
} }
var clientAcc = clientResult.Value.ClientAccount; var clientAcc = clientResult.Value.ClientAccount;
@@ -79,18 +102,8 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
return ytClientResult; return ytClientResult;
} }
AddClient(ytClientResult.Value); _loadedClients.Add(ytClientResult.Value);
return Result.Success(); return ytClientResult.Value;
}
public void AddClient(YouTubeClient client)
{
if (_loadedClients.Contains(client))
{
return;
}
_loadedClients.Add(client);
} }
public async Task<Result> SaveClientAsync(YouTubeClient client, CancellationToken cancellationToken = default) public async Task<Result> SaveClientAsync(YouTubeClient client, CancellationToken cancellationToken = default)
@@ -105,9 +118,11 @@ public class ClientService(IServiceScopeFactory scopeFactory, ILogger<ClientServ
LogEvent("Failed to store client no ID!", LogSeverity.Warning); LogEvent("Failed to store client no ID!", LogSeverity.Warning);
return ResultError.Fail("Client does not have an ID, cannot save to library database!"); return ResultError.Fail("Client does not have an ID, cannot save to library database!");
} }
_loadedClients.Add(client);
List<HttpCookieEntity> httpCookies = []; List<HttpCookieEntity> httpCookies = [];
httpCookies.AddRange(client.CookieContainer.GetAllCookies() httpCookies.AddRange(client.CookieContainer.GetAllCookies().Where(c => c.Expires != DateTime.MinValue)
.ToList() .ToList()
.Select(cookie => new HttpCookieEntity .Select(cookie => new HttpCookieEntity
{ {

View File

@@ -56,7 +56,7 @@ public class AuditInterceptor : SaveChangesInterceptor
break; break;
case EntityState.Modified: case EntityState.Modified:
audits.AddRange(allowedProperties audits.AddRange(allowedProperties
.Where(p => p.IsModified) .Where(p => p.IsModified && !Equals(p.OriginalValue, p.CurrentValue))
.Select(p => CreateAudit(entry, p, entry.State, primaryKey)) .Select(p => CreateAudit(entry, p, entry.State, primaryKey))
); );
break; break;
@@ -82,7 +82,7 @@ public class AuditInterceptor : SaveChangesInterceptor
EntityName = entry.Entity.GetType().Name, EntityName = entry.Entity.GetType().Name,
EntityId = primaryKey ?? "Unknown", EntityId = primaryKey ?? "Unknown",
PropertyName = prop.Metadata.Name, PropertyName = prop.Metadata.Name,
OldValue = SerializeValue(prop.OriginalValue), OldValue = changeType == EntityState.Added ? null : SerializeValue(prop.OriginalValue),
NewValue = SerializeValue(prop.CurrentValue), NewValue = SerializeValue(prop.CurrentValue),
ModifiedUtc = DateTime.UtcNow, ModifiedUtc = DateTime.UtcNow,
ChangedBy = "SYSTEM", ChangedBy = "SYSTEM",

View File

@@ -0,0 +1,22 @@
namespace Manager.YouTube.Constants;
public static class CookieConstants
{
public static readonly IReadOnlyCollection<string> RequiredCookiesNames = new HashSet<string>
{
"SID",
"SIDCC",
"HSID",
"SSID",
"APISID",
"SAPISID",
"__Secure-1PAPISID",
"__Secure-1PSID",
"__Secure-1PSIDCC",
"__Secure-1PSIDTS",
"__Secure-3PAPISID",
"__Secure-3PSID",
"__Secure-3PSIDCC",
"__Secure-3PSIDTS"
};
}

View File

@@ -0,0 +1,58 @@
using System.Text;
using DotBased.Monads;
namespace Manager.YouTube.Interpreter;
public static class JavaScriptEngineManager
{
private static readonly PlayerEngineCollection Engines = [];
public static async Task<Result<PlayerEngine>> GetPlayerEngine(string playerUrl)
{
if (string.IsNullOrEmpty(playerUrl))
{
return ResultError.Fail("player url is empty or null!");
}
var version = GetScriptVersion(playerUrl);
if (Engines.TryGetValue(version, out var engine))
{
return engine;
}
var playerJsSourceResult = await DownloadPlayerScriptAsync(playerUrl);
if (!playerJsSourceResult.IsSuccess)
{
return playerJsSourceResult.Error ?? ResultError.Fail("Download player script failed!");
}
return new PlayerEngine(version, playerJsSourceResult.Value);
}
private static string GetScriptVersion(string relativePlayerUrl)
{
var split = relativePlayerUrl.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var v = split[2];
var lang = split[4];
return $"{v}-{lang}";
}
private static async Task<Result<string>> DownloadPlayerScriptAsync(string relativeUrl, YouTubeClient? client = null)
{
var downloadRequest = new HttpRequestMessage(HttpMethod.Get, new Uri($"{NetworkService.Origin}/{relativeUrl}"));
var downloadResponse = await NetworkService.DownloadBytesAsync(downloadRequest, client);
if (!downloadResponse.IsSuccess)
{
return downloadResponse.Error ?? ResultError.Fail($"Failed to download script from url: {relativeUrl}");
}
var playerJs = Encoding.UTF8.GetString(downloadResponse.Value.Data);
if (string.IsNullOrWhiteSpace(playerJs))
{
return ResultError.Fail("Script value is empty!");
}
return playerJs;
}
}

View File

@@ -0,0 +1,42 @@
using DotBased.Logging;
using Jint;
namespace Manager.YouTube.Interpreter;
public class PlayerEngine
{
public string Version { get; set; }
public Engine JsEngine { get; set; }
private ILogger Logger { get; set; }
public PlayerEngine(string version, string script)
{
if (string.IsNullOrEmpty(version))
{
throw new ArgumentNullException(nameof(version));
}
if (string.IsNullOrEmpty(script))
{
throw new ArgumentNullException(nameof(script));
}
Logger = LogService.RegisterLogger(typeof(PlayerEngine), version);
Version = version;
JsEngine = new Engine().Execute(script).SetValue("log", new Action<object>(obj =>
{
var logStr = obj.ToString();
if (string.IsNullOrEmpty(logStr))
{
return;
}
Logger.Information(logStr);
}));
}
public void InitializePlayer()
{
JsEngine.Execute("createPlayer");
}
}

View File

@@ -0,0 +1,11 @@
using System.Collections.ObjectModel;
namespace Manager.YouTube.Interpreter;
public class PlayerEngineCollection : KeyedCollection<string, PlayerEngine>
{
protected override string GetKeyForItem(PlayerEngine item)
{
return item.Version;
}
}

View File

@@ -9,6 +9,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="DotBased" Version="1.0.0" /> <PackageReference Include="DotBased" Version="1.0.0" />
<PackageReference Include="HtmlAgilityPack" Version="1.12.2" /> <PackageReference Include="HtmlAgilityPack" Version="1.12.2" />
<PackageReference Include="Jint" Version="4.4.1" />
</ItemGroup> </ItemGroup>
</Project> </Project>

View File

@@ -35,6 +35,9 @@ public class ClientState : AdditionalJsonData
[JsonPropertyName("SERVER_VERSION")] [JsonPropertyName("SERVER_VERSION")]
public string? ServerVersion { get; set; } public string? ServerVersion { get; set; }
[JsonPropertyName("PLAYER_JS_URL")]
public string? PlayerJsUrl { get; set; }
[JsonPropertyName("INNERTUBE_CONTEXT")] [JsonPropertyName("INNERTUBE_CONTEXT")]
public InnerTubeContext? InnerTubeContext { get; set; } public InnerTubeContext? InnerTubeContext { get; set; }

View File

@@ -0,0 +1,13 @@
using System.Text.Json.Serialization;
namespace Manager.YouTube.Models.Innertube;
public class ColorInfo
{
[JsonPropertyName("primaries")]
public string Primaries { get; set; } = "";
[JsonPropertyName("transferCharacteristics")]
public string TransferCharacteristics { get; set; } = "";
[JsonPropertyName("matrixCoefficients")]
public string MatrixCoefficients { get; set; } = "";
}

View File

@@ -0,0 +1,11 @@
using System.Text.Json.Serialization;
namespace Manager.YouTube.Models.Innertube;
public class Range
{
[JsonPropertyName("start")]
public uint Start { get; set; }
[JsonPropertyName("end")]
public uint End { get; set; }
}

View File

@@ -0,0 +1,8 @@
namespace Manager.YouTube.Models.Innertube;
public class StoryBoard
{
public string Spec { get; set; } = "";
public int RecommendedLevel { get; set; }
public int HighResolutionRecommendedLevel { get; set; }
}

View File

@@ -0,0 +1,18 @@
using System.Text.Json.Serialization;
using Manager.YouTube.Util.Converters;
namespace Manager.YouTube.Models.Innertube;
public class StreamingData
{
public DateTime FetchedUtc { get; set; } = DateTime.UtcNow;
[JsonPropertyName("expiresInSeconds")]
public int ExpiresInSeconds { get; set; }
[JsonPropertyName("serverAbrStreamingUrl")]
[JsonConverter(typeof(JsonUrlEscapeConverter))]
public string ServerAbrStreamingUrl { get; set; } = "";
[JsonPropertyName("formats")]
public List<StreamingFormat> Formats { get; set; } = [];
[JsonPropertyName("adaptiveFormats")]
public List<StreamingFormat> AdaptiveFormats { get; set; } = [];
}

View File

@@ -0,0 +1,59 @@
using System.Text.Json.Serialization;
namespace Manager.YouTube.Models.Innertube;
public class StreamingFormat
{
[JsonPropertyName("itag")]
public int Itag { get; set; }
[JsonPropertyName("url")]
public string? Url { get; set; }
[JsonPropertyName("mimeType")]
public string MimeType { get; set; } = "";
[JsonPropertyName("bitrate")]
public uint Bitrate { get; set; }
[JsonPropertyName("width")]
public uint? Width { get; set; }
[JsonPropertyName("height")]
public uint? Height { get; set; }
[JsonPropertyName("initRange")]
public Range? InitRange { get; set; }
[JsonPropertyName("indexRange")]
public Range? IndexRange { get; set; }
[JsonPropertyName("lastModified")]
public long LastModified { get; set; }
[JsonPropertyName("contentLength")]
public long ContentLength { get; set; }
[JsonPropertyName("quality")]
public string Quality { get; set; } = "";
[JsonPropertyName("xtags")]
public string? Xtags { get; set; }
[JsonPropertyName("fps")]
public uint Fps { get; set; }
[JsonPropertyName("qualityLabel")]
public string QualityLabel { get; set; } = "";
[JsonPropertyName("projectionType")]
public string ProjectionType { get; set; } = "";
[JsonPropertyName("averagebitrate")]
public uint? AverageBitrate { get; set; }
[JsonPropertyName("highReplication")]
public bool? HighReplication { get; set; }
[JsonPropertyName("colorInfo")]
public ColorInfo? ColorInfo { get; set; }
[JsonPropertyName("audioQuality")]
public string? AudioQuality { get; set; } = "";
[JsonPropertyName("approxDurationMs")]
public long ApproxDurationMs { get; set; }
[JsonPropertyName("audioSampleRate")]
public int? AudioSampleRate { get; set; }
[JsonPropertyName("audioChannels")]
public int? AudioChannels { get; set; }
[JsonPropertyName("loudnessDb")]
public double? LoudnessDb { get; set; }
[JsonPropertyName("isDrc")]
public bool? IsDrc { get; set; }
[JsonPropertyName("signatureCipher")]
public string? SignatureCipher { get; set; }
[JsonPropertyName("qualityOrdinal")]
public string QualityOrdinal { get; set; } = "";
}

View File

@@ -1,8 +1,13 @@
using System.Text.Json.Serialization;
namespace Manager.YouTube.Models.Innertube; namespace Manager.YouTube.Models.Innertube;
public class WebImage public class WebImage
{ {
[JsonPropertyName("width")]
public int Width { get; set; } public int Width { get; set; }
[JsonPropertyName("height")]
public int Height { get; set; } public int Height { get; set; }
[JsonPropertyName("url")]
public string Url { get; set; } = ""; public string Url { get; set; } = "";
} }

View File

@@ -0,0 +1,9 @@
using System.Text.Json.Nodes;
namespace Manager.YouTube.Models.Parser;
public class YouTubeVideoData
{
public JsonNode? YouTubePlayerData { get; set; }
public JsonNode? YouTubeInitialData { get; set; }
}

View File

@@ -0,0 +1,12 @@
namespace Manager.YouTube.Models;
public class PlayerConfig
{
public double AudioLoudnessDb { get; set; }
public double AudioPerceptualLoudnessDb { get; set; }
public bool AudioEnablePerFormatLoudness { get; set; }
public uint MaxBitrate { get; set; }
public uint MaxReadAheadMediaTimeMs { get; set; }
public uint MinReadAheadMediaTimeMs { get; set; }
public uint ReadAheadGrowthRateMs { get; set; }
}

View File

@@ -0,0 +1,13 @@
using Manager.YouTube.Models.Innertube;
namespace Manager.YouTube.Models.Playlist;
public class PlaylistVideo
{
public required string VideoId { get; set; }
public List<WebImage> Thumbnails { get; set; } = [];
public required string Title { get; set; }
public required string Author { get; set; }
public long LengthSeconds { get; set; }
public bool IsPlayable { get; set; }
}

View File

@@ -0,0 +1,19 @@
using Manager.YouTube.Models.Playlist;
namespace Manager.YouTube.Models;
public class YouTubePlaylist
{
public required string Id { get; set; }
public required string Title { get; set; }
public required string Description { get; set; }
public required string Owner { get; set; }
public required string OwnerId { get; set; }
public bool NoIndex { get; set; }
public bool Unlisted { get; set; }
public bool CanReorder { get; set; }
public bool IsEditable { get; set; }
public List<PlaylistVideo> Videos { get; set; } = [];
public string? ContinuationToken { get; set; }
}

View File

@@ -0,0 +1,38 @@
using Manager.YouTube.Models.Innertube;
namespace Manager.YouTube.Models;
public class YouTubeVideo
{
public required string VideoId { get; set; }
public string Title { get; set; } = "";
public string Description { get; set; } = "";
public string[] HashTags { get; set; } = [];
public long ViewCount { get; set; }
public long LikeCount { get; set; }
public string ChannelId { get; set; } = "";
public string Author { get; set; } = "";
public string PlayabilityStatus { get; set; } = "";
public long LengthSeconds { get; set; }
public string[] Keywords { get; set; } = [];
public bool IsOwnerViewing { get; set; }
public bool AllowRating { get; set; }
public bool IsCrawlable { get; set; }
public bool IsPrivate { get; set; }
public bool IsUnpluggedCorpus { get; set; }
public bool IsLive { get; set; }
public bool IsFamilySave { get; set; }
public string[] AvailableCountries { get; set; } = [];
public bool IsUnlisted { get; set; }
public bool HasYpcMetadata { get; set; }
public DateTime PublishDate { get; set; }
public DateTime UploadDate { get; set; }
public bool IsShortsEligible { get; set; }
public string Category { get; set; } = "";
public StreamingData? StreamingData { get; set; }
public List<WebImage> Thumbnails { get; set; } = [];
public PlayerConfig? PlayerConfig { get; set; }
public StoryBoard? StoryBoard { get; set; }
}

View File

@@ -8,7 +8,7 @@ public static class NetworkService
public const string Origin = "https://www.youtube.com"; public const string Origin = "https://www.youtube.com";
private static readonly HttpClient HttpClient = new(); private static readonly HttpClient HttpClient = new();
public static async Task<Result<string>> MakeRequestAsync(HttpRequestMessage request, YouTubeClient client, bool skipAuthenticationHeader = false) public static async Task<Result<string>> MakeRequestAsync(HttpRequestMessage request, YouTubeClient client, bool skipAuthenticationHeader = false, CancellationToken cancellationToken = default)
{ {
request.Headers.Add("Origin", Origin); request.Headers.Add("Origin", Origin);
request.Headers.UserAgent.ParseAdd(client.UserAgent); request.Headers.UserAgent.ParseAdd(client.UserAgent);
@@ -19,8 +19,8 @@ public static class NetworkService
try try
{ {
var response = await client.HttpClient.SendAsync(request); var response = await client.HttpClient.SendAsync(request, cancellationToken);
var contentString = await response.Content.ReadAsStringAsync(); var contentString = await response.Content.ReadAsStringAsync(cancellationToken);
if (!response.IsSuccessStatusCode) if (!response.IsSuccessStatusCode)
{ {
return ResultError.Fail(contentString); return ResultError.Fail(contentString);

View File

@@ -0,0 +1,53 @@
using System.Net;
namespace Manager.YouTube.Parsers;
public static class CookieTxtParser
{
public static async Task<CookieCollection> ParseAsync(Stream stream, HashSet<string>? allowedCookies = null)
{
var cookieCollection = new CookieCollection();
using var reader = new StreamReader(stream);
while (await reader.ReadLineAsync() is { } line)
{
if (string.IsNullOrWhiteSpace(line) || line.StartsWith('#'))
{
continue;
}
var lineParts = line.Split('\t');
if (lineParts.Length < 7)
{
continue;
}
var domain = lineParts[0];
//var includeSubdomains = lineParts[1].Equals("TRUE", StringComparison.OrdinalIgnoreCase);
var path = lineParts[2];
var secure = lineParts[3].Equals("TRUE", StringComparison.OrdinalIgnoreCase);
var unixExpiry = long.TryParse(lineParts[4], out var exp) ? exp : 0;
var name = lineParts[5];
var value = lineParts[6];
if (!allowedCookies?.Contains(name) ?? false)
{
continue;
}
var cookie = new Cookie(name, value, path, domain)
{
Secure = secure
};
if (unixExpiry is > 0 and < int.MaxValue)
{
cookie.Expires = DateTimeOffset.FromUnixTimeSeconds(unixExpiry).DateTime;
}
cookieCollection.Add(cookie);
}
return cookieCollection;
}
}

View File

@@ -1,5 +1,7 @@
using System.Text.Json.Nodes;
using DotBased.Monads; using DotBased.Monads;
using HtmlAgilityPack; using HtmlAgilityPack;
using Manager.YouTube.Models.Parser;
namespace Manager.YouTube.Parsers; namespace Manager.YouTube.Parsers;
@@ -31,7 +33,57 @@ public static class HtmlParser
return (json, isPremiumUser); return (json, isPremiumUser);
} }
public static Result<YouTubeVideoData> GetVideoDataFromHtml(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return ResultError.Fail("html cannot be empty!");
}
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(html);
const string initialYoutubeData = "var ytInitialPlayerResponse = {";
var initialPlayerDataNode = htmlDocument.DocumentNode.SelectSingleNode($"//script[contains(., '{initialYoutubeData}')]");
if (string.IsNullOrWhiteSpace(initialPlayerDataNode.InnerText))
{
return ResultError.Fail("Could not find {initialPlayerData} in html script nodes!");
}
var initialPlayerDataString = ExtractJson(initialPlayerDataNode.InnerText, "var ytInitialPlayerResponse = ");
if (string.IsNullOrWhiteSpace(initialPlayerDataString))
{
return ResultError.Fail("Failed to extract initial player date from JSON.");
}
var parsedPlayerInitialData = JsonNode.Parse(initialPlayerDataString);
const string initialData = "var ytInitialData = {";
var initialDataNode = htmlDocument.DocumentNode.SelectSingleNode($"//script[contains(., '{initialData}')]");
if (string.IsNullOrWhiteSpace(initialDataNode.InnerText))
{
return ResultError.Fail("Could not find {initialData} in html script nodes!");
}
var initialDataJsonString = ExtractJson(initialDataNode.InnerText, "var ytInitialData = ");
if (string.IsNullOrWhiteSpace(initialDataJsonString))
{
return ResultError.Fail("Failed to extract initial player date from JSON.");
}
var parsedInitialData = JsonNode.Parse(initialDataJsonString);
try
{
return new YouTubeVideoData
{
YouTubePlayerData = parsedPlayerInitialData,
YouTubeInitialData = parsedInitialData
};
}
catch (Exception e)
{
return ResultError.Error(e, "Could not parse youtube player data.");
}
}
static string? ExtractJson(string input, string marker) static string? ExtractJson(string input, string marker)
{ {
var start = input.IndexOf(marker, StringComparison.Ordinal); var start = input.IndexOf(marker, StringComparison.Ordinal);

View File

@@ -1,4 +1,6 @@
using System.Text;
using System.Text.Json; using System.Text.Json;
using System.Text.Json.Nodes;
using Manager.YouTube.Models.Innertube; using Manager.YouTube.Models.Innertube;
namespace Manager.YouTube.Parsers.Json; namespace Manager.YouTube.Parsers.Json;
@@ -9,4 +11,81 @@ public static class JsonParser
array array
.Select(image => new WebImage { Width = image.GetProperty("width").GetInt32(), Height = image.GetProperty("height").GetInt32(), Url = image.GetProperty("url").GetString() ?? "" }) .Select(image => new WebImage { Width = image.GetProperty("width").GetInt32(), Height = image.GetProperty("height").GetInt32(), Url = image.GetProperty("url").GetString() ?? "" })
.ToList(); .ToList();
public static string ExtractTextOrHtml(JsonNode? node)
{
if (node is not JsonObject nodeObj)
{
return "";
}
// Case 1: Simple text (no formatting)
if (nodeObj.TryGetPropertyValue("simpleText", out var simpleText))
return simpleText?.GetValue<string>() ?? string.Empty;
// Case 2: Runs (formatted text segments)
if (nodeObj.TryGetPropertyValue("runs", out var runs) && runs != null && runs.GetValueKind() == JsonValueKind.Array)
{
var sb = new StringBuilder();
foreach (var runNode in runs.AsArray())
{
if (runNode is not JsonObject run)
{
continue;
}
var text = runNode["text"]?.GetValue<string>() ?? string.Empty;
var formatted = System.Net.WebUtility.HtmlEncode(text);
var bold = run.TryGetPropertyValue("bold", out var boldNode) && boldNode is JsonValue bv && bv.GetValue<bool>();
var italic = run.TryGetPropertyValue("italic", out var italicNode) && italicNode is JsonValue iv && iv.GetValue<bool>();
var underline = run.TryGetPropertyValue("underline", out var underlineNode) && underlineNode is JsonValue uv && uv.GetValue<bool>();
var strikethrough = run.TryGetPropertyValue("strikethrough", out var strikeNode) && strikeNode is JsonValue sv && sv.GetValue<bool>();
if (bold) formatted = $"<b>{formatted}</b>";
if (italic) formatted = $"<i>{formatted}</i>";
if (underline) formatted = $"<u>{formatted}</u>";
if (strikethrough) formatted = $"<s>{formatted}</s>";
if (run.TryGetPropertyValue("navigationEndpoint", out var nav) && nav is JsonObject navObj &&
navObj.TryGetPropertyValue("url", out var urlProp))
{
var url = urlProp?.GetValue<string>();
if (!string.IsNullOrEmpty(url))
formatted = $"<a href=\"{url}\">{formatted}</a>";
}
if (run.TryGetPropertyValue("emoji", out var emoji) && emoji is JsonObject emojiObj)
{
if (emojiObj.TryGetPropertyValue("url", out var emojiUrl))
{
var src = emojiUrl?.GetValue<string>();
if (!string.IsNullOrEmpty(src))
formatted = $"<img src=\"{src}\" alt=\"{text}\" class=\"emoji\" />";
}
}
sb.Append(formatted);
}
return sb.ToString();
}
return string.Empty;
}
public static List<WebImage> ExtractWebImages(JsonNode? node)
{
if (node == null)
{
return [];
}
var thumbnailsArray = node["thumbnails"];
return thumbnailsArray?.Deserialize<List<WebImage>>() ?? [];
}
} }

View File

@@ -0,0 +1,35 @@
using System.Text.Json;
using DotBased.Logging;
using DotBased.Monads;
using Manager.YouTube.Models;
using Manager.YouTube.Models.Parser;
using Manager.YouTube.Util.Converters;
namespace Manager.YouTube.Parsers.Json;
public static class VideoJsonParser
{
private static readonly JsonSerializerOptions VideoParserOptions = new() { Converters = { new YouTubeVideoJsonConverter() } };
private static readonly ILogger Logger = LogService.RegisterLogger(typeof(VideoJsonParser), "Video JSON parser");
public static Result<YouTubeVideo> ParseVideoData(YouTubeVideoData videoData)
{
if (videoData.YouTubePlayerData == null)
{
return ResultError.Fail("No initial video data found!");
}
YouTubeVideo? video;
try
{
video = videoData.YouTubePlayerData.Deserialize<YouTubeVideo>(VideoParserOptions);
}
catch (Exception e)
{
Logger.Error(e, "Failed to parse video data");
return ResultError.Fail("Failed to parse video data");
}
return video != null? video : ResultError.Fail("Failed to parse video data!");
}
}

View File

@@ -0,0 +1,125 @@
using System.Collections.Frozen;
using System.Text;
using System.Text.RegularExpressions;
using Manager.YouTube.Util.Cipher.Operations;
namespace Manager.YouTube.Util.Cipher;
public partial class CipherDecoder
{
public required string Version { get; init; }
public readonly IReadOnlySet<ICipherOperation> Operations;
private CipherDecoder(IEnumerable<ICipherOperation> operations)
{
Operations = operations.ToFrozenSet();
if (Operations.Count == 0)
{
throw new ArgumentNullException(nameof(operations), "No decipher operations given.");
}
}
public static async Task<CipherDecoder> CreateAsync(string relativeUrl, string version, YouTubeClient? client = null)
{
var operations = await GetCipherOperations(relativeUrl, client);
var decoder = new CipherDecoder(operations)
{
Version = version
};
return decoder;
}
public string Decipher(string? signatureCipher)
{
if (string.IsNullOrEmpty(signatureCipher))
{
return "";
}
var urlBuilder = new StringBuilder();
var indexStart = signatureCipher.IndexOf("s=", StringComparison.Ordinal);
var indexEnd = signatureCipher.IndexOf("&", StringComparison.Ordinal);
var signature = signatureCipher.Substring(indexStart, indexEnd);
indexStart = signatureCipher.IndexOf("&sp", StringComparison.Ordinal);
indexEnd = signatureCipher.IndexOf("&url", StringComparison.Ordinal);
var spParam = signatureCipher.Substring(indexStart, indexEnd - indexStart);
indexStart = signatureCipher.IndexOf("&url", StringComparison.Ordinal);
var videoUrl = signatureCipher[indexStart..];
signature = signature[(signature.IndexOf('=') + 1)..];
spParam = spParam[(spParam.IndexOf('=') + 1)..];
videoUrl = videoUrl[(videoUrl.IndexOf('=') + 1)..];
if (string.IsNullOrWhiteSpace(signature))
{
throw new InvalidOperationException("Invalid signature.");
}
var signatureDeciphered = Operations.Aggregate(signature, (acc, op) => op.Decipher(acc));
urlBuilder.Append(videoUrl);
urlBuilder.Append($"&{spParam}=");
urlBuilder.Append(signatureDeciphered);
return urlBuilder.ToString();
}
private static async Task<IEnumerable<ICipherOperation>> GetCipherOperations(string relativeUrl, YouTubeClient? client = null)
{
var downloadRequest = new HttpRequestMessage(HttpMethod.Get, new Uri($"{NetworkService.Origin}/{relativeUrl}"));
var downloadResponse = await NetworkService.DownloadBytesAsync(downloadRequest, client);
if (!downloadResponse.IsSuccess)
{
return [];
}
var playerJs = Encoding.UTF8.GetString(downloadResponse.Value.Data);
if (string.IsNullOrWhiteSpace(playerJs))
{
return [];
}
var functionBody = FunctionBodyRegex().Match(playerJs).Groups[0].ToString();
var definitionBody = DefinitionBodyRegex().Match(functionBody).Groups[1].Value;
var decipherDefinition = Regex.Match(playerJs, $@"var\s+{definitionBody}=\{{(\w+:function\(\w+(,\w+)?\)\{{(.*?)\}}),?\}};", RegexOptions.Singleline).Groups[0].ToString();
SortedSet<ICipherOperation> operations = [];
foreach (var statement in functionBody.Split(';'))
{
// Get the name of the function called in this statement
var calledFuncName = StatementFunctionNameRegex().Match(statement).Groups[1].Value;
if (string.IsNullOrWhiteSpace(calledFuncName))
continue;
if (Regex.IsMatch(decipherDefinition, $@"{Regex.Escape(calledFuncName)}:\bfunction\b\([a],b\).(\breturn\b)?.?\w+\."))
{
var index = int.Parse(OperationIndexRegex().Match(statement).Groups[1].Value);
operations.Add(new CipherSlice(index));
}
else if (Regex.IsMatch(decipherDefinition, $@"{Regex.Escape(calledFuncName)}:\bfunction\b\(\w+\,\w\).\bvar\b.\bc=a\b"))
{
var index = int.Parse(OperationIndexRegex().Match(statement).Groups[1].Value);
operations.Add(new CipherSwap(index));
}
else if (Regex.IsMatch(decipherDefinition, $@"{Regex.Escape(calledFuncName)}:\bfunction\b\(\w+\)"))
{
operations.Add(new CipherReverse());
}
}
return operations;
}
[GeneratedRegex(@"([A-Za-z_$][A-Za-z0-9_$]*)=function\([A-Za-z_$][A-Za-z0-9_$]*\)\{\s*([A-Za-z_$][A-Za-z0-9_$]*)=\2\.split\(\x22\x22\);[\s\S]*?return\s+\2\.join\(\x22\x22\)\s*\}")]
private static partial Regex FunctionBodyRegex();
[GeneratedRegex("([\\$_\\w]+).\\w+\\(\\w+,\\d+\\);")]
private static partial Regex DefinitionBodyRegex();
[GeneratedRegex(@"\(\w+,(\d+)\)")]
private static partial Regex OperationIndexRegex();
[GeneratedRegex(@"\w+(?:.|\[)(\""?\w+(?:\"")?)\]?\(")]
private static partial Regex StatementFunctionNameRegex();
}

View File

@@ -0,0 +1,11 @@
using System.Collections.ObjectModel;
namespace Manager.YouTube.Util.Cipher;
public class CipherDecoderCollection : KeyedCollection<string, CipherDecoder>
{
protected override string GetKeyForItem(CipherDecoder item)
{
return item.Version;
}
}

View File

@@ -0,0 +1,47 @@
using DotBased.Logging;
using DotBased.Monads;
using Manager.YouTube.Models.Innertube;
namespace Manager.YouTube.Util.Cipher;
public static class CipherManager
{
private static readonly CipherDecoderCollection LoadedCiphers = [];
private static readonly ILogger Logger = LogService.RegisterLogger(typeof(CipherManager));
public static async Task<Result<CipherDecoder>> GetDecoderAsync(ClientState clientState, YouTubeClient? client = null)
{
var relativePlayerJsUrl = clientState.PlayerJsUrl;
if (string.IsNullOrEmpty(relativePlayerJsUrl))
{
return ResultError.Fail("Could not get player js url.");
}
var version = GetCipherVersion(relativePlayerJsUrl);
Logger.Debug($"Getting cipher decoder for version: {version}");
if (LoadedCiphers.TryGetValue(version, out var cipher))
{
return cipher;
}
try
{
var decoder = await CipherDecoder.CreateAsync(relativePlayerJsUrl, version, client);
LoadedCiphers.Add(decoder);
}
catch (Exception e)
{
Logger.Error(e, "Could not create cipher decoder. Version: {DecoderVersion}", version);
}
return ResultError.Fail($"Could not create cipher decoder for {relativePlayerJsUrl} (v: {version})");
}
private static string GetCipherVersion(string relativePlayerUrl)
{
var split = relativePlayerUrl.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
var v = split[2];
var lang = split[4];
return $"{v}_{lang}";
}
}

View File

@@ -0,0 +1,18 @@
using System.Text;
namespace Manager.YouTube.Util.Cipher.Operations;
public class CipherReverse : ICipherOperation
{
public string Decipher(string cipherSignature)
{
var buffer = new StringBuilder(cipherSignature.Length);
for (var i = cipherSignature.Length - 1; i >= 0; i--)
{
buffer.Append(cipherSignature[i]);
}
return buffer.ToString();
}
}

View File

@@ -0,0 +1,6 @@
namespace Manager.YouTube.Util.Cipher.Operations;
public class CipherSlice(int indexToSlice) : ICipherOperation
{
public string Decipher(string cipherSignature) => cipherSignature[indexToSlice..];
}

View File

@@ -0,0 +1,12 @@
using System.Text;
namespace Manager.YouTube.Util.Cipher.Operations;
public class CipherSwap(int indexToSwap) : ICipherOperation
{
public string Decipher(string cipherSignature) => new StringBuilder(cipherSignature)
{
[0] = cipherSignature[indexToSwap],
[indexToSwap] = cipherSignature[0]
}.ToString();
}

View File

@@ -0,0 +1,6 @@
namespace Manager.YouTube.Util.Cipher.Operations;
public interface ICipherOperation
{
string Decipher(string cipherSignature);
}

View File

@@ -0,0 +1,27 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
namespace Manager.YouTube.Util.Converters;
public partial class JsonUrlEscapeConverter : JsonConverter<string>
{
public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var url = reader.GetString();
if (string.IsNullOrWhiteSpace(url))
{
return url;
}
return UrlPatternRegex().IsMatch(url) ? Uri.UnescapeDataString(url) : url;
}
public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options)
{
writer.WriteStringValue(value);
}
[GeneratedRegex("^(https?|ftp)://", RegexOptions.IgnoreCase | RegexOptions.Compiled, "nl-NL")]
private static partial Regex UrlPatternRegex();
}

View File

@@ -0,0 +1,39 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace Manager.YouTube.Util.Converters;
public class NumericJsonConverter<T> : JsonConverter<T> where T : struct, IConvertible
{
public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
try
{
if (reader.TokenType == JsonTokenType.Number)
{
// Direct numeric value
return (T)Convert.ChangeType(reader.GetDouble(), typeof(T));
}
if (reader.TokenType == JsonTokenType.String)
{
var str = reader.GetString();
if (string.IsNullOrWhiteSpace(str))
throw new JsonException("Empty string cannot be converted to a number.");
return (T)Convert.ChangeType(str, typeof(T));
}
throw new JsonException($"Unexpected token {reader.TokenType} for type {typeof(T)}.");
}
catch (Exception ex)
{
throw new JsonException($"Error converting value to {typeof(T)}.", ex);
}
}
public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options)
{
writer.WriteNumberValue(Convert.ToDouble(value));
}
}

View File

@@ -0,0 +1,121 @@
using System.Runtime.Serialization;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using DotBased.Logging;
using Manager.YouTube.Models;
using Manager.YouTube.Models.Innertube;
using Manager.YouTube.Parsers.Json;
namespace Manager.YouTube.Util.Converters;
public class YouTubeVideoJsonConverter : JsonConverter<YouTubeVideo>
{
private readonly ILogger _logger = LogService.RegisterLogger<YouTubeVideoJsonConverter>();
private readonly JsonSerializerOptions _serializerOptions = new()
{
Converters = {
new NumericJsonConverter<int>(),
new NumericJsonConverter<uint>(),
new NumericJsonConverter<long>(),
new NumericJsonConverter<double>(),
new NumericJsonConverter<decimal>() },
PropertyNameCaseInsensitive = true
};
public override YouTubeVideo Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
var node = JsonNode.Parse(ref reader);
if (node == null)
{
throw new SerializationException("Failed to parse JSON reader.");
}
var rootObject = node.AsObject();
var playabilityStatus = rootObject["playabilityStatus"];
var streamingDataJson = rootObject["streamingData"];
var videoDetails = rootObject["videoDetails"];
var playerConfigJson = rootObject["playerConfig"];
var microformat = rootObject["microformat"]?["playerMicroformatRenderer"];
var videoId = videoDetails?["videoId"]?.GetValue<string>() ?? microformat?["externalVideoId"]?.GetValue<string>();
if (string.IsNullOrEmpty(videoId))
{
throw new SerializationException("Failed to get videoId");
}
var thumbnails = JsonParser.ExtractWebImages(videoDetails?["thumbnail"]);
thumbnails.AddRange(JsonParser.ExtractWebImages(microformat?["thumbnail"]));
var streamingData = streamingDataJson.Deserialize<StreamingData>(_serializerOptions);
var playerConfig = ExtractPlayerConfig(playerConfigJson);
var video = new YouTubeVideo
{
VideoId = videoId,
Title = JsonParser.ExtractTextOrHtml(microformat?["title"]),
Description = JsonParser.ExtractTextOrHtml(microformat?["description"]),
ViewCount = long.TryParse(microformat?["viewCount"]?.GetValue<string>(), out var viewCountParsed) ? viewCountParsed : -1,
LikeCount = long.TryParse(microformat?["likeCount"]?.GetValue<string>(), out var likeCountParsed) ? likeCountParsed : -1,
ChannelId = videoDetails?["channelId"]?.GetValue<string>() ?? "",
Author = videoDetails?["author"]?.GetValue<string>() ?? "",
PlayabilityStatus = playabilityStatus?["status"]?.GetValue<string>() ?? "",
LengthSeconds = long.TryParse(videoDetails?["lengthSeconds"]?.GetValue<string>(), out var lengthSecondsParsed) ? lengthSecondsParsed : -1,
Keywords = videoDetails?["keywords"]?.AsArray().Select(v => v?.GetValue<string>() ?? "").ToArray() ?? [],
IsOwnerViewing = videoDetails?["isOwnerViewing"]?.GetValue<bool>() ?? false,
AllowRating = videoDetails?["allowRating"]?.GetValue<bool>() ?? false,
IsCrawlable = videoDetails?["isCrawlable"]?.GetValue<bool>() ?? false,
IsPrivate = videoDetails?["isPrivate"]?.GetValue<bool>() ?? false,
IsUnpluggedCorpus = videoDetails?["isUnpluggedCorpus"]?.GetValue<bool>() ?? false,
IsLive = videoDetails?["isLiveContent"]?.GetValue<bool>() ?? false,
IsFamilySave = microformat?["isFamilySave"]?.GetValue<bool>() ?? false,
AvailableCountries = microformat?["availableCountries"]?.AsArray().Select(v => v?.GetValue<string>() ?? "").ToArray() ?? [],
IsUnlisted = microformat?["isUnlisted"]?.GetValue<bool>() ?? false,
HasYpcMetadata = microformat?["hasYpcMetadata"]?.GetValue<bool>() ?? false,
PublishDate = DateTime.TryParse(microformat?["publishDate"]?.GetValue<string>(), out var parsedPublishDate) ? parsedPublishDate : DateTime.MinValue,
UploadDate = DateTime.TryParse(microformat?["uploadDate"]?.GetValue<string>(), out var parsedUploadDate) ? parsedUploadDate : DateTime.MinValue,
IsShortsEligible = microformat?["isShortsEligible"]?.GetValue<bool>() ?? false,
Category = microformat?["category"]?.GetValue<string>() ?? "",
StreamingData = streamingData,
Thumbnails = thumbnails,
PlayerConfig = playerConfig
};
return video;
}
public override void Write(Utf8JsonWriter writer, YouTubeVideo value, JsonSerializerOptions options)
{
throw new NotImplementedException("Converter only supports reading.");
}
private PlayerConfig? ExtractPlayerConfig(JsonNode? playerConfigNode)
{
if (playerConfigNode == null)
{
return null;
}
try
{
var playerConfigObj = playerConfigNode.AsObject();
var playerConfig = new PlayerConfig
{
AudioLoudnessDb = playerConfigObj["audioConfig"]?["loudnessDb"]?.GetValue<double>() ?? 0,
AudioPerceptualLoudnessDb = playerConfigObj["audioConfig"]?["perceptualLoudnessDb"]?.GetValue<double>() ?? 0,
AudioEnablePerFormatLoudness = playerConfigObj["audioConfig"]?["enablePerFormatLoudness"]?.GetValue<bool>() ?? false,
MaxBitrate = uint.TryParse(playerConfigObj["streamSelectionConfig"]?["maxBitrate"]?.GetValue<string>(), out var parsedMaxBitrate) ? parsedMaxBitrate : 0,
MaxReadAheadMediaTimeMs = playerConfigObj["mediaCommonConfig"]?["dynamicReadaheadConfig"]?["maxReadAheadMediaTimeMs"]?.GetValue<uint>() ?? 0,
MinReadAheadMediaTimeMs = playerConfigObj["mediaCommonConfig"]?["dynamicReadaheadConfig"]?["minReadAheadMediaTimeMs"]?.GetValue<uint>() ?? 0,
ReadAheadGrowthRateMs = playerConfigObj["mediaCommonConfig"]?["dynamicReadaheadConfig"]?["readAheadGrowthRateMs"]?.GetValue<uint>() ?? 0,
};
return playerConfig;
}
catch (Exception e)
{
_logger.Error(e, "Failed to extract player config from JSON.");
return null;
}
}
}

View File

@@ -3,7 +3,9 @@ using System.Net.Mime;
using System.Text; using System.Text;
using System.Text.Json; using System.Text.Json;
using System.Text.Json.Nodes; using System.Text.Json.Nodes;
using DotBased.Logging;
using DotBased.Monads; using DotBased.Monads;
using Manager.YouTube.Models;
using Manager.YouTube.Models.Innertube; using Manager.YouTube.Models.Innertube;
using Manager.YouTube.Parsers; using Manager.YouTube.Parsers;
using Manager.YouTube.Parsers.Json; using Manager.YouTube.Parsers.Json;
@@ -13,20 +15,23 @@ namespace Manager.YouTube;
public sealed class YouTubeClient : IDisposable public sealed class YouTubeClient : IDisposable
{ {
public string Id { get; private set; } = ""; public string Id { get; private set; } = "";
public string? UserAgent { get; set; } public string UserAgent { get; private set; }
public bool IsAnonymous { get; } public bool IsAnonymous { get; private set; }
public CookieContainer CookieContainer { get; } = new() { PerDomainCapacity = 50 }; public CookieContainer CookieContainer { get; } = new() { PerDomainCapacity = 50 };
public ClientState? State { get; private set; } public ClientState? State { get; private set; }
public List<string> DatasyncIds { get; } = []; public List<string> DatasyncIds { get; } = [];
public Cookie? SapisidCookie => CookieContainer.GetAllCookies()["SAPISID"]; public Cookie? SapisidCookie => CookieContainer.GetAllCookies()["SAPISID"];
public HttpClient HttpClient { get; } public HttpClient HttpClient { get; }
private YouTubeClient(CookieCollection? cookies, string userAgent) private readonly ILogger _logger;
private YouTubeClient(CookieCollection? cookies, string userAgent, ILogger logger)
{ {
if (string.IsNullOrWhiteSpace(userAgent)) if (string.IsNullOrWhiteSpace(userAgent))
{ {
throw new ArgumentNullException(nameof(userAgent)); throw new ArgumentNullException(nameof(userAgent));
} }
_logger = logger;
UserAgent = userAgent; UserAgent = userAgent;
if (cookies == null || cookies.Count == 0) if (cookies == null || cookies.Count == 0)
{ {
@@ -46,10 +51,13 @@ public sealed class YouTubeClient : IDisposable
/// </summary> /// </summary>
/// <param name="cookies">The cookies to use for making requests. Empty collection or null for anonymous requests.</param> /// <param name="cookies">The cookies to use for making requests. Empty collection or null for anonymous requests.</param>
/// <param name="userAgent">The user agent to use for the requests. Only WEB client is supported.</param> /// <param name="userAgent">The user agent to use for the requests. Only WEB client is supported.</param>
/// <param name="logger">The logger that the client is going to use, null will create a new logger.</param>
/// <returns></returns> /// <returns></returns>
public static async Task<Result<YouTubeClient>> CreateAsync(CookieCollection? cookies, string userAgent) public static async Task<Result<YouTubeClient>> CreateAsync(CookieCollection? cookies, string userAgent, ILogger? logger = null)
{ {
var client = new YouTubeClient(cookies, userAgent); logger ??= LogService.RegisterLogger<YouTubeClient>();
var client = new YouTubeClient(cookies, userAgent, logger);
var clientInitializeResult = await client.FetchClientDataAsync(); var clientInitializeResult = await client.FetchClientDataAsync();
if (!clientInitializeResult.IsSuccess) if (!clientInitializeResult.IsSuccess)
{ {
@@ -59,96 +67,55 @@ public sealed class YouTubeClient : IDisposable
return client; return client;
} }
private HttpClientHandler GetHttpClientHandler() public void SetUserAgent(string userAgent)
{ {
var clientHandler = new HttpClientHandler if (string.IsNullOrWhiteSpace(userAgent))
{ {
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip, _logger.Warning("UserAgent cannot be null or empty!");
UseCookies = true, return;
CookieContainer = CookieContainer }
}; UserAgent = userAgent;
return clientHandler;
} }
internal async Task<Result> FetchClientDataAsync() public async Task<Result<YouTubeVideo>> GetVideoByIdAsync(string videoId, CancellationToken cancellationToken = default)
{ {
if (State is not { LoggedIn: true }) if (string.IsNullOrWhiteSpace(videoId))
{ {
var state = await GetClientStateAsync(); return ResultError.Fail("Video id is empty!");
if (!state.IsSuccess)
{
return state;
}
} }
if (string.IsNullOrWhiteSpace(State?.WebPlayerContextConfig?.WebPlayerContext?.DatasyncId)) var request = new HttpRequestMessage(HttpMethod.Get, new Uri($"{NetworkService.Origin}/watch?v={videoId}"));
{
var datasyncResult = await GetDatasyncIds();
if (!datasyncResult.IsSuccess)
{
return datasyncResult;
}
foreach (var id in datasyncResult.Value)
{
if (DatasyncIds.Contains(id))
continue;
DatasyncIds.Add(id);
}
}
if (string.IsNullOrWhiteSpace(Id))
{
var accountInfoResult = await GetCurrentAccountIdAsync();
if (!accountInfoResult.IsSuccess)
{
return accountInfoResult;
}
Id = accountInfoResult.Value;
}
return Result.Success();
}
private async Task<Result> GetClientStateAsync()
{
var httpRequest = new HttpRequestMessage
{
Method = HttpMethod.Get,
RequestUri = new Uri(NetworkService.Origin)
};
var result = await NetworkService.MakeRequestAsync(httpRequest, this, true); var videoResponse = await NetworkService.MakeRequestAsync(request, this, true, cancellationToken);
if (!result.IsSuccess) if (!videoResponse.IsSuccess && !string.IsNullOrWhiteSpace(videoResponse.Value))
{ {
return result.Error ?? ResultError.Fail("Request failed!"); return videoResponse.Error ?? ResultError.Fail("Request failed!");
} }
var clientStateResult = HtmlParser.GetStateJson(result.Value); var html = videoResponse.Value;
if (clientStateResult is { IsSuccess: false, Error: not null })
{
return clientStateResult.Error;
}
try
{
State = JsonSerializer.Deserialize<ClientState>(clientStateResult.Value.Item1);
}
catch (Exception e)
{
return ResultError.Error(e, "Error while parsing JSON!");
}
if (State == null)
{
return ResultError.Fail("Unable to parse client state!");
}
State.IsPremiumUser = clientStateResult.Value.Item2;
return Result.Success(); var stateResult = GetClientStateFromHtml(html);
var state = stateResult.Value;
if (!stateResult.IsSuccess && State != null)
{
state = State;
}
var htmlParseResult = HtmlParser.GetVideoDataFromHtml(html);
if (!htmlParseResult.IsSuccess)
{
return htmlParseResult.Error ?? ResultError.Fail("Failed to parse HTML video data!");
}
var videoParseResult = VideoJsonParser.ParseVideoData(htmlParseResult.Value);
if (!videoParseResult.IsSuccess)
{
return videoParseResult;
}
//await DecipherSignaturesAsync(videoParseResult.Value, state);
return videoParseResult.Value;
} }
public async Task<Result<InnertubeChannel>> GetChannelByIdAsync(string channelId) public async Task<Result<InnertubeChannel>> GetChannelByIdAsync(string channelId)
@@ -231,11 +198,137 @@ public sealed class YouTubeClient : IDisposable
var rotateRequest = new HttpRequestMessage(HttpMethod.Post, new Uri("https://accounts.youtube.com/RotateCookies")); var rotateRequest = new HttpRequestMessage(HttpMethod.Post, new Uri("https://accounts.youtube.com/RotateCookies"));
return await NetworkService.MakeRequestAsync(rotateRequest, this, true); return await NetworkService.MakeRequestAsync(rotateRequest, this, true);
} }
public void Dispose() public void Dispose()
{ {
HttpClient.Dispose(); HttpClient.Dispose();
} }
private async Task<Result> FetchClientDataAsync()
{
if (State is not { LoggedIn: true })
{
var stateResult = await GetClientStateAsync();
if (!stateResult.IsSuccess)
{
return stateResult;
}
}
if (string.IsNullOrWhiteSpace(State?.WebPlayerContextConfig?.WebPlayerContext?.DatasyncId))
{
var datasyncResult = await GetDatasyncIdsAsync();
if (!datasyncResult.IsSuccess)
{
return datasyncResult;
}
foreach (var id in datasyncResult.Value)
{
if (DatasyncIds.Contains(id))
continue;
DatasyncIds.Add(id);
}
}
if (string.IsNullOrWhiteSpace(Id))
{
var accountInfoResult = await GetCurrentAccountIdAsync();
if (!accountInfoResult.IsSuccess)
{
return accountInfoResult;
}
Id = accountInfoResult.Value;
}
return Result.Success();
}
private HttpClientHandler GetHttpClientHandler()
{
var clientHandler = new HttpClientHandler
{
AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip,
UseCookies = true,
CookieContainer = CookieContainer
};
return clientHandler;
}
private async Task<Result> GetClientStateAsync()
{
var httpRequest = new HttpRequestMessage
{
Method = HttpMethod.Get,
RequestUri = new Uri(NetworkService.Origin)
};
var result = await NetworkService.MakeRequestAsync(httpRequest, this, true);
if (!result.IsSuccess)
{
return result.Error ?? ResultError.Fail("Request failed!");
}
var stateResult = SetClientStateFromHtml(result.Value);
if (!stateResult.IsSuccess)
{
return stateResult;
}
if (State is { LoggedIn: false })
{
_logger.Warning("Client is not logged in!");
return ResultError.Fail("Client login failed!");
}
var cookieRotationResult = await RotateCookiesPageAsync();
return !cookieRotationResult.IsSuccess ? cookieRotationResult : Result.Success();
}
private Result SetClientStateFromHtml(string html)
{
if (string.IsNullOrWhiteSpace(html))
{
return ResultError.Fail("HTML is empty!!");
}
var clientStateResult = GetClientStateFromHtml(html);
if (clientStateResult is { IsSuccess: false, Error: not null })
{
return clientStateResult.Error;
}
State = clientStateResult.Value;
IsAnonymous = !State.LoggedIn;
return Result.Success();
}
private Result<ClientState> GetClientStateFromHtml(string html)
{
var clientStateResult = HtmlParser.GetStateJson(html);
if (clientStateResult is { IsSuccess: false, Error: not null })
{
return clientStateResult.Error;
}
ClientState? state;
try
{
state = JsonSerializer.Deserialize<ClientState>(clientStateResult.Value.Item1);
if (state != null)
{
state.IsPremiumUser = clientStateResult.Value.Item2;
}
}
catch (Exception e)
{
return ResultError.Error(e, "Error while parsing JSON!");
}
return state == null ? ResultError.Fail("Unable to parse client state!") : state;
}
private async Task<Result<string>> GetCurrentAccountIdAsync() private async Task<Result<string>> GetCurrentAccountIdAsync()
{ {
@@ -262,7 +355,7 @@ public sealed class YouTubeClient : IDisposable
return JsonAccountParser.ParseAccountId(responseResult.Value); return JsonAccountParser.ParseAccountId(responseResult.Value);
} }
private async Task<Result<string[]>> GetDatasyncIds() private async Task<Result<string[]>> GetDatasyncIdsAsync()
{ {
if (State is not { LoggedIn: true } || CookieContainer.Count == 0) if (State is not { LoggedIn: true } || CookieContainer.Count == 0)
{ {
@@ -291,4 +384,49 @@ public sealed class YouTubeClient : IDisposable
return ResultError.Fail("Failed to get datasyncIds! Client not logged in."); return ResultError.Fail("Failed to get datasyncIds! Client not logged in.");
} }
/*private async Task DecipherSignaturesAsync(YouTubeVideo video, ClientState state)
{
var streamingData = video.StreamingData;
if (streamingData == null)
{
_logger.Debug("No streaming data available, skipping decipher.");
return;
}
if (string.IsNullOrWhiteSpace(state.PlayerJsUrl))
{
_logger.Warning("No player js url found.");
}
var formatsWithCipher = streamingData.Formats.Concat(streamingData.AdaptiveFormats).Where(x => !string.IsNullOrWhiteSpace(x.SignatureCipher)).ToList();
if (formatsWithCipher.Count == 0)
{
_logger.Debug("Skipping signature decipher, no signatures found to decipher.");
}
if (string.IsNullOrWhiteSpace(streamingData.ServerAbrStreamingUrl))
{
_logger.Warning("No ABR streaming url available.");
}
var abrStreamUri = new Uri(streamingData.ServerAbrStreamingUrl);
var queries = HttpUtility.ParseQueryString(abrStreamUri.Query);
var nSig = queries.Get("n");
if (string.IsNullOrWhiteSpace(nSig))
{
_logger.Warning("No N signature found.");
}
/*var jsEngineResult = await JavaScriptEngineManager.GetPlayerEngine(state.PlayerJsUrl ?? "");
if (!jsEngineResult.IsSuccess)
{
_logger.Warning(jsEngineResult.Error?.Description ?? "Failed to get player script engine.");
return;
}
var engine = jsEngineResult.Value;
engine.InitializePlayer();#1#
}*/
} }

View File

@@ -8,4 +8,17 @@ public class YouTubeClientCollection : KeyedCollection<string, YouTubeClient>
{ {
return item.Id; return item.Id;
} }
public new void Add(YouTubeClient client)
{
if (Items.Contains(client))
{
var index = Items.IndexOf(client);
Items[index] = client;
}
else
{
Items.Add(client);
}
}
} }