[CHANGE] Linked decipher to video implementation

Manager.YouTube/Models/Innertube/ClientState.cs

@@ -35,6 +35,9 @@ public class ClientState : AdditionalJsonData
     [JsonPropertyName("SERVER_VERSION")]
     public string? ServerVersion { get; set; }
 
+    [JsonPropertyName("PLAYER_JS_URL")]
+    public string? PlayerJsUrl { get; set; }
+
     [JsonPropertyName("INNERTUBE_CONTEXT")]
     public InnerTubeContext? InnerTubeContext { get; set; }
 

Manager.YouTube/Models/Innertube/StreamingFormat.cs

@@ -3,7 +3,7 @@ namespace Manager.YouTube.Models.Innertube;
 public class StreamingFormat
 {
     public int Itag { get; set; }
-    public string Url { get; set; } = "";
+    public string? Url { get; set; }
     public string MimeType { get; set; } = "";
     public uint Bitrate { get; set; }
     public uint? Width { get; set; }
@@ -26,5 +26,6 @@ public class StreamingFormat
     public int? AudioChannels { get; set; }
     public double? LoudnessDb { get; set; }
     public bool? IsDrc { get; set; }
+    public string? SignatureCipher { get; set; }
     public string QualityOrdinal { get; set; } = "";
 }

Manager.YouTube/Util/Cipher/CipherDecoder.cs

@@ -0,0 +1,127 @@
+using System.Collections.Frozen;
+using System.Text;
+using System.Text.RegularExpressions;
+using Manager.YouTube.Util.Cipher.Operations;
+
+namespace Manager.YouTube.Util.Cipher;
+
+public partial class CipherDecoder
+{
+    public required string Version { get; init; }
+    public required string OriginalRelativeUrl { get; init; }
+    public readonly IReadOnlySet<ICipherOperation> Operations;
+
+    private CipherDecoder(IEnumerable<ICipherOperation> operations)
+    {
+        Operations = operations.ToFrozenSet();
+        if (Operations.Count == 0)
+        {
+            throw new ArgumentNullException(nameof(operations), "No decipher operations given.");
+        }
+    }
+
+    public static async Task<CipherDecoder> CreateAsync(string relativeUrl, string version, YouTubeClient? client = null)
+    {
+        var operations = await GetCipherOperations(relativeUrl, client);
+        var decoder = new CipherDecoder(operations)
+        {
+            OriginalRelativeUrl = relativeUrl,
+            Version = version
+        };
+        return decoder;
+    }
+
+    public string Decipher(string? signatureCipher)
+    {
+        if (string.IsNullOrEmpty(signatureCipher))
+        {
+            return "";
+        }
+
+        var urlBuilder = new StringBuilder();
+
+        var indexStart = signatureCipher.IndexOf("s=", StringComparison.Ordinal);
+        var indexEnd = signatureCipher.IndexOf("&", StringComparison.Ordinal);
+        var signature = signatureCipher.Substring(indexStart, indexEnd);
+
+        indexStart = signatureCipher.IndexOf("&sp", StringComparison.Ordinal);
+        indexEnd = signatureCipher.IndexOf("&url", StringComparison.Ordinal);
+        var spParam = signatureCipher.Substring(indexStart, indexEnd - indexStart);
+
+        indexStart = signatureCipher.IndexOf("&url", StringComparison.Ordinal);
+        var videoUrl = signatureCipher[indexStart..];
+
+
+        signature = signature[(signature.IndexOf('=') + 1)..];
+        spParam = spParam[(spParam.IndexOf('=') + 1)..];
+        videoUrl = videoUrl[(videoUrl.IndexOf('=') + 1)..];
+        if (string.IsNullOrWhiteSpace(signature))
+        {
+            throw new InvalidOperationException("Invalid signature.");
+        }
+        var signatureDeciphered = Operations.Aggregate(signature, (acc, op) => op.Decipher(acc));
+
+        urlBuilder.Append(videoUrl);
+        urlBuilder.Append($"&{spParam}=");
+        urlBuilder.Append(signatureDeciphered);
+        return urlBuilder.ToString();
+    }
+
+    private static async Task<IEnumerable<ICipherOperation>> GetCipherOperations(string relativeUrl, YouTubeClient? client = null)
+    {
+        var downloadRequest = new HttpRequestMessage(HttpMethod.Get, new Uri($"{NetworkService.Origin}/{relativeUrl}"));
+        var downloadResponse = await NetworkService.DownloadBytesAsync(downloadRequest, client);
+        if (!downloadResponse.IsSuccess)
+        {
+            return [];
+        }
+
+        var playerJs = Encoding.UTF8.GetString(downloadResponse.Value.Data);
+        if (string.IsNullOrWhiteSpace(playerJs))
+        {
+            return [];
+        }
+        
+        var functionBody = FunctionBodyRegex().Match(playerJs).Groups[0].ToString();
+        var definitionBody = DefinitionBodyRegex().Match(functionBody).Groups[1].Value;
+        var decipherDefinition = Regex.Match(playerJs, $@"var\s+{definitionBody}=\{{(\w+:function\(\w+(,\w+)?\)\{{(.*?)\}}),?\}};", RegexOptions.Singleline).Groups[0].ToString();
+
+        SortedSet<ICipherOperation> operations = [];
+        foreach (var statement in functionBody.Split(';'))
+        {
+            // Get the name of the function called in this statement
+            var calledFuncName = StatementFunctionNameRegex().Match(statement).Groups[1].Value;
+            if (string.IsNullOrWhiteSpace(calledFuncName))
+                continue;
+
+            if (Regex.IsMatch(decipherDefinition, $@"{Regex.Escape(calledFuncName)}:\bfunction\b\([a],b\).(\breturn\b)?.?\w+\."))
+            {
+                var index = int.Parse(OperationIndexRegex().Match(statement).Groups[1].Value);
+                operations.Add(new CipherSlice(index));
+            }
+            else if (Regex.IsMatch(decipherDefinition, $@"{Regex.Escape(calledFuncName)}:\bfunction\b\(\w+\,\w\).\bvar\b.\bc=a\b"))
+            {
+                var index = int.Parse(OperationIndexRegex().Match(statement).Groups[1].Value);
+                operations.Add(new CipherSwap(index));
+            }
+            else if (Regex.IsMatch(decipherDefinition, $@"{Regex.Escape(calledFuncName)}:\bfunction\b\(\w+\)"))
+            {
+                operations.Add(new CipherReverse());
+            }
+        }
+
+        return operations;
+    }
+
+    
+    [GeneratedRegex(@"(\w+)=function\(\w+\){(\w+)=\2\.split\(\x22{2}\);.*?return\s+\2\.join\(\x22{2}\)}")]
+    private static partial Regex FunctionBodyRegex();
+    [GeneratedRegex("([\\$_\\w]+).\\w+\\(\\w+,\\d+\\);")]
+    private static partial Regex DefinitionBodyRegex();
+    
+    
+    [GeneratedRegex(@"\(\w+,(\d+)\)")]
+    private static partial Regex OperationIndexRegex();
+    [GeneratedRegex(@"\w+(?:.|\[)(\""?\w+(?:\"")?)\]?\(")]
+    private static partial Regex StatementFunctionNameRegex();
+}

Manager.YouTube/Util/Cipher/CipherDecoderCollection.cs

@@ -0,0 +1,11 @@
+using System.Collections.ObjectModel;
+
+namespace Manager.YouTube.Util.Cipher;
+
+public class CipherDecoderCollection : KeyedCollection<string, CipherDecoder>
+{
+    protected override string GetKeyForItem(CipherDecoder item)
+    {
+        return item.Version;
+    }
+}

Manager.YouTube/Util/Cipher/CipherManager.cs

@@ -0,0 +1,46 @@
+using DotBased.Logging;
+using DotBased.Monads;
+
+namespace Manager.YouTube.Util.Cipher;
+
+public static class CipherManager
+{
+    private static readonly CipherDecoderCollection LoadedCiphers = [];
+    private static readonly ILogger Logger = LogService.RegisterLogger(typeof(CipherManager));
+
+    public static async Task<Result<CipherDecoder>> GetDecoderAsync(YouTubeClient client)
+    {
+        var relativePlayerJsUrl = client.State?.PlayerJsUrl;
+        if (string.IsNullOrEmpty(relativePlayerJsUrl))
+        {
+            return ResultError.Fail("Could not get player js url.");
+        }
+        var version = GetCipherVersion(relativePlayerJsUrl);
+
+        Logger.Debug($"Getting cipher decoder for version: {version}");
+        if (LoadedCiphers.TryGetValue(version, out var cipher))
+        {
+            return cipher;
+        }
+        
+        try
+        {
+            var decoder = await CipherDecoder.CreateAsync(relativePlayerJsUrl, version);
+            LoadedCiphers.Add(decoder);
+        }
+        catch (Exception e)
+        {
+            Logger.Error(e, "Could not create cipher decoder. Version: {DecoderVersion}", version);
+        }
+        
+        return ResultError.Fail($"Could not create cipher decoder for {relativePlayerJsUrl} (v: {version})");
+    }
+
+    private static string GetCipherVersion(string relativePlayerUrl)
+    {
+        var split = relativePlayerUrl.Split('/');
+        var v = split[2];
+        var lang = split[4];
+        return $"{v}_{lang}";
+    }
+}

Manager.YouTube/Util/Cipher/Operations/CipherReverse.cs

@@ -0,0 +1,18 @@
+using System.Text;
+
+namespace Manager.YouTube.Util.Cipher.Operations;
+
+public class CipherReverse : ICipherOperation
+{
+    public string Decipher(string cipherSignature)
+    {
+        var buffer = new StringBuilder(cipherSignature.Length);
+
+        for (var i = cipherSignature.Length - 1; i >= 0; i--)
+        {
+            buffer.Append(cipherSignature[i]);
+        }
+
+        return buffer.ToString();
+    }
+}

Manager.YouTube/Util/Cipher/Operations/CipherSlice.cs

@@ -0,0 +1,6 @@
+namespace Manager.YouTube.Util.Cipher.Operations;
+
+public class CipherSlice(int indexToSlice) : ICipherOperation
+{
+    public string Decipher(string cipherSignature) => cipherSignature[indexToSlice..];
+}

Manager.YouTube/Util/Cipher/Operations/CipherSwap.cs

@@ -0,0 +1,12 @@
+using System.Text;
+
+namespace Manager.YouTube.Util.Cipher.Operations;
+
+public class CipherSwap(int indexToSwap) : ICipherOperation
+{
+    public string Decipher(string cipherSignature) => new StringBuilder(cipherSignature)
+    {
+        [0] = cipherSignature[indexToSwap],
+        [indexToSwap] = cipherSignature[0]
+    }.ToString();
+}

Manager.YouTube/Util/Cipher/Operations/ICipherOperation.cs

@@ -0,0 +1,6 @@
+namespace Manager.YouTube.Util.Cipher.Operations;
+
+public interface ICipherOperation
+{
+    string Decipher(string cipherSignature);
+}

Manager.YouTube/Util/Converters/YouTubeVideoJsonConverter.cs

@@ -1,3 +1,4 @@
+using System.Runtime.Serialization;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using DotBased.Logging;
@@ -21,13 +22,19 @@ public class YouTubeVideoJsonConverter : JsonConverter<YouTubeVideo>
         var videoDetails = root.GetProperty("videoDetails");
         var playerConfigJson = root.GetProperty("playerConfig");
         var microformat = root.GetProperty("microformat").GetProperty("playerMicroformatRenderer");
+        
+        var videoId = videoDetails.GetProperty("videoId").GetString() ?? microformat.GetProperty("externalVideoId").GetString();
+        if (string.IsNullOrEmpty(videoId))
+        {
+            throw new SerializationException("Failed to get videoId");
+        }
 
         var thumbnails = JsonParser.ExtractWebImages(videoDetails.GetProperty("thumbnail"));
         thumbnails.AddRange(JsonParser.ExtractWebImages(microformat.GetProperty("thumbnail")));
         
         var video = new YouTubeVideo
         {
-            VideoId = videoDetails.GetProperty("videoId").GetString() ?? "",
+            VideoId = videoId,
             Title = JsonParser.ExtractTextOrHtml(microformat.GetProperty("title")),
             Description = JsonParser.ExtractTextOrHtml(microformat.GetProperty("description")),
             ViewCount = videoDetails.GetProperty("viewCount").GetInt32(),

Manager.YouTube/YouTubeClient.cs

@@ -3,11 +3,13 @@ using System.Net.Mime;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Nodes;
+using DotBased.Logging;
 using DotBased.Monads;
 using Manager.YouTube.Models;
 using Manager.YouTube.Models.Innertube;
 using Manager.YouTube.Parsers;
 using Manager.YouTube.Parsers.Json;
+using Manager.YouTube.Util.Cipher;
 
 namespace Manager.YouTube;
 
@@ -22,12 +24,15 @@ public sealed class YouTubeClient : IDisposable
     public Cookie? SapisidCookie => CookieContainer.GetAllCookies()["SAPISID"];
     public HttpClient HttpClient { get; }
     
-    private YouTubeClient(CookieCollection? cookies, string userAgent)
+    private readonly ILogger _logger;
+    
+    private YouTubeClient(CookieCollection? cookies, string userAgent, ILogger logger)
     {
         if (string.IsNullOrWhiteSpace(userAgent))
         {
             throw new ArgumentNullException(nameof(userAgent));
         }
+        _logger = logger;
         UserAgent = userAgent;
         if (cookies == null || cookies.Count == 0)
         {
@@ -47,10 +52,13 @@ public sealed class YouTubeClient : IDisposable
     /// </summary>
     /// <param name="cookies">The cookies to use for making requests. Empty collection or null for anonymous requests.</param>
     /// <param name="userAgent">The user agent to use for the requests. Only WEB client is supported.</param>
+    /// <param name="logger">The logger that the client is going to use, if null will create a new logger.</param>
     /// <returns></returns>
-    public static async Task<Result<YouTubeClient>> CreateAsync(CookieCollection? cookies, string userAgent)
+    public static async Task<Result<YouTubeClient>> CreateAsync(CookieCollection? cookies, string userAgent, ILogger? logger = null)
     {
-        var client = new YouTubeClient(cookies, userAgent);
+        logger ??= LogService.RegisterLogger<YouTubeClient>();
+        
+        var client = new YouTubeClient(cookies, userAgent, logger);
         var clientInitializeResult = await client.FetchClientDataAsync();
         if (!clientInitializeResult.IsSuccess)
         {
@@ -329,7 +337,7 @@ public sealed class YouTubeClient : IDisposable
         var stateResult = SetClientStateFromHtml(html);
         if (!stateResult.IsSuccess)
         {
-            //TODO: Log warning: failed to update client state!
+            _logger.Warning("Failed to update client state.");
         }
         
         var htmlParseResult = HtmlParser.GetVideoDataFromHtml(html);
@@ -344,8 +352,38 @@ public sealed class YouTubeClient : IDisposable
             return videoParseResult;
         }
         
-        //TODO: decipher stream urls
+        await DecipherSignatures(videoParseResult.Value);
 
         return videoParseResult.Value;
     }
+
+    private async Task DecipherSignatures(YouTubeVideo video)
+    {
+        var streamingData = video.StreamingData;
+        if (streamingData == null)
+        {
+            _logger.Debug("No streaming data available, skipping decipher.");
+            return;
+        }
+
+        var formatsWithCipher = streamingData.Formats.Concat(streamingData.AdaptiveFormats).Where(x => !string.IsNullOrWhiteSpace(x.SignatureCipher)).ToList();
+        if (formatsWithCipher.Count == 0)
+        {
+            _logger.Debug("Skipping decipher, no signatures found to decipher.");
+            return;
+        }
+
+        var decipherDecoderResult = await CipherManager.GetDecoderAsync(this);
+        if (!decipherDecoderResult.IsSuccess)
+        {
+            _logger.Warning(decipherDecoderResult.Error?.Description ?? "Failed to get the cipher decoder!");
+            return;
+        }
+        var decoder = decipherDecoderResult.Value;
+        
+        foreach (var format in formatsWithCipher)
+        {
+            format.Url = decoder.Decipher(format.SignatureCipher);
+        }
+    }
 }