diff --git a/Manager.YouTube/Models/YouTubeVideo.cs b/Manager.YouTube/Models/YouTubeVideo.cs index 0d43844..2ab2f49 100644 --- a/Manager.YouTube/Models/YouTubeVideo.cs +++ b/Manager.YouTube/Models/YouTubeVideo.cs @@ -1,61 +1,36 @@ -using System.Text.Json.Serialization; using Manager.YouTube.Models.Innertube; namespace Manager.YouTube.Models; public class YouTubeVideo { - [JsonPropertyName("videoId")] public required string VideoId { get; set; } - [JsonPropertyName("title")] public string Title { get; set; } = ""; - [JsonPropertyName("description")] public string Description { get; set; } = ""; - [JsonPropertyName("hashTags")] public string[] HashTags { get; set; } = []; - [JsonPropertyName("viewCount")] public long ViewCount { get; set; } - [JsonPropertyName("likeCount")] public long LikeCount { get; set; } - [JsonPropertyName("channelId")] public string ChannelId { get; set; } = ""; - [JsonPropertyName("author")] public string Author { get; set; } = ""; - [JsonPropertyName("playabilityStatus")] public string PlayabilityStatus { get; set; } = ""; - [JsonPropertyName("lengthSeconds")] public long LengthSeconds { get; set; } - [JsonPropertyName("isOwnerViewing")] + public string[] Keywords { get; set; } = []; public bool IsOwnerViewing { get; set; } - [JsonPropertyName("allowRating")] public bool AllowRating { get; set; } - [JsonPropertyName("isCrawlable")] public bool IsCrawlable { get; set; } - [JsonPropertyName("isPrivate")] public bool IsPrivate { get; set; } - [JsonPropertyName("isUnpluggedCorpus")] public bool IsUnpluggedCorpus { get; set; } - [JsonPropertyName("isLive")] public bool IsLive { get; set; } - [JsonPropertyName("isFamilySafe")] public bool IsFamilySave { get; set; } - [JsonPropertyName("availableCountries")] public string[] AvailableCountries { get; set; } = []; - [JsonPropertyName("isUnlisted")] public bool IsUnlisted { get; set; } - [JsonPropertyName("hasYpcMetadata")] public bool HasYpcMetadata { get; set; } - [JsonPropertyName("publishDate")] public DateTime PublishDate { get; set; } - [JsonPropertyName("uploadDate")] public DateTime UploadDate { get; set; } - [JsonPropertyName("isShortsEligible")] public bool IsShortsEligible { get; set; } - [JsonPropertyName("category")] public string Category { get; set; } = ""; - public StreamingData StreamingData { get; set; } = new(); - [JsonPropertyName("thumbnail")] + public StreamingData? StreamingData { get; set; } public List Thumbnails { get; set; } = []; public PlayerConfig? PlayerConfig { get; set; } diff --git a/Manager.YouTube/Parsers/Json/JsonParser.cs b/Manager.YouTube/Parsers/Json/JsonParser.cs index 3ae103a..def4fa8 100644 --- a/Manager.YouTube/Parsers/Json/JsonParser.cs +++ b/Manager.YouTube/Parsers/Json/JsonParser.cs @@ -1,4 +1,6 @@ +using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; using Manager.YouTube.Models.Innertube; namespace Manager.YouTube.Parsers.Json; @@ -9,4 +11,63 @@ public static class JsonParser array .Select(image => new WebImage { Width = image.GetProperty("width").GetInt32(), Height = image.GetProperty("height").GetInt32(), Url = image.GetProperty("url").GetString() ?? "" }) .ToList(); + + public static string ExtractTextOrHtml(JsonElement element) + { + // Case 1: Simple text (no formatting) + if (element.TryGetProperty("simpleText", out var simpleText)) + return simpleText.GetString() ?? string.Empty; + + // Case 2: Runs (formatted text segments) + if (element.TryGetProperty("runs", out var runs) && runs.ValueKind == JsonValueKind.Array) + { + var sb = new StringBuilder(); + + foreach (var run in runs.EnumerateArray()) + { + var text = run.GetProperty("text").GetString() ?? string.Empty; + var formatted = System.Net.WebUtility.HtmlEncode(text); + + var bold = run.TryGetProperty("bold", out var boldProp) && boldProp.GetBoolean(); + var italic = run.TryGetProperty("italic", out var italicProp) && italicProp.GetBoolean(); + var underline = run.TryGetProperty("underline", out var underlineProp) && underlineProp.GetBoolean(); + var strikethrough = run.TryGetProperty("strikethrough", out var strikeProp) && strikeProp.GetBoolean(); + + if (bold) formatted = $"{formatted}"; + if (italic) formatted = $"{formatted}"; + if (underline) formatted = $"{formatted}"; + if (strikethrough) formatted = $"{formatted}"; + + if (run.TryGetProperty("navigationEndpoint", out var nav) && + nav.TryGetProperty("url", out var urlProp)) + { + var url = urlProp.GetString(); + if (!string.IsNullOrEmpty(url)) + formatted = $"{formatted}"; + } + + if (run.TryGetProperty("emoji", out var emoji) && emoji.ValueKind == JsonValueKind.Object) + { + if (emoji.TryGetProperty("url", out var emojiUrl)) + { + var src = emojiUrl.GetString(); + if (!string.IsNullOrEmpty(src)) + formatted = $"\"{text}\""; + } + } + + sb.Append(formatted); + } + + return sb.ToString(); + } + + return string.Empty; + } + + public static List ExtractWebImages(JsonElement element) + { + var thumbnailsArray = element.GetProperty("thumbnail").GetProperty("thumbnails"); + return thumbnailsArray.Deserialize>() ?? []; + } } \ No newline at end of file diff --git a/Manager.YouTube/Parsers/Json/VideoJsonParser.cs b/Manager.YouTube/Parsers/Json/VideoJsonParser.cs index c17465d..eadcecb 100644 --- a/Manager.YouTube/Parsers/Json/VideoJsonParser.cs +++ b/Manager.YouTube/Parsers/Json/VideoJsonParser.cs @@ -1,51 +1,35 @@ using System.Text.Json; -using System.Text.Json.Nodes; +using DotBased.Logging; using DotBased.Monads; using Manager.YouTube.Models; using Manager.YouTube.Models.Parser; +using Manager.YouTube.Util.Converters; namespace Manager.YouTube.Parsers.Json; public static class VideoJsonParser { + private static readonly JsonSerializerOptions VideoParserOptions = new() { Converters = { new YouTubeVideoJsonConverter() } }; + private static readonly ILogger Logger = LogService.RegisterLogger(typeof(VideoJsonParser), "Video JSON parser"); + public static Result ParseVideoData(YouTubeVideoData videoData) { if (videoData.YouTubeInitialData == null || videoData.YouTubeInitialData.Count == 0) { return ResultError.Fail("No initial video data found!"); } - - var videoDetails = videoData.YouTubeInitialData["videoDetails"]; - var microformat = videoData.YouTubeInitialData["microformat"]?["playerMicroformatRenderer"]; - if (videoDetails == null) + + YouTubeVideo? video; + try { - return ResultError.Fail("No video details found!"); + video = videoData.YouTubeInitialData.Deserialize(VideoParserOptions); } - - if (microformat == null) + catch (Exception e) { - return ResultError.Fail("No microformat found!"); + Logger.Error(e, "Failed to parse video data"); + return ResultError.Fail("Failed to parse video data"); } - - FlattenThumbnailArray(videoDetails); - FlattenThumbnailArray(microformat); - - - - - var video = videoDetails.Deserialize(); - - - return ResultError.Fail("Not implemented."); - } - - private static void FlattenThumbnailArray(JsonNode node) - { - var thumbnailsArray = node["thumbnail"]?["thumbnails"]; - if (thumbnailsArray != null) - { - node["thumbnail"]?.ReplaceWith(thumbnailsArray); - } + return video != null? video : ResultError.Fail("Failed to parse video data!"); } } \ No newline at end of file diff --git a/Manager.YouTube/Util/Converters/YouTubeVideoJsonConverter.cs b/Manager.YouTube/Util/Converters/YouTubeVideoJsonConverter.cs new file mode 100644 index 0000000..cdeb522 --- /dev/null +++ b/Manager.YouTube/Util/Converters/YouTubeVideoJsonConverter.cs @@ -0,0 +1,93 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using DotBased.Logging; +using Manager.YouTube.Models; +using Manager.YouTube.Models.Innertube; +using Manager.YouTube.Parsers.Json; + +namespace Manager.YouTube.Util.Converters; + +public class YouTubeVideoJsonConverter : JsonConverter +{ + private readonly ILogger _logger = LogService.RegisterLogger(); + + public override YouTubeVideo Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + using var document = JsonDocument.ParseValue(ref reader); + var root = document.RootElement; + + var playabilityStatus = root.GetProperty("playabilityStatus"); + var streamingData = root.GetProperty("streamingData"); + var videoDetails = root.GetProperty("videoDetails"); + var playerConfigJson = root.GetProperty("playerConfig"); + var microformat = root.GetProperty("microformat").GetProperty("playerMicroformatRenderer"); + + var thumbnails = JsonParser.ExtractWebImages(videoDetails.GetProperty("thumbnail")); + thumbnails.AddRange(JsonParser.ExtractWebImages(microformat.GetProperty("thumbnail"))); + + var video = new YouTubeVideo + { + VideoId = videoDetails.GetProperty("videoId").GetString() ?? "", + Title = JsonParser.ExtractTextOrHtml(microformat.GetProperty("title")), + Description = JsonParser.ExtractTextOrHtml(microformat.GetProperty("description")), + ViewCount = videoDetails.GetProperty("viewCount").GetInt32(), + LikeCount = videoDetails.GetProperty("likeCount").GetInt32(), + ChannelId = videoDetails.GetProperty("channelId").GetString() ?? "", + Author = JsonParser.ExtractTextOrHtml(videoDetails.GetProperty("author")), + PlayabilityStatus = playabilityStatus.GetProperty("status").GetString() ?? "", + LengthSeconds = videoDetails.GetProperty("lengthSeconds").GetInt32(), + Keywords = videoDetails.GetProperty("keywords").EnumerateArray().Select(v => v.GetString()).Cast().ToArray(), + IsOwnerViewing = videoDetails.GetProperty("isOwnerViewing").GetBoolean(), + AllowRating = videoDetails.GetProperty("allowRating").GetBoolean(), + IsCrawlable = videoDetails.GetProperty("isCrawlable").GetBoolean(), + IsPrivate = videoDetails.GetProperty("isPrivate").GetBoolean(), + IsUnpluggedCorpus = videoDetails.GetProperty("isUnpluggedCorpus").GetBoolean(), + IsLive = videoDetails.GetProperty("isLiveContent").GetBoolean(), + IsFamilySave = microformat.GetProperty("isFamilySave").GetBoolean(), + AvailableCountries = microformat.GetProperty("availableCountries").EnumerateArray().Select(v => v.GetString()).Cast().ToArray(), + IsUnlisted = microformat.GetProperty("isUnlisted").GetBoolean(), + HasYpcMetadata = microformat.GetProperty("hasYpcMetadata").GetBoolean(), + PublishDate = microformat.GetProperty("publishDate").GetDateTime(), + UploadDate = microformat.GetProperty("uploadDate").GetDateTime(), + IsShortsEligible = microformat.GetProperty("isShortsEligible").GetBoolean(), + Category = microformat.GetProperty("category").GetString() ?? "", + StreamingData = streamingData.Deserialize(), + Thumbnails = thumbnails, + PlayerConfig = ExtractPlayerConfig(playerConfigJson) + }; + + return video; + } + + public override void Write(Utf8JsonWriter writer, YouTubeVideo value, JsonSerializerOptions options) + { + throw new NotImplementedException("Converter only supports reading."); + } + + private PlayerConfig? ExtractPlayerConfig(JsonElement element) + { + try + { + var playerConfig = new PlayerConfig + { + AudioLoudnessDb = element.GetProperty("audioConfig").GetProperty("loudnessDb").GetDouble(), + AudioPerceptualLoudnessDb = element.GetProperty("audioConfig").GetProperty("perceptualLoudnessDb").GetDouble(), + AudioEnablePerFormatLoudness = element.GetProperty("audioConfig").GetProperty("enablePerFormatLoudness") + .GetBoolean(), + MaxBitrate = element.GetProperty("streamSelectionConfig").GetProperty("maxBitrate").GetUInt32(), + MaxReadAheadMediaTimeMs = element.GetProperty("mediaCommonConfig").GetProperty("dynamicReadaheadConfig") + .GetProperty("maxReadAheadMediaTimeMs").GetUInt32(), + MinReadAheadMediaTimeMs = element.GetProperty("mediaCommonConfig").GetProperty("dynamicReadaheadConfig") + .GetProperty("minReadAheadMediaTimeMs").GetUInt32(), + ReadAheadGrowthRateMs = element.GetProperty("mediaCommonConfig").GetProperty("dynamicReadaheadConfig") + .GetProperty("readAheadGrowthRateMs").GetUInt32(), + }; + return playerConfig; + } + catch (Exception e) + { + _logger.Error(e, "Failed to extract player config from JSON."); + return null; + } + } +} \ No newline at end of file diff --git a/Manager.YouTube/YouTubeClient.cs b/Manager.YouTube/YouTubeClient.cs index 64115f2..15cd925 100644 --- a/Manager.YouTube/YouTubeClient.cs +++ b/Manager.YouTube/YouTubeClient.cs @@ -4,6 +4,7 @@ using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using DotBased.Monads; +using Manager.YouTube.Models; using Manager.YouTube.Models.Innertube; using Manager.YouTube.Parsers; using Manager.YouTube.Parsers.Json; @@ -308,7 +309,7 @@ public sealed class YouTubeClient : IDisposable return ResultError.Fail("Failed to get datasyncIds! Client not logged in."); } - public async Task GetVideoByIdAsync(string videoId, CancellationToken cancellationToken = default) + public async Task> GetVideoByIdAsync(string videoId, CancellationToken cancellationToken = default) { if (string.IsNullOrWhiteSpace(videoId)) { @@ -331,14 +332,20 @@ public sealed class YouTubeClient : IDisposable //TODO: Log warning: failed to update client state! } - var htmlParseReult = HtmlParser.GetVideoDataFromHtml(html); - if (!htmlParseReult.IsSuccess) + var htmlParseResult = HtmlParser.GetVideoDataFromHtml(html); + if (!htmlParseResult.IsSuccess) { - return htmlParseReult; + return htmlParseResult.Error ?? ResultError.Fail("Failed to parse HTML video data!"); } - //TODO: Process to YouTubeVideo model && decipher stream urls + var videoParseResult = VideoJsonParser.ParseVideoData(htmlParseResult.Value); + if (!videoParseResult.IsSuccess) + { + return videoParseResult; + } + + //TODO: decipher stream urls - return Result.Success(); + return videoParseResult.Value; } } \ No newline at end of file