Add API support for ELRC word-based lyrics (#12941)

* Add API support for ELRC word-based lyrics

Adds support for word-based timestamps from within ELRC files.

* Create TimeTags object

* redo TimeTag implementation

Change TimeTag to long, redo TimeTag implementation
Make timestamp not nullable
Update MediaBrowser.Model/Lyrics/LyricLine.cs
Make TimeTag list IReadOnlyList
Remove nullable Timestamp
Update TimeTag description

Co-Authored-By: Cody Robibero <cody@robibe.ro>

* Changes to LyricLineTimeTag

Moved TimeTag to LyricLineTimeTag
Change "timestamp" to "start" for consistency
Change plural "TimeTags" to "Cues"
Change comments

* Change LyricLineTimeTag to LyricLineCue, include info about end times

* Remove width

* Remove width tag

* Rewrite cue parser and add tests

---------

Co-authored-by: Cody Robibero <cody@robibe.ro>

MediaBrowser.Model/Lyrics/LyricLine.cs

@@ -1,3 +1,5 @@
+using System.Collections.Generic;
+
 namespace MediaBrowser.Model.Lyrics;
 
 /// <summary>
@@ -10,10 +12,12 @@ public class LyricLine
     /// </summary>
     /// <param name="text">The lyric text.</param>
     /// <param name="start">The lyric start time in ticks.</param>
-    public LyricLine(string text, long? start = null)
+    /// <param name="cues">The time-aligned cues for the song's lyrics.</param>
+    public LyricLine(string text, long? start = null, IReadOnlyList<LyricLineCue>? cues = null)
     {
         Text = text;
         Start = start;
+        Cues = cues;
     }
 
     /// <summary>
@@ -25,4 +29,9 @@ public class LyricLine
     /// Gets the start time in ticks.
     /// </summary>
     public long? Start { get; }
+
+    /// <summary>
+    /// Gets the time-aligned cues for the song's lyrics.
+    /// </summary>
+    public IReadOnlyList<LyricLineCue>? Cues { get; }
 }

MediaBrowser.Model/Lyrics/LyricLineCue.cs

@@ -0,0 +1,35 @@
+namespace MediaBrowser.Model.Lyrics;
+
+/// <summary>
+/// LyricLineCue model, holds information about the timing of words within a LyricLine.
+/// </summary>
+public class LyricLineCue
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="LyricLineCue"/> class.
+    /// </summary>
+    /// <param name="position">The start of the character index of the lyric.</param>
+    /// <param name="start">The start of the timestamp the lyric is synced to in ticks.</param>
+    /// <param name="end">The end of the timestamp the lyric is synced to in ticks.</param>
+    public LyricLineCue(int position, long start, long? end)
+    {
+        Position = position;
+        Start = start;
+        End = end;
+    }
+
+    /// <summary>
+    /// Gets the character index of the lyric.
+    /// </summary>
+    public int Position { get; }
+
+    /// <summary>
+    /// Gets the timestamp the lyric is synced to in ticks.
+    /// </summary>
+    public long Start { get; }
+
+    /// <summary>
+    /// Gets the end timestamp the lyric is synced to in ticks.
+    /// </summary>
+    public long? End { get; }
+}

MediaBrowser.Providers/Lyric/LrcLyricParser.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text.RegularExpressions;
 using Jellyfin.Extensions;
 using LrcParser.Model;
 using LrcParser.Parser;
@@ -14,7 +15,7 @@ namespace MediaBrowser.Providers.Lyric;
 /// <summary>
 /// LRC Lyric Parser.
 /// </summary>
-public class LrcLyricParser : ILyricParser
+public partial class LrcLyricParser : ILyricParser
 {
     private readonly LyricParser _lrcLyricParser;
 
@@ -65,13 +66,47 @@ public class LrcLyricParser : ILyricParser
         }
 
         List<LyricLine> lyricList = [];
-
-        for (int i = 0; i < sortedLyricData.Count; i++)
+        for (var l = 0; l < sortedLyricData.Count; l++)
         {
-            long ticks = TimeSpan.FromMilliseconds(sortedLyricData[i].StartTime).Ticks;
-            lyricList.Add(new LyricLine(sortedLyricData[i].Text.Trim(), ticks));
+            var cues = new List<LyricLineCue>();
+            var lyric = sortedLyricData[l];
+
+            if (lyric.TimeTags.Count != 0)
+            {
+                var keys = lyric.TimeTags.Keys.ToList();
+                int current = 0, next = 1;
+                while (next < keys.Count)
+                {
+                    var currentKey = keys[current];
+                    var currentMs = lyric.TimeTags[currentKey] ?? 0;
+                    var nextMs = lyric.TimeTags[keys[next]] ?? 0;
+
+                    cues.Add(new LyricLineCue(
+                        position: Math.Max(currentKey.Index, 0),
+                        start: TimeSpan.FromMilliseconds(currentMs).Ticks,
+                        end: TimeSpan.FromMilliseconds(nextMs).Ticks));
+
+                    current++;
+                    next++;
+                }
+
+                var lastKey = keys[current];
+                var lastMs = lyric.TimeTags[lastKey] ?? 0;
+
+                cues.Add(new LyricLineCue(
+                    position: Math.Max(lastKey.Index, 0),
+                    start: TimeSpan.FromMilliseconds(lastMs).Ticks,
+                    end: l + 1 < sortedLyricData.Count ? TimeSpan.FromMilliseconds(sortedLyricData[l + 1].StartTime).Ticks : null));
+            }
+
+            long lyricStartTicks = TimeSpan.FromMilliseconds(lyric.StartTime).Ticks;
+            lyricList.Add(new LyricLine(WhitespaceRegex().Replace(lyric.Text.Trim(), " "), lyricStartTicks, cues));
         }
 
         return new LyricDto { Lyrics = lyricList };
     }
+
+    // Replacement is required until https://github.com/karaoke-dev/LrcParser/issues/83 is resolved.
+    [GeneratedRegex(@"\s+")]
+    private static partial Regex WhitespaceRegex();
 }

tests/Jellyfin.Providers.Tests/Lyrics/LrcLyricParserTests.cs

@@ -0,0 +1,41 @@
+using System.IO;
+using MediaBrowser.Model.Lyrics;
+using MediaBrowser.Providers.Lyric;
+using Xunit;
+
+namespace Jellyfin.Providers.Tests.Lyrics;
+
+public static class LrcLyricParserTests
+{
+    [Fact]
+    public static void ParseElrcCues()
+    {
+        var parser = new LrcLyricParser();
+        var fileContents = File.ReadAllText(Path.Combine("Test Data", "Lyrics", "Fleetwood Mac - Rumors.elrc"));
+        var parsed = parser.ParseLyrics(new LyricFile("Fleetwood Mac - Rumors.elrc", fileContents));
+
+        Assert.NotNull(parsed);
+        Assert.Equal(31, parsed.Lyrics.Count);
+
+        var line1 = parsed.Lyrics[0];
+        Assert.Equal("Every night that goes between", line1.Text);
+        Assert.NotNull(line1.Cues);
+        Assert.Equal(9, line1.Cues.Count);
+        Assert.Equal(68400000, line1.Cues[0].Start);
+        Assert.Equal(72000000, line1.Cues[0].End);
+
+        var line5 = parsed.Lyrics[4];
+        Assert.Equal("Every night you do not come", line5.Text);
+        Assert.NotNull(line5.Cues);
+        Assert.Equal(11, line5.Cues.Count);
+        Assert.Equal(377300000, line5.Cues[5].Start);
+        Assert.Equal(380000000, line5.Cues[5].End);
+
+        var lastLine = parsed.Lyrics[^1];
+        Assert.Equal("I have always been a storm", lastLine.Text);
+        Assert.NotNull(lastLine.Cues);
+        Assert.Equal(11, lastLine.Cues.Count);
+        Assert.Equal(2358000000, lastLine.Cues[^1].Start);
+        Assert.Null(lastLine.Cues[^1].End);
+    }
+}

tests/Jellyfin.Providers.Tests/Test Data/Lyrics/Fleetwood Mac - Rumors.elrc

@@ -0,0 +1,31 @@
+[00:06.84] <00:06.84> Every <00:07.20>   <00:07.56> night <00:07.87>   <00:08.19> that <00:08.46>   <00:08.79> goes <00:09.19>   <00:09.59> between
+[00:14.69] <00:14.69> I <00:14.78>   <00:14.87> feel <00:15.15>   <00:15.44> a <00:15.54>   <00:15.65> little <00:15.96>   <00:16.28> less
+[00:20.98] <00:20.98> As <00:21.10>   <00:21.22> you <00:21.27>   <00:21.31> slowly <00:21.79>   <00:22.27> go <00:22.57>   <00:22.87> away <00:23.74>   <00:24.19> from <00:24.51>   <00:24.82> me
+[00:28.73] <00:28.73> This <00:28.91>   <00:29.09> is <00:29.22>   <00:29.36> only <00:29.75>   <00:30.14> another <00:30.92>   <00:31.25> test
+[00:36.11] <00:36.11> Every <00:36.44>   <00:36.77> night <00:37.14>   <00:37.52> you <00:37.73>   <00:38.00> do <00:38.24>   <00:38.48> not <00:38.93>   <00:39.41> come
+[00:43.56] <00:43.56> Your <00:43.65>   <00:43.74> softness <00:44.21>   <00:44.69> fades <00:45.01>   <00:45.33> away
+[00:50.29] <00:50.29> Did <00:50.42>   <00:50.56> I <00:50.70>   <00:50.85> ever <00:51.41>   <00:51.97> really <00:52.48>   <00:52.99> care <00:53.56>   <00:53.86> that <00:54.09>   <00:54.34> much
+[00:58.07] <00:58.07> Is <00:58.20>   <00:58.34> there <00:58.48>   <00:58.63> anything <00:59.44>   <00:59.69> left <00:59.94>   <01:00.20> to <01:00.30>   <01:00.41> say?
+[01:05.59] <01:05.59> Every <01:06.58>   <01:07.78> hour <01:08.05>   <01:08.32> of <01:08.92>   <01:09.97> fear <01:10.39>   <01:10.81> I <01:11.41>   <01:11.47> spend
+[01:13.84] <01:13.84> My <01:13.99>   <01:14.14> body <01:14.57>   <01:15.01> tries <01:15.32>   <01:15.64> to <01:15.71>   <01:15.79> cry
+[01:18.60] <01:18.60> Living <01:19.41>   <01:20.79> through <01:21.12>   <01:21.45> each <01:21.90>   <01:23.13> empty <01:23.83>   <01:24.54> night
+[01:27.51] <01:27.51> A <01:27.60>   <01:27.69> deadly <01:28.12>   <01:28.56> call <01:28.90>   <01:29.25> inside
+[01:34.29] <01:34.29> I <01:34.39>   <01:34.50> haven't <01:35.37>   <01:35.64> felt <01:35.92>   <01:36.21> this <01:36.34>   <01:36.48> way <01:36.79>   <01:37.11> I <01:37.35>   <01:37.62> feel
+[01:42.11] <01:42.11> Since <01:42.27>   <01:42.44> many <01:42.71>   <01:42.98> a <01:43.08>   <01:43.19> years <01:43.46>   <01:43.85> ago
+[01:48.92] <01:48.92> But <01:49.08>   <01:49.25> in <01:49.36>   <01:49.48> those <01:49.78>   <01:50.09> years <01:50.50>   <01:50.92> and <01:51.07>   <01:51.23> the <01:51.30>   <01:51.38> lifetime's <01:51.99>   <01:52.61> past
+[01:56.60] <01:56.60> I <01:56.69>   <01:56.78> did <01:56.91>   <01:57.05> not <01:57.50>   <01:57.92> deal <01:58.22>   <01:58.52> with <01:58.68>   <01:58.85> the <01:58.91>   <01:58.97> road
+[02:03.13] <02:03.13> And <02:03.25>   <02:03.37> I <02:03.45>   <02:03.55> did <02:03.68>   <02:03.81> not <02:04.20>   <02:04.60> deal <02:04.94>   <02:05.29> with <02:05.45>   <02:05.62> you, <02:05.90>   <02:06.19> I <02:06.50>   <02:06.82> know
+[02:10.95] <02:10.95> Though <02:11.11>   <02:11.28> the <02:11.35>   <02:11.43> love <02:11.79>   <02:12.15> has <02:12.32>   <02:12.39> always <02:13.06>   <02:13.74> been
+[02:17.70] <02:17.70> So <02:17.91>   <02:18.12> I <02:18.16>   <02:18.21> search <02:18.55>   <02:18.90> to <02:19.03>   <02:19.17> find <02:19.51>   <02:19.86> an <02:20.14>   <02:20.43> answer <02:21.24>   <02:21.71> there
+[02:25.72] <02:25.72> So <02:25.96>   <02:26.20> I <02:26.39>   <02:26.59> can <02:27.04>   <02:27.19> truly <02:27.65>   <02:28.12> win
+[02:33.02] <02:33.02> Every <02:34.04>   <02:35.29> hour <02:35.77>   <02:35.84> of <02:36.38>   <02:37.43> fear <02:37.92>   <02:38.42> I <02:38.69>   <02:38.96> spend
+[02:41.35] <02:41.35> My <02:41.63>   <02:41.91> body <02:42.35>   <02:42.79> tries <02:43.09>   <02:43.39> to <02:43.48>   <02:43.57> cry
+[02:46.32] <02:46.32> Living <02:47.04>   <02:48.54> through <02:48.88>   <02:49.23> each <02:49.56>   <02:50.85> empty <02:51.55>   <02:52.26> night
+[02:55.06] <02:55.06> A <02:55.13>   <02:55.21> deadly <02:55.64>   <02:56.08> call <02:56.42>   <02:56.77> inside
+[03:01.50] <03:01.50> So <03:01.74>   <03:01.98> I <03:02.04>   <03:02.10> try <03:02.42>   <03:02.76> to <03:02.81>   <03:02.87> say <03:03.45>   <03:04.20> goodbye, <03:04.53>   <03:04.86> my <03:05.02>   <03:05.19> friend
+[03:09.09] <03:09.09> I'd <03:09.19>   <03:09.30> like <03:09.42>   <03:09.54> to <03:09.70>   <03:09.87> leave <03:10.02>   <03:10.17> you <03:10.29>   <03:10.41> with <03:10.71>   <03:10.89> something <03:11.32>   <03:11.76> warm
+[03:16.25] <03:16.25> But <03:16.34>   <03:16.43> never <03:16.77>   <03:17.12> have <03:17.25>   <03:17.38> I <03:17.44>   <03:17.51> been <03:17.73>   <03:17.96> a <03:18.14>   <03:18.32> blue <03:18.62>   <03:18.92> calm <03:19.58>   <03:19.76> sea
+[03:24.18] <03:24.18> I <03:24.31>   <03:24.45> have <03:24.57>   <03:24.69> always <03:25.59>   <03:28.02> been <03:28.19>   <03:28.38> a <03:28.51>   <03:28.65> storm
+[03:33.69] <03:33.69> Always <03:35.43>   <03:36.99> been <03:37.18>   <03:37.38> a <03:37.50>   <03:37.62> storm
+[03:41.45] <03:41.45> Ooh, <03:41.99>   <03:42.53> always <03:44.42>   <03:45.86> been <03:46.05>   <03:46.25> a <03:46.41>   <03:46.58> storm
+[03:50.34] <03:50.34> I <03:50.49>   <03:50.64> have <03:51.07>   <03:51.51> always <03:53.31>   <03:54.78> been <03:54.96>   <03:55.14> a <03:55.47>   <03:55.80> storm