Trickplay generation, manager, storage

Emby.Server.Implementations/ApplicationHost.cs

@@ -78,6 +78,7 @@ using MediaBrowser.Controller.Session;
 using MediaBrowser.Controller.Sorting;
 using MediaBrowser.Controller.Subtitles;
 using MediaBrowser.Controller.SyncPlay;
+using MediaBrowser.Controller.Trickplay;
 using MediaBrowser.Controller.TV;
 using MediaBrowser.LocalMetadata.Savers;
 using MediaBrowser.MediaEncoding.BdInfo;
@@ -96,6 +97,7 @@ using MediaBrowser.Providers.Lyric;
 using MediaBrowser.Providers.Manager;
 using MediaBrowser.Providers.Plugins.Tmdb;
 using MediaBrowser.Providers.Subtitles;
+using MediaBrowser.Providers.Trickplay;
 using MediaBrowser.XbmcMetadata.Providers;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
@@ -591,6 +593,7 @@ namespace Emby.Server.Implementations
             serviceCollection.AddSingleton<IDeviceDiscovery, DeviceDiscovery>();
 
             serviceCollection.AddSingleton<IChapterManager, ChapterManager>();
+            serviceCollection.AddSingleton<ITrickplayManager, TrickplayManager>();
 
             serviceCollection.AddSingleton<IEncodingManager, MediaEncoder.EncodingManager>();
 

Emby.Server.Implementations/Data/SqliteItemRepository.cs

@@ -48,6 +48,7 @@ namespace Emby.Server.Implementations.Data
     {
         private const string FromText = " from TypedBaseItems A";
         private const string ChaptersTableName = "Chapters2";
+        private const string TrickplayTableName = "Trickplay";
 
         private const string SaveItemCommandText =
             @"replace into TypedBaseItems
@@ -383,6 +384,8 @@ namespace Emby.Server.Implementations.Data
 
                 "create table if not exists " + ChaptersTableName + " (ItemId GUID, ChapterIndex INT NOT NULL, StartPositionTicks BIGINT NOT NULL, Name TEXT, ImagePath TEXT, PRIMARY KEY (ItemId, ChapterIndex))",
 
+                "create table if not exists " + TrickplayTableName + " (ItemId GUID, Width INT NOT NULL, Height INT NOT NULL, TileWidth INT NOT NULL, TileHeight INT NOT NULL, TileCount INT NOT NULL, Interval INT NOT NULL, Bandwidth INT NOT NULL, PRIMARY KEY (ItemId, Width))",
+
                 CreateMediaStreamsTableCommand,
                 CreateMediaAttachmentsTableCommand,
 
@@ -2135,6 +2138,126 @@ namespace Emby.Server.Implementations.Data
             }
         }
 
+        /// <inheritdoc />
+        public Dictionary<int, TrickplayTilesInfo> GetTilesResolutions(Guid itemId)
+        {
+            CheckDisposed();
+
+            var tilesResolutions = new Dictionary<int, TrickplayTilesInfo>();
+            using (var connection = GetConnection(true))
+            {
+                using (var statement = PrepareStatement(connection, "select Width,Height,TileWidth,TileHeight,TileCount,Interval,Bandwidth from " + TrickplayTableName + " where ItemId = @ItemId order by Width asc"))
+                {
+                    statement.TryBind("@ItemId", itemId);
+
+                    foreach (var row in statement.ExecuteQuery())
+                    {
+                        TrickplayTilesInfo tilesInfo = GetTrickplayTilesInfo(row);
+                        tilesResolutions[tilesInfo.Width] = tilesInfo;
+                    }
+                }
+            }
+
+            return tilesResolutions;
+        }
+
+        /// <inheritdoc />
+        public void SaveTilesInfo(Guid itemId, TrickplayTilesInfo tilesInfo)
+        {
+            CheckDisposed();
+
+            ArgumentNullException.ThrowIfNull(tilesInfo);
+
+            var idBlob = itemId.ToByteArray();
+            using (var connection = GetConnection(false))
+            {
+                connection.RunInTransaction(
+                    db =>
+                    {
+                        // Delete old tiles info
+                        db.Execute("delete from " + TrickplayTableName + " where ItemId=@ItemId and Width=@Width", idBlob, tilesInfo.Width);
+                        db.Execute(
+                            "insert into " + TrickplayTableName + " values (@ItemId, @Width, @Height, @TileWidth, @TileHeight, @TileCount, @Interval, @Bandwidth)",
+                            idBlob,
+                            tilesInfo.Width,
+                            tilesInfo.Height,
+                            tilesInfo.TileWidth,
+                            tilesInfo.TileHeight,
+                            tilesInfo.TileCount,
+                            tilesInfo.Interval,
+                            tilesInfo.Bandwidth);
+                    },
+                    TransactionMode);
+            }
+        }
+
+        /// <inheritdoc />
+        public Dictionary<Guid, Dictionary<int, TrickplayTilesInfo>> GetTrickplayManifest(BaseItem item)
+        {
+            CheckDisposed();
+
+            var trickplayManifest = new Dictionary<Guid, Dictionary<int, TrickplayTilesInfo>>();
+            foreach (var mediaSource in item.GetMediaSources(false))
+            {
+                var mediaSourceId = Guid.Parse(mediaSource.Id);
+                var tilesResolutions = GetTilesResolutions(mediaSourceId);
+
+                if (tilesResolutions.Count > 0)
+                {
+                    trickplayManifest[mediaSourceId] = tilesResolutions;
+                }
+            }
+
+            return trickplayManifest;
+        }
+
+        /// <summary>
+        /// Gets the trickplay tiles info.
+        /// </summary>
+        /// <param name="reader">The reader.</param>
+        /// <returns>TrickplayTilesInfo.</returns>
+        private TrickplayTilesInfo GetTrickplayTilesInfo(IReadOnlyList<ResultSetValue> reader)
+        {
+            var tilesInfo = new TrickplayTilesInfo();
+
+            if (reader.TryGetInt32(0, out var width))
+            {
+                tilesInfo.Width = width;
+            }
+
+            if (reader.TryGetInt32(1, out var height))
+            {
+                tilesInfo.Height = height;
+            }
+
+            if (reader.TryGetInt32(2, out var tileWidth))
+            {
+                tilesInfo.TileWidth = tileWidth;
+            }
+
+            if (reader.TryGetInt32(3, out var tileHeight))
+            {
+                tilesInfo.TileHeight = tileHeight;
+            }
+
+            if (reader.TryGetInt32(4, out var tileCount))
+            {
+                tilesInfo.TileCount = tileCount;
+            }
+
+            if (reader.TryGetInt32(5, out var interval))
+            {
+                tilesInfo.Interval = interval;
+            }
+
+            if (reader.TryGetInt32(6, out var bandwidth))
+            {
+                tilesInfo.Bandwidth = bandwidth;
+            }
+
+            return tilesInfo;
+        }
+
         private static bool EnableJoinUserData(InternalItemsQuery query)
         {
             if (query.User is null)

Emby.Server.Implementations/Dto/DtoService.cs

@@ -1058,6 +1058,11 @@ namespace Emby.Server.Implementations.Dto
                     dto.Chapters = _itemRepo.GetChapters(item);
                 }
 
+                if (options.ContainsField(ItemFields.Trickplay))
+                {
+                    dto.Trickplay = _itemRepo.GetTrickplayManifest(item);
+                }
+
                 if (video.ExtraType.HasValue)
                 {
                     dto.ExtraType = video.ExtraType.Value.ToString();

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -149,6 +149,36 @@ namespace MediaBrowser.Controller.MediaEncoding
             return defaultEncoder;
         }
 
+        private string GetMjpegEncoder(EncodingJobInfo state, EncodingOptions encodingOptions)
+        {
+            var defaultEncoder = "mjpeg";
+
+            if (state.VideoType == VideoType.VideoFile)
+            {
+                var hwType = encodingOptions.HardwareAccelerationType;
+
+                var codecMap = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
+                {
+                    { "vaapi",                  defaultEncoder + "_vaapi" },
+                    { "qsv",                defaultEncoder + "_qsv" }
+                };
+
+                if (!string.IsNullOrEmpty(hwType)
+                    && encodingOptions.EnableHardwareEncoding
+                    && codecMap.ContainsKey(hwType))
+                {
+                    var preferredEncoder = codecMap[hwType];
+
+                    if (_mediaEncoder.SupportsEncoder(preferredEncoder))
+                    {
+                        return preferredEncoder;
+                    }
+                }
+            }
+
+            return defaultEncoder;
+        }
+
         private bool IsVaapiSupported(EncodingJobInfo state)
         {
             // vaapi will throw an error with this input
@@ -277,6 +307,11 @@ namespace MediaBrowser.Controller.MediaEncoding
                     return GetH264Encoder(state, encodingOptions);
                 }
 
+                if (string.Equals(codec, "mjpeg", StringComparison.OrdinalIgnoreCase))
+                {
+                    return GetMjpegEncoder(state, encodingOptions);
+                }
+
                 if (string.Equals(codec, "vp8", StringComparison.OrdinalIgnoreCase)
                     || string.Equals(codec, "vpx", StringComparison.OrdinalIgnoreCase))
                 {

MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs

@@ -6,6 +6,7 @@ using System;
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Model.Configuration;
 using MediaBrowser.Model.Dlna;
 using MediaBrowser.Model.Drawing;
 using MediaBrowser.Model.Dto;
@@ -137,6 +138,32 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// <returns>Location of video image.</returns>
         Task<string> ExtractVideoImage(string inputFile, string container, MediaSourceInfo mediaSource, MediaStream imageStream, int? imageStreamIndex, ImageFormat? targetFormat, CancellationToken cancellationToken);
 
+        /// <summary>
+        /// Extracts the video images on interval.
+        /// </summary>
+        /// <param name="inputFile">Input file.</param>
+        /// <param name="container">Video container type.</param>
+        /// <param name="mediaSource">Media source information.</param>
+        /// <param name="imageStream">Media stream information.</param>
+        /// <param name="interval">The interval.</param>
+        /// <param name="maxWidth">The maximum width.</param>
+        /// <param name="allowHwAccel">Allow for hardware acceleration.</param>
+        /// <param name="allowHwEncode">Allow for hardware encoding. allowHwAccel must also be true.</param>
+        /// <param name="encodingHelper">EncodingHelper instance.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Directory where images where extracted. A given image made before another will always be named with a lower number.</returns>
+        Task<string> ExtractVideoImagesOnIntervalAccelerated(
+            string inputFile,
+            string container,
+            MediaSourceInfo mediaSource,
+            MediaStream imageStream,
+            TimeSpan interval,
+            int maxWidth,
+            bool allowHwAccel,
+            bool allowHwEncode,
+            EncodingHelper encodingHelper,
+            CancellationToken cancellationToken);
+
         /// <summary>
         /// Gets the media info.
         /// </summary>

MediaBrowser.Controller/Persistence/IItemRepository.cs

@@ -61,6 +61,27 @@ namespace MediaBrowser.Controller.Persistence
         /// <param name="chapters">The list of chapters to save.</param>
         void SaveChapters(Guid id, IReadOnlyList<ChapterInfo> chapters);
 
+        /// <summary>
+        /// Get available trickplay resolutions and corresponding info.
+        /// </summary>
+        /// <param name="itemId">The item.</param>
+        /// <returns>Map of width resolutions to trickplay tiles info.</returns>
+        Dictionary<int, TrickplayTilesInfo> GetTilesResolutions(Guid itemId);
+
+        /// <summary>
+        /// Saves trickplay tiles info.
+        /// </summary>
+        /// <param name="itemId">The item.</param>
+        /// <param name="tilesInfo">The trickplay tiles info.</param>
+        void SaveTilesInfo(Guid itemId, TrickplayTilesInfo tilesInfo);
+
+        /// <summary>
+        /// Gets trickplay data for an item.
+        /// </summary>
+        /// <param name="item">The item.</param>
+        /// <returns>A map of media source id to a map of tile width to tile info.</returns>
+        Dictionary<Guid, Dictionary<int, TrickplayTilesInfo>> GetTrickplayManifest(BaseItem item);
+
         /// <summary>
         /// Gets the media streams.
         /// </summary>

MediaBrowser.Controller/Trickplay/ITrickplayManager.cs

@@ -0,0 +1,54 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Model.Entities;
+
+namespace MediaBrowser.Controller.Trickplay
+{
+    /// <summary>
+    /// Interface ITrickplayManager.
+    /// </summary>
+    public interface ITrickplayManager
+    {
+        /// <summary>
+        /// Generate or replace trickplay data.
+        /// </summary>
+        /// <param name="video">The video.</param>
+        /// <param name="replace">Whether or not existing data should be replaced.</param>
+        /// <param name="cancellationToken">CancellationToken to use for operation.</param>
+        /// <returns>Task.</returns>
+        Task RefreshTrickplayData(Video video, bool replace, CancellationToken cancellationToken);
+
+        /// <summary>
+        /// Get available trickplay resolutions and corresponding info.
+        /// </summary>
+        /// <param name="itemId">The item.</param>
+        /// <returns>Map of width resolutions to trickplay tiles info.</returns>
+        Dictionary<int, TrickplayTilesInfo> GetTilesResolutions(Guid itemId);
+
+        /// <summary>
+        /// Saves trickplay tiles info.
+        /// </summary>
+        /// <param name="itemId">The item.</param>
+        /// <param name="tilesInfo">The trickplay tiles info.</param>
+        void SaveTilesInfo(Guid itemId, TrickplayTilesInfo tilesInfo);
+
+        /// <summary>
+        /// Gets the trickplay manifest.
+        /// </summary>
+        /// <param name="item">The item.</param>
+        /// <returns>A map of media source id to a map of tile width to tile info.</returns>
+        Dictionary<Guid, Dictionary<int, TrickplayTilesInfo>> GetTrickplayManifest(BaseItem item);
+
+        /// <summary>
+        /// Gets the path to a trickplay tiles image.
+        /// </summary>
+        /// <param name="item">The item.</param>
+        /// <param name="width">The width of a single tile.</param>
+        /// <param name="index">The tile grid's index.</param>
+        /// <returns>The absolute path.</returns>
+        string GetTrickplayTilePath(BaseItem item, int width, int index);
+    }
+}

MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs

@@ -21,6 +21,7 @@ using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Extensions;
 using MediaBrowser.Controller.MediaEncoding;
 using MediaBrowser.MediaEncoding.Probing;
+using MediaBrowser.Model.Configuration;
 using MediaBrowser.Model.Dlna;
 using MediaBrowser.Model.Drawing;
 using MediaBrowser.Model.Dto;
@@ -28,8 +29,10 @@ using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.Globalization;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.MediaInfo;
+using Microsoft.AspNetCore.Components.Forms;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
+using static Nikse.SubtitleEdit.Core.Common.IfoParser;
 
 namespace MediaBrowser.MediaEncoding.Encoder
 {
@@ -775,6 +778,176 @@ namespace MediaBrowser.MediaEncoding.Encoder
         }
 
         /// <inheritdoc />
+        public Task<string> ExtractVideoImagesOnIntervalAccelerated(
+            string inputFile,
+            string container,
+            MediaSourceInfo mediaSource,
+            MediaStream imageStream,
+            TimeSpan interval,
+            int maxWidth,
+            bool allowHwAccel,
+            bool allowHwEncode,
+            EncodingHelper encodingHelper,
+            CancellationToken cancellationToken)
+        {
+            var options = allowHwAccel ? _configurationManager.GetEncodingOptions() : new EncodingOptions();
+
+            // A new EncodingOptions instance must be used as to not disable HW acceleration for all of Jellyfin.
+            // Additionally, we must set a few fields without defaults to prevent null pointer exceptions.
+            if (!allowHwAccel)
+            {
+                options.EnableHardwareEncoding = false;
+                options.HardwareAccelerationType = string.Empty;
+                options.EnableTonemapping = false;
+            }
+
+            var baseRequest = new BaseEncodingJobOptions { MaxWidth = maxWidth };
+            var jobState = new EncodingJobInfo(TranscodingJobType.Progressive)
+            {
+                IsVideoRequest = true,  // must be true for InputVideoHwaccelArgs to return non-empty value
+                MediaSource = mediaSource,
+                VideoStream = imageStream,
+                BaseRequest = baseRequest,  // GetVideoProcessingFilterParam errors if null
+                MediaPath = inputFile,
+                OutputVideoCodec = "mjpeg"
+            };
+            var vidEncoder = options.AllowMjpegEncoding ? encodingHelper.GetVideoEncoder(jobState, options) : jobState.OutputVideoCodec;
+
+            // Get input and filter arguments
+            var inputArg = encodingHelper.GetInputArgument(jobState, options, container).Trim();
+            if (string.IsNullOrWhiteSpace(inputArg))
+            {
+                throw new InvalidOperationException("EncodingHelper returned empty input arguments.");
+            }
+
+            if (!allowHwAccel)
+            {
+                inputArg = "-threads " + _threads + " " + inputArg; // HW accel args set a different input thread count, only set if disabled
+            }
+
+            var filterParam = encodingHelper.GetVideoProcessingFilterParam(jobState, options, jobState.OutputVideoCodec).Trim();
+            if (string.IsNullOrWhiteSpace(filterParam) || filterParam.IndexOf("\"", StringComparison.Ordinal) == -1)
+            {
+                throw new InvalidOperationException("EncodingHelper returned empty or invalid filter parameters.");
+            }
+
+            return ExtractVideoImagesOnIntervalInternal(inputArg, filterParam, interval, vidEncoder, _threads, cancellationToken);
+        }
+
+        private async Task<string> ExtractVideoImagesOnIntervalInternal(
+            string inputArg,
+            string filterParam,
+            TimeSpan interval,
+            string vidEncoder,
+            int outputThreads,
+            CancellationToken cancellationToken)
+        {
+            if (string.IsNullOrWhiteSpace(inputArg))
+            {
+                throw new InvalidOperationException("Empty or invalid input argument.");
+            }
+
+            // Output arguments
+            string fps = "fps=1/" + interval.TotalSeconds.ToString(CultureInfo.InvariantCulture);
+            if (string.IsNullOrWhiteSpace(filterParam))
+            {
+                filterParam = "-vf \"" + fps + "\"";
+            }
+            else
+            {
+                filterParam = filterParam.Insert(filterParam.IndexOf("\"", StringComparison.Ordinal) + 1, fps + ",");
+            }
+
+            var targetDirectory = Path.Combine(_configurationManager.ApplicationPaths.TempDirectory, Guid.NewGuid().ToString("N"));
+            Directory.CreateDirectory(targetDirectory);
+            var outputPath = Path.Combine(targetDirectory, "%08d.jpg");
+
+            // Final command arguments
+            var args = string.Format(
+                CultureInfo.InvariantCulture,
+                "-loglevel error {0} -an -sn {1} -threads {2} -c:v {3} -f {4} \"{5}\"",
+                inputArg,
+                filterParam,
+                outputThreads,
+                vidEncoder,
+                "image2",
+                outputPath);
+
+            // Start ffmpeg process
+            var process = new Process
+            {
+                StartInfo = new ProcessStartInfo
+                {
+                    CreateNoWindow = true,
+                    UseShellExecute = false,
+                    FileName = _ffmpegPath,
+                    Arguments = args,
+                    WindowStyle = ProcessWindowStyle.Hidden,
+                    ErrorDialog = false,
+                },
+                EnableRaisingEvents = true
+            };
+
+            var processDescription = string.Format(CultureInfo.InvariantCulture, "{0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments);
+            _logger.LogDebug("{ProcessDescription}", processDescription);
+
+            using (var processWrapper = new ProcessWrapper(process, this))
+            {
+                bool ranToCompletion = false;
+
+                await _thumbnailResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
+                try
+                {
+                    StartProcess(processWrapper);
+
+                    // Need to give ffmpeg enough time to make all the thumbnails, which could be a while,
+                    // but we still need to detect if the process hangs.
+                    // Making the assumption that as long as new jpegs are showing up, everything is good.
+
+                    bool isResponsive = true;
+                    int lastCount = 0;
+
+                    while (isResponsive)
+                    {
+                        if (await process.WaitForExitAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false))
+                        {
+                            ranToCompletion = true;
+                            break;
+                        }
+
+                        cancellationToken.ThrowIfCancellationRequested();
+
+                        var jpegCount = _fileSystem.GetFilePaths(targetDirectory)
+                            .Count(i => string.Equals(Path.GetExtension(i), ".jpg", StringComparison.OrdinalIgnoreCase));
+
+                        isResponsive = jpegCount > lastCount;
+                        lastCount = jpegCount;
+                    }
+
+                    if (!ranToCompletion)
+                    {
+                        _logger.LogInformation("Killing ffmpeg extraction process due to inactivity.");
+                        StopProcess(processWrapper, 1000);
+                    }
+                }
+                finally
+                {
+                    _thumbnailResourcePool.Release();
+                }
+
+                var exitCode = ranToCompletion ? processWrapper.ExitCode ?? 0 : -1;
+
+                if (exitCode == -1)
+                {
+                    _logger.LogError("ffmpeg image extraction failed for {ProcessDescription}", processDescription);
+
+                    throw new FfmpegException(string.Format(CultureInfo.InvariantCulture, "ffmpeg image extraction failed for {0}", processDescription));
+                }
+
+                return targetDirectory;
+            }
+        }
+
         public string GetTimeParameter(long ticks)
         {
             var time = TimeSpan.FromTicks(ticks);

MediaBrowser.Model/Configuration/EncodingOptions.cs

@@ -48,7 +48,9 @@ public class EncodingOptions
         EnableIntelLowPowerH264HwEncoder = false;
         EnableIntelLowPowerHevcHwEncoder = false;
         EnableHardwareEncoding = true;
+        EnableTrickplayHwAccel = false;
         AllowHevcEncoding = false;
+        AllowMjpegEncoding = false;
         EnableSubtitleExtraction = true;
         AllowOnDemandMetadataBasedKeyframeExtractionForExtensions = new[] { "mkv" };
         HardwareDecodingCodecs = new string[] { "h264", "vc1" };
@@ -244,11 +246,21 @@ public class EncodingOptions
     /// </summary>
     public bool EnableHardwareEncoding { get; set; }
 
+    /// <summary>
+    /// Gets or sets a value indicating whether hardware acceleration is enabled for trickplay generation.
+    /// </summary>
+    public bool EnableTrickplayHwAccel { get; set; }
+
     /// <summary>
     /// Gets or sets a value indicating whether HEVC encoding is enabled.
     /// </summary>
     public bool AllowHevcEncoding { get; set; }
 
+    /// <summary>
+    /// Gets or sets a value indicating whether MJPEG encoding is enabled.
+    /// </summary>
+    public bool AllowMjpegEncoding { get; set; }
+
     /// <summary>
     /// Gets or sets a value indicating whether subtitle extraction is enabled.
     /// </summary>

MediaBrowser.Model/Dto/BaseItemDto.cs

@@ -568,6 +568,12 @@ namespace MediaBrowser.Model.Dto
         /// <value>The chapters.</value>
         public List<ChapterInfo> Chapters { get; set; }
 
+        /// <summary>
+        /// Gets or sets the trickplay manifest.
+        /// </summary>
+        /// <value>The trickplay manifest.</value>
+        public Dictionary<Guid, Dictionary<int, TrickplayTilesInfo>> Trickplay { get; set; }
+
         /// <summary>
         /// Gets or sets the type of the location.
         /// </summary>

MediaBrowser.Model/Entities/TrickplayTilesInfo.cs

@@ -0,0 +1,50 @@
+namespace MediaBrowser.Model.Entities
+{
+    /// <summary>
+    /// Class TrickplayTilesInfo.
+    /// </summary>
+    public class TrickplayTilesInfo
+    {
+        /// <summary>
+        /// Gets or sets width of an individual tile.
+        /// </summary>
+        /// <value>The width.</value>
+        public int Width { get; set; }
+
+        /// <summary>
+        /// Gets or sets height of an individual tile.
+        /// </summary>
+        /// <value>The height.</value>
+        public int Height { get; set; }
+
+        /// <summary>
+        /// Gets or sets amount of tiles per row.
+        /// </summary>
+        /// <value>The tile grid's width.</value>
+        public int TileWidth { get; set; }
+
+        /// <summary>
+        /// Gets or sets amount of tiles per column.
+        /// </summary>
+        /// <value>The tile grid's height.</value>
+        public int TileHeight { get; set; }
+
+        /// <summary>
+        /// Gets or sets total amount of non-black tiles.
+        /// </summary>
+        /// <value>The tile count.</value>
+        public int TileCount { get; set; }
+
+        /// <summary>
+        /// Gets or sets interval in milliseconds between each trickplay tile.
+        /// </summary>
+        /// <value>The interval.</value>
+        public int Interval { get; set; }
+
+        /// <summary>
+        /// Gets or sets peak bandwith usage in bits per second.
+        /// </summary>
+        /// <value>The bandwidth.</value>
+        public int Bandwidth { get; set; }
+    }
+}

MediaBrowser.Model/Querying/ItemFields.cs

@@ -34,6 +34,11 @@ namespace MediaBrowser.Model.Querying
         /// </summary>
         Chapters,
 
+        /// <summary>
+        /// The trickplay manifest.
+        /// </summary>
+        Trickplay,
+
         ChildCount,
 
         /// <summary>

MediaBrowser.Providers/MediaBrowser.Providers.csproj

@@ -22,6 +22,7 @@
     <PackageReference Include="Microsoft.Extensions.Http" />
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="PlaylistsNET" />
+    <PackageReference Include="SkiaSharp" />
     <PackageReference Include="TagLibSharp" />
     <PackageReference Include="TMDbLib" />
   </ItemGroup>

MediaBrowser.Providers/Trickplay/TrickplayImagesTask.cs

@@ -0,0 +1,116 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.Trickplay;
+using MediaBrowser.Model.Entities;
+using MediaBrowser.Model.Globalization;
+using MediaBrowser.Model.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace MediaBrowser.Providers.Trickplay
+{
+    /// <summary>
+    /// Class TrickplayImagesTask.
+    /// </summary>
+    public class TrickplayImagesTask : IScheduledTask
+    {
+        private readonly ILogger<TrickplayImagesTask> _logger;
+        private readonly ILibraryManager _libraryManager;
+        private readonly ILocalizationManager _localization;
+        private readonly IServerConfigurationManager _configurationManager;
+        private readonly ITrickplayManager _trickplayManager;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="TrickplayImagesTask"/> class.
+        /// </summary>
+        /// <param name="logger">The logger.</param>
+        /// <param name="libraryManager">The library manager.</param>
+        /// <param name="localization">The localization manager.</param>
+        /// <param name="configurationManager">The configuration manager.</param>
+        /// <param name="trickplayManager">The trickplay manager.</param>
+        public TrickplayImagesTask(
+            ILogger<TrickplayImagesTask> logger,
+            ILibraryManager libraryManager,
+            ILocalizationManager localization,
+            IServerConfigurationManager configurationManager,
+            ITrickplayManager trickplayManager)
+        {
+            _libraryManager = libraryManager;
+            _logger = logger;
+            _localization = localization;
+            _configurationManager = configurationManager;
+            _trickplayManager = trickplayManager;
+        }
+
+        /// <inheritdoc />
+        public string Name => _localization.GetLocalizedString("TaskRefreshTrickplayImages");
+
+        /// <inheritdoc />
+        public string Description => _localization.GetLocalizedString("TaskRefreshTrickplayImagesDescription");
+
+        /// <inheritdoc />
+        public string Key => "RefreshTrickplayImages";
+
+        /// <inheritdoc />
+        public string Category => _localization.GetLocalizedString("TasksLibraryCategory");
+
+        /// <inheritdoc />
+        public IEnumerable<TaskTriggerInfo> GetDefaultTriggers()
+        {
+            return new[]
+            {
+                new TaskTriggerInfo
+                {
+                    Type = TaskTriggerInfo.TriggerDaily,
+                    TimeOfDayTicks = TimeSpan.FromHours(3).Ticks,
+                    MaxRuntimeTicks = TimeSpan.FromHours(5).Ticks
+                }
+            };
+        }
+
+        /// <inheritdoc />
+        public async Task ExecuteAsync(IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            // TODO: libraryoptions dont run on libraries with trickplay disabled
+            var items = _libraryManager.GetItemList(new InternalItemsQuery
+            {
+                MediaTypes = new[] { MediaType.Video },
+                IsVirtualItem = false,
+                IsFolder = false,
+                Recursive = false
+            }).OfType<Video>().ToList();
+
+            var numComplete = 0;
+
+            foreach (var item in items)
+            {
+                try
+                {
+                    cancellationToken.ThrowIfCancellationRequested();
+                    await _trickplayManager.RefreshTrickplayData(item, false, cancellationToken).ConfigureAwait(false);
+                }
+                catch (OperationCanceledException)
+                {
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError("Error creating trickplay files for {ItemName}: {Msg}", item.Name, ex);
+                }
+
+                numComplete++;
+                double percent = numComplete;
+                percent /= items.Count;
+                percent *= 100;
+
+                progress.Report(percent);
+            }
+        }
+    }
+}

MediaBrowser.Providers/Trickplay/TrickplayManager.cs

@@ -0,0 +1,363 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.MediaEncoding;
+using MediaBrowser.Controller.Persistence;
+using MediaBrowser.Controller.Trickplay;
+using MediaBrowser.Model.Dto;
+using MediaBrowser.Model.Entities;
+using MediaBrowser.Model.IO;
+using Microsoft.Extensions.Logging;
+using SkiaSharp;
+
+namespace MediaBrowser.Providers.Trickplay
+{
+    /// <summary>
+    /// ITrickplayManager implementation.
+    /// </summary>
+    public class TrickplayManager : ITrickplayManager
+    {
+        private readonly ILogger<TrickplayManager> _logger;
+        private readonly IItemRepository _itemRepo;
+        private readonly IMediaEncoder _mediaEncoder;
+        private readonly IFileSystem _fileSystem;
+        private readonly EncodingHelper _encodingHelper;
+
+        private static readonly SemaphoreSlim _resourcePool = new(1, 1);
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="TrickplayManager"/> class.
+        /// </summary>
+        /// <param name="logger">The logger.</param>
+        /// <param name="itemRepo">The item repository.</param>
+        /// <param name="mediaEncoder">The media encoder.</param>
+        /// <param name="fileSystem">The file systen.</param>
+        /// <param name="encodingHelper">The encoding helper.</param>
+        public TrickplayManager(
+            ILogger<TrickplayManager> logger,
+            IItemRepository itemRepo,
+            IMediaEncoder mediaEncoder,
+            IFileSystem fileSystem,
+            EncodingHelper encodingHelper)
+        {
+            _logger = logger;
+            _itemRepo = itemRepo;
+            _mediaEncoder = mediaEncoder;
+            _fileSystem = fileSystem;
+            _encodingHelper = encodingHelper;
+        }
+
+        /// <inheritdoc />
+        public async Task RefreshTrickplayData(Video video, bool replace, CancellationToken cancellationToken)
+        {
+            _logger.LogDebug("Trickplay refresh for {ItemId} (replace existing: {Replace})", video.Id, replace);
+
+            foreach (var width in new int[] { 320 } /*todo conf*/)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+                await RefreshTrickplayData(video, replace, width, 10000/*todo conf*/, 10/*todo conf*/, 10/*todo conf*/, true/*todo conf*/, true/*todo conf*/, cancellationToken).ConfigureAwait(false);
+            }
+        }
+
+        private async Task RefreshTrickplayData(Video video, bool replace, int width, int interval, int tileWidth, int tileHeight, bool doHwAccel, bool doHwEncode, CancellationToken cancellationToken)
+        {
+            if (!CanGenerateTrickplay(video))
+            {
+                return;
+            }
+
+            var imgTempDir = string.Empty;
+            var outputDir = GetTrickplayDirectory(video, width);
+
+            try
+            {
+                await _resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+                if (!replace && Directory.Exists(outputDir))
+                {
+                    _logger.LogDebug("Found existing trickplay files for {ItemId}. Exiting.", video.Id);
+                    return;
+                }
+
+                // Extract images
+                // Note: Media sources under parent items exist as their own video/item as well. Only use this video stream for trickplay.
+                var mediaSource = video.GetMediaSources(false).Find(source => Guid.Parse(source.Id).Equals(video.Id));
+
+                if (mediaSource is null)
+                {
+                    _logger.LogDebug("Found no matching media source for item {ItemId}", video.Id);
+                    return;
+                }
+
+                var mediaPath = mediaSource.Path;
+                var mediaStream = mediaSource.VideoStream;
+                var container = mediaSource.Container;
+
+                _logger.LogInformation("Creating trickplay files at {Width} width, for {Path} [ID: {ItemId}]", width, mediaPath, video.Id);
+                imgTempDir = await _mediaEncoder.ExtractVideoImagesOnIntervalAccelerated(
+                    mediaPath,
+                    container,
+                    mediaSource,
+                    mediaStream,
+                    TimeSpan.FromMilliseconds(interval),
+                    width,
+                    doHwAccel,
+                    doHwEncode,
+                    _encodingHelper,
+                    cancellationToken).ConfigureAwait(false);
+
+                if (string.IsNullOrEmpty(imgTempDir) || !Directory.Exists(imgTempDir))
+                {
+                    throw new InvalidOperationException("Null or invalid directory from media encoder.");
+                }
+
+                var images = _fileSystem.GetFiles(imgTempDir, new string[] { ".jpg" }, false, false)
+                    .Where(img => string.Equals(img.Extension, ".jpg", StringComparison.Ordinal))
+                    .OrderBy(i => i.FullName)
+                    .ToList();
+
+                // Create tiles
+                var tilesTempDir = Path.Combine(imgTempDir, Guid.NewGuid().ToString("N"));
+                var tilesInfo = CreateTiles(images, width, interval, tileWidth, tileHeight, tilesTempDir, outputDir);
+
+                // Save tiles info
+                try
+                {
+                    if (tilesInfo is not null)
+                    {
+                        SaveTilesInfo(video.Id, tilesInfo);
+                        _logger.LogInformation("Finished creation of trickplay files for {0}", mediaPath);
+                    }
+                    else
+                    {
+                        throw new InvalidOperationException("Null trickplay tiles info from CreateTiles.");
+                    }
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError(ex, "Error while saving trickplay tiles info.");
+
+                    // Make sure no files stay in metadata folders on failure
+                    // if tiles info wasn't saved.
+                    Directory.Delete(outputDir, true);
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error creating trickplay images.");
+            }
+            finally
+            {
+                _resourcePool.Release();
+
+                if (!string.IsNullOrEmpty(imgTempDir))
+                {
+                    Directory.Delete(imgTempDir, true);
+                }
+            }
+        }
+
+        private TrickplayTilesInfo CreateTiles(List<FileSystemMetadata> images, int width, int interval, int tileWidth, int tileHeight, string workDir, string outputDir)
+        {
+            if (images.Count == 0)
+            {
+                throw new InvalidOperationException("Can't create trickplay from 0 images.");
+            }
+
+            Directory.CreateDirectory(workDir);
+
+            var tilesInfo = new TrickplayTilesInfo
+            {
+                Width = width,
+                Interval = interval,
+                TileWidth = tileWidth,
+                TileHeight = tileHeight,
+                TileCount = (int)Math.Ceiling((decimal)images.Count / tileWidth / tileHeight),
+                Bandwidth = 0
+            };
+
+            var firstImg = SKBitmap.Decode(images[0].FullName);
+            if (firstImg == null)
+            {
+                throw new InvalidDataException("Could not decode image data.");
+            }
+
+            tilesInfo.Height = firstImg.Height;
+            if (tilesInfo.Width != firstImg.Width)
+            {
+                throw new InvalidOperationException("Image width does not match config width.");
+            }
+
+            /*
+             * Generate grids of trickplay image tiles
+             */
+            var imgNo = 0;
+            var i = 0;
+            while (i < images.Count)
+            {
+                var tileGrid = new SKBitmap(tilesInfo.Width * tilesInfo.TileWidth, tilesInfo.Height * tilesInfo.TileHeight);
+                var tileCount = 0;
+
+                using (var canvas = new SKCanvas(tileGrid))
+                {
+                    for (var y = 0; y < tilesInfo.TileHeight; y++)
+                    {
+                        for (var x = 0; x < tilesInfo.TileWidth; x++)
+                        {
+                            if (i >= images.Count)
+                            {
+                                break;
+                            }
+
+                            var img = SKBitmap.Decode(images[i].FullName);
+                            if (img == null)
+                            {
+                                throw new InvalidDataException("Could not decode image data.");
+                            }
+
+                            if (tilesInfo.Width != img.Width)
+                            {
+                                throw new InvalidOperationException("Image width does not match config width.");
+                            }
+
+                            if (tilesInfo.Height != img.Height)
+                            {
+                                throw new InvalidOperationException("Image height does not match first image height.");
+                            }
+
+                            canvas.DrawBitmap(img, x * tilesInfo.Width, y * tilesInfo.Height);
+                            tileCount++;
+                            i++;
+                        }
+                    }
+                }
+
+                // Output each tile grid to singular file
+                var tileGridPath = Path.Combine(workDir, $"{imgNo}.jpg");
+                using (var stream = File.OpenWrite(tileGridPath))
+                {
+                    tileGrid.Encode(stream, SKEncodedImageFormat.Jpeg, 100/* todo _config.JpegQuality*/);
+                }
+
+                var bitrate = (int)Math.Ceiling((decimal)new FileInfo(tileGridPath).Length * 8 / tilesInfo.TileWidth / tilesInfo.TileHeight / (tilesInfo.Interval / 1000));
+                tilesInfo.Bandwidth = Math.Max(tilesInfo.Bandwidth, bitrate);
+
+                imgNo++;
+            }
+
+            /*
+             * Move trickplay tiles to output directory
+             */
+            Directory.CreateDirectory(outputDir);
+
+            // Replace existing tile grids if they already exist
+            if (Directory.Exists(outputDir))
+            {
+                Directory.Delete(outputDir, true);
+            }
+
+            MoveDirectory(workDir, outputDir);
+
+            return tilesInfo;
+        }
+
+        private bool CanGenerateTrickplay(Video video)
+        {
+            var videoType = video.VideoType;
+            if (videoType == VideoType.Iso || videoType == VideoType.Dvd || videoType == VideoType.BluRay)
+            {
+                return false;
+            }
+
+            if (video.IsPlaceHolder)
+            {
+                return false;
+            }
+
+            /* TODO config options
+            var libraryOptions = _libraryManager.GetLibraryOptions(video);
+            if (libraryOptions is not null)
+            {
+                if (!libraryOptions.EnableChapterImageExtraction)
+                {
+                    return false;
+                }
+            }
+            else
+            {
+                return false;
+            }
+            */
+
+            // TODO: media length is shorter than configured interval
+
+            if (video.IsShortcut)
+            {
+                return false;
+            }
+
+            if (!video.IsCompleteMedia)
+            {
+                return false;
+            }
+
+            // Can't extract images if there are no video streams
+            return video.GetMediaStreams().Count > 0;
+        }
+
+        /// <inheritdoc />
+        public Dictionary<int, TrickplayTilesInfo> GetTilesResolutions(Guid itemId)
+        {
+            return _itemRepo.GetTilesResolutions(itemId);
+        }
+
+        /// <inheritdoc />
+        public void SaveTilesInfo(Guid itemId, TrickplayTilesInfo tilesInfo)
+        {
+            _itemRepo.SaveTilesInfo(itemId, tilesInfo);
+        }
+
+        /// <inheritdoc />
+        public Dictionary<Guid, Dictionary<int, TrickplayTilesInfo>> GetTrickplayManifest(BaseItem item)
+        {
+            return _itemRepo.GetTrickplayManifest(item);
+        }
+
+        /// <inheritdoc />
+        public string GetTrickplayTilePath(BaseItem item, int width, int index)
+        {
+            return Path.Combine(GetTrickplayDirectory(item, width), index + ".jpg");
+        }
+
+        private string GetTrickplayDirectory(BaseItem item, int? width = null)
+        {
+            var path = Path.Combine(item.GetInternalMetadataPath(), "trickplay");
+
+            return width.HasValue ? Path.Combine(path, width.Value.ToString(CultureInfo.InvariantCulture)) : path;
+        }
+
+        private void MoveDirectory(string source, string destination)
+        {
+            try
+            {
+                Directory.Move(source, destination);
+            }
+            catch (System.IO.IOException)
+            {
+                // Cross device move requires a copy
+                Directory.CreateDirectory(destination);
+                foreach (string file in Directory.GetFiles(source))
+                {
+                    File.Copy(file, Path.Join(destination, Path.GetFileName(file)), true);
+                }
+
+                Directory.Delete(source, true);
+            }
+        }
+    }
+}

MediaBrowser.Providers/Trickplay/TrickplayProvider.cs

@@ -0,0 +1,109 @@
+using System.Threading;
+using System.Threading.Tasks;
+using MediaBrowser.Controller.Configuration;
+using MediaBrowser.Controller.Entities;
+using MediaBrowser.Controller.Entities.Movies;
+using MediaBrowser.Controller.Entities.TV;
+using MediaBrowser.Controller.Library;
+using MediaBrowser.Controller.Providers;
+using MediaBrowser.Controller.Trickplay;
+using Microsoft.Extensions.Logging;
+
+namespace MediaBrowser.Providers.Trickplay
+{
+    /// <summary>
+    /// Class TrickplayProvider. Provides images and metadata for trickplay
+    /// scrubbing previews.
+    /// </summary>
+    public class TrickplayProvider : ICustomMetadataProvider<Episode>,
+        ICustomMetadataProvider<MusicVideo>,
+        ICustomMetadataProvider<Movie>,
+        ICustomMetadataProvider<Trailer>,
+        ICustomMetadataProvider<Video>,
+        IHasItemChangeMonitor,
+        IHasOrder,
+        IForcedProvider
+    {
+        private readonly ILogger<TrickplayProvider> _logger;
+        private readonly IServerConfigurationManager _configurationManager;
+        private readonly ITrickplayManager _trickplayManager;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="TrickplayProvider"/> class.
+        /// </summary>
+        /// <param name="logger">The logger.</param>
+        /// <param name="configurationManager">The configuration manager.</param>
+        /// <param name="trickplayManager">The trickplay manager.</param>
+        public TrickplayProvider(
+            ILogger<TrickplayProvider> logger,
+            IServerConfigurationManager configurationManager,
+            ITrickplayManager trickplayManager)
+        {
+            _logger = logger;
+            _configurationManager = configurationManager;
+            _trickplayManager = trickplayManager;
+        }
+
+        /// <inheritdoc />
+        public string Name => "Trickplay Preview";
+
+        /// <inheritdoc />
+        public int Order => 1000;
+
+        /// <inheritdoc />
+        public bool HasChanged(BaseItem item, IDirectoryService directoryService)
+        {
+            if (item.IsFileProtocol)
+            {
+                var file = directoryService.GetFile(item.Path);
+                if (file is not null && item.DateModified != file.LastWriteTimeUtc)
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
+        /// <inheritdoc />
+        public Task<ItemUpdateType> FetchAsync(Episode item, MetadataRefreshOptions options, CancellationToken cancellationToken)
+        {
+            return FetchInternal(item, options, cancellationToken);
+        }
+
+        /// <inheritdoc />
+        public Task<ItemUpdateType> FetchAsync(MusicVideo item, MetadataRefreshOptions options, CancellationToken cancellationToken)
+        {
+            return FetchInternal(item, options, cancellationToken);
+        }
+
+        /// <inheritdoc />
+        public Task<ItemUpdateType> FetchAsync(Movie item, MetadataRefreshOptions options, CancellationToken cancellationToken)
+        {
+            return FetchInternal(item, options, cancellationToken);
+        }
+
+        /// <inheritdoc />
+        public Task<ItemUpdateType> FetchAsync(Trailer item, MetadataRefreshOptions options, CancellationToken cancellationToken)
+        {
+            return FetchInternal(item, options, cancellationToken);
+        }
+
+        /// <inheritdoc />
+        public Task<ItemUpdateType> FetchAsync(Video item, MetadataRefreshOptions options, CancellationToken cancellationToken)
+        {
+            return FetchInternal(item, options, cancellationToken);
+        }
+
+        private async Task<ItemUpdateType> FetchInternal(Video item, MetadataRefreshOptions options, CancellationToken cancellationToken)
+        {
+            // TODO: will "search for missing metadata" always trigger this?
+            // TODO: implement all config options -->
+            // TODO: this is always blocking for metadata collection, make non-blocking option
+            await _trickplayManager.RefreshTrickplayData(item, options.ReplaceAllImages, cancellationToken).ConfigureAwait(false);
+
+            // The core doesn't need to trigger any save operations over this
+            return ItemUpdateType.None;
+        }
+    }
+}