EncodingHelper hwaccel pipelines refactor

separate the HW pipeline according to HWA method for maintainability.

Jellyfin.Api/Controllers/DynamicHlsController.cs

@@ -1567,24 +1567,18 @@ namespace Jellyfin.Api.Controllers
 
                 // args += " -mixed-refs 0 -refs 3 -x264opts b_pyramid=0:weightb=0:weightp=0";
 
-                var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
-
-                if (hasGraphicalSubs)
-                {
-                    // Graphical subs overlay and resolution params.
-                    args += _encodingHelper.GetGraphicalSubtitleParam(state, _encodingOptions, codec);
-                }
-                else
-                {
-                    // Resolution params.
-                    args += _encodingHelper.GetOutputSizeParam(state, _encodingOptions, codec);
-                }
+                // video processing filters.
+                args += _encodingHelper.GetVideoProcessingFilterParam(state, _encodingOptions, codec);
 
                 // -start_at_zero is necessary to use with -ss when seeking,
                 // otherwise the target position cannot be determined.
-                if (!(state.SubtitleStream != null && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream))
+                if (state.SubtitleStream != null)
                 {
-                    args += " -start_at_zero";
+                    // Disable start_at_zero for external graphical subs
+                    if (!(state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream))
+                    {
+                        args += " -start_at_zero";
+                    }
                 }
 
                 // args += " -flags -global_header";

Jellyfin.Api/Controllers/VideoHlsController.cs

@@ -552,22 +552,18 @@ namespace Jellyfin.Api.Controllers
                     args += " -bf 0";
                 }
 
-                var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
+                // video processing filters.
+                args += _encodingHelper.GetVideoProcessingFilterParam(state, _encodingOptions, codec);
 
-                if (hasGraphicalSubs)
+                // -start_at_zero is necessary to use with -ss when seeking,
+                // otherwise the target position cannot be determined.
+                if (state.SubtitleStream != null)
                 {
-                    // Graphical subs overlay and resolution params.
-                    args += _encodingHelper.GetGraphicalSubtitleParam(state, _encodingOptions, codec);
-                }
-                else
-                {
-                    // Resolution params.
-                    args += _encodingHelper.GetOutputSizeParam(state, _encodingOptions, codec);
-                }
-
-                if (state.SubtitleStream == null || !state.SubtitleStream.IsExternal || state.SubtitleStream.IsTextSubtitleStream)
-                {
-                    args += " -start_at_zero";
+                    // Disable start_at_zero for external graphical subs
+                    if (!(state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream))
+                    {
+                        args += " -start_at_zero";
+                    }
                 }
             }
 

MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs

@@ -18,6 +18,16 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// <summary>
         /// The tonemap_opencl_bt2390.
         /// </summary>
-        TonemapOpenclBt2390 = 2
+        TonemapOpenclBt2390 = 2,
+
+        /// <summary>
+        /// The overlay_opencl_framesync.
+        /// </summary>
+        OverlayOpenclFrameSync = 3,
+
+        /// <summary>
+        /// The overlay_vaapi_framesync.
+        /// </summary>
+        OverlayVaapiFrameSync = 4
     }
 }

MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs

@@ -60,6 +60,24 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// <returns><c>true</c> if the filter is supported, <c>false</c> otherwise.</returns>
         bool SupportsFilterWithOption(FilterOptionType option);
 
+        /// <summary>
+        /// Whether the configured Vaapi device is from AMD(radeonsi/r600 Mesa driver).
+        /// </summary>
+        /// <returns><c>true</c> if the Vaapi device is an AMD(radeonsi/r600 Mesa driver) GPU, <c>false</c> otherwise.</returns>
+        bool IsVaapiDeviceAmd();
+
+        /// <summary>
+        /// Whether the configured Vaapi device is from Intel(iHD driver).
+        /// </summary>
+        /// <returns><c>true</c> if the Vaapi device is an Intel(iHD driver) GPU, <c>false</c> otherwise.</returns>
+        bool IsVaapiDeviceInteliHD();
+
+        /// <summary>
+        /// Whether the configured Vaapi device is from Intel(legacy i965 driver).
+        /// </summary>
+        /// <returns><c>true</c> if the Vaapi device is an Intel(legacy i965 driver) GPU, <c>false</c> otherwise.</returns>
+        bool IsVaapiDeviceInteli965();
+
         /// <summary>
         /// Get the version of media encoder.
         /// </summary>

MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs

@@ -16,6 +16,12 @@ namespace MediaBrowser.MediaEncoding.Encoder
         {
             "h264",
             "hevc",
+            "vp8",
+            "libvpx",
+            "vp9",
+            "libvpx-vp9",
+            "av1",
+            "libdav1d",
             "mpeg2video",
             "mpeg4",
             "msmpeg4",
@@ -30,6 +36,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             "vc1_qsv",
             "vp8_qsv",
             "vp9_qsv",
+            "av1_qsv",
             "h264_cuvid",
             "hevc_cuvid",
             "mpeg2_cuvid",
@@ -37,16 +44,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
             "mpeg4_cuvid",
             "vp8_cuvid",
             "vp9_cuvid",
+            "av1_cuvid",
             "h264_mmal",
             "mpeg2_mmal",
             "mpeg4_mmal",
             "vc1_mmal",
-            "h264_mediacodec",
-            "hevc_mediacodec",
-            "mpeg2_mediacodec",
-            "mpeg4_mediacodec",
-            "vp8_mediacodec",
-            "vp9_mediacodec",
             "h264_opencl",
             "hevc_opencl",
             "mpeg2_opencl",
@@ -89,20 +91,39 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
         private static readonly string[] _requiredFilters = new[]
         {
+            // sw
+            "alphasrc",
+            "zscale",
+            // qsv
+            "scale_qsv",
+            "vpp_qsv",
+            "deinterlace_qsv",
+            "overlay_qsv",
+            // cuda
             "scale_cuda",
             "yadif_cuda",
-            "hwupload_cuda",
-            "overlay_cuda",
             "tonemap_cuda",
+            "overlay_cuda",
+            "hwupload_cuda",
+            // opencl
+            "scale_opencl",
             "tonemap_opencl",
+            "overlay_opencl",
+            // vaapi
+            "scale_vaapi",
+            "deinterlace_vaapi",
             "tonemap_vaapi",
+            "overlay_vaapi",
+            "hwupload_vaapi"
         };
 
         private static readonly IReadOnlyDictionary<int, string[]> _filterOptionsDict = new Dictionary<int, string[]>
         {
             { 0, new string[] { "scale_cuda", "Output format (default \"same\")" } },
             { 1, new string[] { "tonemap_cuda", "GPU accelerated HDR to SDR tonemapping" } },
-            { 2, new string[] { "tonemap_opencl", "bt2390" } }
+            { 2, new string[] { "tonemap_opencl", "bt2390" } },
+            { 3, new string[] { "overlay_opencl", "Action to take when encountering EOF from secondary input" } },
+            { 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } }
         };
 
         // These are the library versions that corresponds to our minimum ffmpeg version 4.x according to the version table below
@@ -144,7 +165,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             string output;
             try
             {
-                output = GetProcessOutput(_encoderPath, "-version");
+                output = GetProcessOutput(_encoderPath, "-version", false);
             }
             catch (Exception ex)
             {
@@ -225,7 +246,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             string output;
             try
             {
-                output = GetProcessOutput(_encoderPath, "-version");
+                output = GetProcessOutput(_encoderPath, "-version", false);
             }
             catch (Exception ex)
             {
@@ -318,12 +339,38 @@ namespace MediaBrowser.MediaEncoding.Encoder
             return map;
         }
 
+        public bool CheckVaapiDeviceByDriverName(string driverName, string renderNodePath)
+        {
+            if (!OperatingSystem.IsLinux())
+            {
+                return false;
+            }
+
+            if (string.IsNullOrEmpty(driverName) || string.IsNullOrEmpty(renderNodePath))
+            {
+                return false;
+            }
+
+            string output;
+            try
+            {
+                output = GetProcessOutput(_encoderPath, "-v verbose -hide_banner -init_hw_device vaapi=va:" + renderNodePath, true);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error detecting the given vaapi render node path");
+                return false;
+            }
+
+            return output.Contains(driverName, StringComparison.Ordinal);
+        }
+
         private IEnumerable<string> GetHwaccelTypes()
         {
             string? output = null;
             try
             {
-                output = GetProcessOutput(_encoderPath, "-hwaccels");
+                output = GetProcessOutput(_encoderPath, "-hwaccels", false);
             }
             catch (Exception ex)
             {
@@ -351,7 +398,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             string output;
             try
             {
-                output = GetProcessOutput(_encoderPath, "-h filter=" + filter);
+                output = GetProcessOutput(_encoderPath, "-h filter=" + filter, false);
             }
             catch (Exception ex)
             {
@@ -375,7 +422,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             string output;
             try
             {
-                output = GetProcessOutput(_encoderPath, "-" + codecstr);
+                output = GetProcessOutput(_encoderPath, "-" + codecstr, false);
             }
             catch (Exception ex)
             {
@@ -406,7 +453,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             string output;
             try
             {
-                output = GetProcessOutput(_encoderPath, "-filters");
+                output = GetProcessOutput(_encoderPath, "-filters", false);
             }
             catch (Exception ex)
             {
@@ -444,7 +491,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             return dict;
         }
 
-        private string GetProcessOutput(string path, string arguments)
+        private string GetProcessOutput(string path, string arguments, bool readStdErr)
         {
             using (var process = new Process()
             {
@@ -455,7 +502,6 @@ namespace MediaBrowser.MediaEncoding.Encoder
                     WindowStyle = ProcessWindowStyle.Hidden,
                     ErrorDialog = false,
                     RedirectStandardOutput = true,
-                    // ffmpeg uses stderr to log info, don't show this
                     RedirectStandardError = true
                 }
             })
@@ -464,7 +510,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
                 process.Start();
 
-                return process.StandardOutput.ReadToEnd();
+                return readStdErr ? process.StandardError.ReadToEnd() : process.StandardOutput.ReadToEnd();
             }
         }
     }

MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs

@@ -65,6 +65,10 @@ namespace MediaBrowser.MediaEncoding.Encoder
         private List<string> _filters = new List<string>();
         private IDictionary<int, bool> _filtersWithOption = new Dictionary<int, bool>();
 
+        private bool _isVaapiDeviceAmd = false;
+        private bool _isVaapiDeviceInteliHD = false;
+        private bool _isVaapiDeviceInteli965 = false;
+
         private Version _ffmpegVersion = null;
         private string _ffmpegPath = string.Empty;
         private string _ffprobePath;
@@ -114,9 +118,9 @@ namespace MediaBrowser.MediaEncoding.Encoder
             }
 
             // Write the FFmpeg path to the config/encoding.xml file as <EncoderAppPathDisplay> so it appears in UI
-            var config = _configurationManager.GetEncodingOptions();
-            config.EncoderAppPathDisplay = _ffmpegPath ?? string.Empty;
-            _configurationManager.SaveConfiguration("encoding", config);
+            var options = _configurationManager.GetEncodingOptions();
+            options.EncoderAppPathDisplay = _ffmpegPath ?? string.Empty;
+            _configurationManager.SaveConfiguration("encoding", options);
 
             // Only if mpeg path is set, try and set path to probe
             if (_ffmpegPath != null)
@@ -134,7 +138,31 @@ namespace MediaBrowser.MediaEncoding.Encoder
                 SetAvailableHwaccels(validator.GetHwaccels());
                 SetMediaEncoderVersion(validator);
 
-                _threads = EncodingHelper.GetNumberOfThreads(null, _configurationManager.GetEncodingOptions(), null);
+                options = _configurationManager.GetEncodingOptions();
+                _threads = EncodingHelper.GetNumberOfThreads(null, options, null);
+
+                // Check the Vaapi device vendor
+                if (OperatingSystem.IsLinux()
+                    && SupportsHwaccel("vaapi")
+                    && !string.IsNullOrEmpty(options.VaapiDevice)
+                    && string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase))
+                {
+                    _isVaapiDeviceAmd = validator.CheckVaapiDeviceByDriverName("Mesa Gallium driver", options.VaapiDevice);
+                    _isVaapiDeviceInteliHD = validator.CheckVaapiDeviceByDriverName("Intel iHD driver", options.VaapiDevice);
+                    _isVaapiDeviceInteli965 = validator.CheckVaapiDeviceByDriverName("Intel i965 driver", options.VaapiDevice);
+                    if (_isVaapiDeviceAmd)
+                    {
+                        _logger.LogInformation("VAAPI device {RenderNodePath} is AMD GPU", options.VaapiDevice);
+                    }
+                    else if (_isVaapiDeviceInteliHD)
+                    {
+                        _logger.LogInformation("VAAPI device {RenderNodePath} is Intel GPU (iHD)", options.VaapiDevice);
+                    }
+                    else if (_isVaapiDeviceInteli965)
+                    {
+                        _logger.LogInformation("VAAPI device {RenderNodePath} is Intel GPU (i965)", options.VaapiDevice);
+                    }
+                }
             }
 
             _logger.LogInformation("FFmpeg: {FfmpegPath}", _ffmpegPath ?? string.Empty);
@@ -301,6 +329,21 @@ namespace MediaBrowser.MediaEncoding.Encoder
             return false;
         }
 
+        public bool IsVaapiDeviceAmd()
+        {
+            return _isVaapiDeviceAmd;
+        }
+
+        public bool IsVaapiDeviceInteliHD()
+        {
+            return _isVaapiDeviceInteliHD;
+        }
+
+        public bool IsVaapiDeviceInteli965()
+        {
+            return _isVaapiDeviceInteli965;
+        }
+
         public Version GetMediaEncoderVersion()
         {
             return _ffmpegVersion;

MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs

@@ -764,18 +764,23 @@ namespace MediaBrowser.MediaEncoding.Probing
 
                 if (!stream.BitDepth.HasValue)
                 {
-                    if (!string.IsNullOrEmpty(streamInfo.PixelFormat)
-                        && streamInfo.PixelFormat.Contains("p10", StringComparison.OrdinalIgnoreCase))
+                    if (!string.IsNullOrEmpty(streamInfo.PixelFormat))
                     {
-                        stream.BitDepth = 10;
-                    }
-
-                    if (!string.IsNullOrEmpty(streamInfo.Profile)
-                        && (streamInfo.Profile.Contains("Main 10", StringComparison.OrdinalIgnoreCase)
-                            || streamInfo.Profile.Contains("High 10", StringComparison.OrdinalIgnoreCase)
-                            || streamInfo.Profile.Contains("Profile 2", StringComparison.OrdinalIgnoreCase)))
-                    {
-                        stream.BitDepth = 10;
+                        if (string.Equals(streamInfo.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase)
+                            || string.Equals(streamInfo.PixelFormat, "yuv444p", StringComparison.OrdinalIgnoreCase))
+                        {
+                            stream.BitDepth = 8;
+                        }
+                        else if (string.Equals(streamInfo.PixelFormat, "yuv420p10le", StringComparison.OrdinalIgnoreCase)
+                            || string.Equals(streamInfo.PixelFormat, "yuv444p10le", StringComparison.OrdinalIgnoreCase))
+                        {
+                            stream.BitDepth = 10;
+                        }
+                        else if (string.Equals(streamInfo.PixelFormat, "yuv420p12le", StringComparison.OrdinalIgnoreCase)
+                            || string.Equals(streamInfo.PixelFormat, "yuv444p12le", StringComparison.OrdinalIgnoreCase))
+                        {
+                            stream.BitDepth = 12;
+                        }
                     }
                 }
 

MediaBrowser.Model/Configuration/EncodingOptions.cs

@@ -16,9 +16,6 @@ namespace MediaBrowser.Model.Configuration
             // This is a DRM device that is almost guaranteed to be there on every intel platform,
             // plus it's the default one in ffmpeg if you don't specify anything
             VaapiDevice = "/dev/dri/renderD128";
-            // This is the OpenCL device that is used for tonemapping.
-            // The left side of the dot is the platform number, and the right side is the device number on the platform.
-            OpenclDevice = "0.0";
             EnableTonemapping = false;
             EnableVppTonemapping = false;
             TonemappingAlgorithm = "hable";
@@ -34,6 +31,9 @@ namespace MediaBrowser.Model.Configuration
             EnableDecodingColorDepth10Hevc = true;
             EnableDecodingColorDepth10Vp9 = true;
             EnableEnhancedNvdecDecoder = true;
+            PreferSystemNativeHwDecoder = true;
+            EnableIntelLowPowerH264HwEncoder = false;
+            EnableIntelLowPowerHevcHwEncoder = false;
             EnableHardwareEncoding = true;
             AllowHevcEncoding = false;
             EnableSubtitleExtraction = true;
@@ -70,8 +70,6 @@ namespace MediaBrowser.Model.Configuration
 
         public string VaapiDevice { get; set; }
 
-        public string OpenclDevice { get; set; }
-
         public bool EnableTonemapping { get; set; }
 
         public bool EnableVppTonemapping { get; set; }
@@ -104,6 +102,12 @@ namespace MediaBrowser.Model.Configuration
 
         public bool EnableEnhancedNvdecDecoder { get; set; }
 
+        public bool PreferSystemNativeHwDecoder { get; set; }
+
+        public bool EnableIntelLowPowerH264HwEncoder { get; set; }
+
+        public bool EnableIntelLowPowerHevcHwEncoder { get; set; }
+
         public bool EnableHardwareEncoding { get; set; }
 
         public bool AllowHevcEncoding { get; set; }