add more separate hw decoding toggles

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -103,6 +103,11 @@ namespace MediaBrowser.Controller.MediaEncoding
                 return false;
             }
 
+            if (!_mediaEncoder.SupportsHwaccel("vaapi"))
+            {
+                return false;
+            }
+
             return true;
         }
 
@@ -444,18 +449,30 @@ namespace MediaBrowser.Controller.MediaEncoding
         public string GetInputArgument(EncodingJobInfo state, EncodingOptions encodingOptions)
         {
             var arg = new StringBuilder();
+            var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions);
+            var outputVideoCodec = GetVideoEncoder(state, encodingOptions);
 
-            if (state.IsVideoRequest
-                && string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase))
+            if (state.IsVideoRequest && string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase))
             {
-                arg.Append("-hwaccel vaapi -hwaccel_output_format vaapi")
-                    .Append(" -vaapi_device ")
-                    .Append(encodingOptions.VaapiDevice)
-                    .Append(' ');
+                // While using VAAPI decoder
+                if ((videoDecoder ?? string.Empty).IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1)
+                {
+                    arg.Append("-hwaccel_output_format vaapi")
+                        .Append(" -vaapi_device ")
+                        .Append(encodingOptions.VaapiDevice)
+                        .Append(" ");
+                }
+                // While using SW decoder and VAAPI encoder
+                else if ((videoDecoder ?? string.Empty).IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) == -1
+                    && (outputVideoCodec ?? string.Empty).IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1)
+                {
+                    arg.Append("-vaapi_device ")
+                        .Append(encodingOptions.VaapiDevice)
+                        .Append(" ");
+                }
             }
 
-            if (state.IsVideoRequest
-                && string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase))
+            if (state.IsVideoRequest && string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase))
             {
                 var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions);
                 var outputVideoCodec = GetVideoEncoder(state, encodingOptions);
@@ -1655,7 +1672,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                     For software decoding and hardware encoding option, frames must be hwuploaded into hardware
                     with fixed frame size.
                 */
-                if (!string.IsNullOrEmpty(videoDecoder) && videoDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase))
+                if ((videoDecoder ?? string.Empty).IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1)
                 {
                     retStr = " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay_qsv=x=(W-w)/2:y=(H-h)/2{3}\"";
                 }
@@ -2511,16 +2528,15 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// </summary>
         protected string GetHardwareAcceleratedVideoDecoder(EncodingJobInfo state, EncodingOptions encodingOptions)
         {
+            var videoType = state.MediaSource.VideoType ?? VideoType.VideoFile;
+            var videoStream = state.VideoStream;
+            var IsColorDepth10 = (videoStream.Profile ?? string.Empty).IndexOf("10", StringComparison.OrdinalIgnoreCase) != -1;
+
             if (string.Equals(state.OutputVideoCodec, "copy", StringComparison.OrdinalIgnoreCase))
             {
                 return null;
             }
 
-            return GetHardwareAcceleratedVideoDecoder(state.MediaSource.VideoType ?? VideoType.VideoFile, state.VideoStream, encodingOptions);
-        }
-
-        public string GetHardwareAcceleratedVideoDecoder(VideoType videoType, MediaStream videoStream, EncodingOptions encodingOptions)
-        {
             // Only use alternative encoders for video files.
             // When using concat with folder rips, if the mfx session fails to initialize, ffmpeg will be stuck retrying and will not exit gracefully
             // Since transcoding of folder rips is expiremental anyway, it's not worth adding additional variables such as this.
@@ -2533,6 +2549,14 @@ namespace MediaBrowser.Controller.MediaEncoding
                 && !string.IsNullOrEmpty(videoStream.Codec)
                 && !string.IsNullOrEmpty(encodingOptions.HardwareAccelerationType))
             {
+                // Only hevc and vp9 formats have 10-bit hardware decoder support now.
+                if (IsColorDepth10 && !(string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(videoStream.Codec, "vp9", StringComparison.OrdinalIgnoreCase)))
+                {
+                    return null;
+                }
+
                 if (string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase))
                 {
                     switch (videoStream.Codec.ToLowerInvariant())
@@ -2554,8 +2578,17 @@ namespace MediaBrowser.Controller.MediaEncoding
                         case "h265":
                             if (_mediaEncoder.SupportsDecoder("hevc_qsv") && encodingOptions.HardwareDecodingCodecs.Contains("hevc", StringComparer.OrdinalIgnoreCase))
                             {
-                                //return "-c:v hevc_qsv -load_plugin hevc_hw ";
-                                return "-c:v hevc_qsv";
+                                if (IsColorDepth10)
+                                {
+                                    if (encodingOptions.EnableDecodingColorDepth10)
+                                    {
+                                        return "-c:v hevc_qsv ";
+                                    }
+
+                                    return null;
+                                }
+
+                                return "-c:v hevc_qsv ";
                             }
                             break;
                         case "mpeg2video":
@@ -2570,6 +2603,28 @@ namespace MediaBrowser.Controller.MediaEncoding
                                 return "-c:v vc1_qsv";
                             }
                             break;
+                        case "vp8":
+                            if (_mediaEncoder.SupportsDecoder("vp8_qsv") && encodingOptions.HardwareDecodingCodecs.Contains("vp8", StringComparer.OrdinalIgnoreCase))
+                            {
+                                return "-c:v vp8_qsv ";
+                            }
+                            break;
+                        case "vp9":
+                            if (_mediaEncoder.SupportsDecoder("vp9_qsv") && encodingOptions.HardwareDecodingCodecs.Contains("vp9", StringComparer.OrdinalIgnoreCase))
+                            {
+                                if (IsColorDepth10)
+                                {
+                                    if (encodingOptions.EnableDecodingColorDepth10)
+                                    {
+                                        return "-c:v vp9_qsv ";
+                                    }
+
+                                    return null;
+                                }
+
+                                return "-c:v vp9_qsv ";
+                            }
+                            break;
                     }
                 }
 
@@ -2594,7 +2649,17 @@ namespace MediaBrowser.Controller.MediaEncoding
                         case "h265":
                             if (_mediaEncoder.SupportsDecoder("hevc_cuvid") && encodingOptions.HardwareDecodingCodecs.Contains("hevc", StringComparer.OrdinalIgnoreCase))
                             {
-                                return "-c:v hevc_cuvid";
+                                if (IsColorDepth10)
+                                {
+                                    if (encodingOptions.EnableDecodingColorDepth10)
+                                    {
+                                        return "-c:v hevc_cuvid ";
+                                    }
+
+                                    return null;
+                                }
+
+                                return "-c:v hevc_cuvid ";
                             }
                             break;
                         case "mpeg2video":
@@ -2615,6 +2680,28 @@ namespace MediaBrowser.Controller.MediaEncoding
                                 return "-c:v mpeg4_cuvid";
                             }
                             break;
+                        case "vp8":
+                            if (_mediaEncoder.SupportsDecoder("vp8_cuvid") && encodingOptions.HardwareDecodingCodecs.Contains("vp8", StringComparer.OrdinalIgnoreCase))
+                            {
+                                return "-c:v vp8_cuvid ";
+                            }
+                            break;
+                        case "vp9":
+                            if (_mediaEncoder.SupportsDecoder("vp9_cuvid") && encodingOptions.HardwareDecodingCodecs.Contains("vp9", StringComparer.OrdinalIgnoreCase))
+                            {
+                                if (IsColorDepth10)
+                                {
+                                    if (encodingOptions.EnableDecodingColorDepth10)
+                                    {
+                                        return "-c:v vp9_cuvid ";
+                                    }
+
+                                    return null;
+                                }
+
+                                return "-c:v vp9_cuvid ";
+                            }
+                            break;
                     }
                 }
 
@@ -2633,7 +2720,17 @@ namespace MediaBrowser.Controller.MediaEncoding
                         case "h265":
                             if (_mediaEncoder.SupportsDecoder("hevc_mediacodec") && encodingOptions.HardwareDecodingCodecs.Contains("hevc", StringComparer.OrdinalIgnoreCase))
                             {
-                                return "-c:v hevc_mediacodec";
+                                if (IsColorDepth10)
+                                {
+                                    if (encodingOptions.EnableDecodingColorDepth10)
+                                    {
+                                        return "-c:v hevc_mediacodec ";
+                                    }
+
+                                    return null;
+                                }
+
+                                return "-c:v hevc_mediacodec ";
                             }
                             break;
                         case "mpeg2video":
@@ -2657,7 +2754,17 @@ namespace MediaBrowser.Controller.MediaEncoding
                         case "vp9":
                             if (_mediaEncoder.SupportsDecoder("vp9_mediacodec") && encodingOptions.HardwareDecodingCodecs.Contains("vp9", StringComparer.OrdinalIgnoreCase))
                             {
-                                return "-c:v vp9_mediacodec";
+                                if (IsColorDepth10)
+                                {
+                                    if (encodingOptions.EnableDecodingColorDepth10)
+                                    {
+                                        return "-c:v vp9_mediacodec ";
+                                    }
+
+                                    return null;
+                                }
+
+                                return "-c:v vp9_mediacodec ";
                             }
                             break;
                     }
@@ -2697,27 +2804,133 @@ namespace MediaBrowser.Controller.MediaEncoding
 
                 else if (string.Equals(encodingOptions.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase))
                 {
-                    if (Environment.OSVersion.Platform == PlatformID.Win32NT)
+                    switch (videoStream.Codec.ToLowerInvariant())
                     {
-                        if (Environment.OSVersion.Version.Major > 6 || (Environment.OSVersion.Version.Major == 6 && Environment.OSVersion.Version.Minor > 1))
-                            return "-hwaccel d3d11va";
-                        else
-                            return "-hwaccel dxva2";
+                        case "avc":
+                        case "h264":
+                            return GetHwaccelType(state, encodingOptions, "h264");
+                        case "hevc":
+                        case "h265":
+                            if (IsColorDepth10)
+                            {
+                                if (encodingOptions.EnableDecodingColorDepth10)
+                                {
+                                    return GetHwaccelType(state, encodingOptions, "hevc");
+                                }
+
+                                return null;
+                            }
+
+                            return GetHwaccelType(state, encodingOptions, "hevc");
+                        case "mpeg2video":
+                            return GetHwaccelType(state, encodingOptions, "mpeg2video");
+                        case "vc1":
+                            return GetHwaccelType(state, encodingOptions, "vc1");
+                        case "mpeg4":
+                            return GetHwaccelType(state, encodingOptions, "mpeg4");
+                        case "vp9":
+                            if (IsColorDepth10)
+                            {
+                                if (encodingOptions.EnableDecodingColorDepth10)
+                                {
+                                    return GetHwaccelType(state, encodingOptions, "vp9");
+                                }
+
+                                return null;
+                            }
+
+                            return GetHwaccelType(state, encodingOptions, "vp9");
                     }
-                    else
+                }
+
+                else if (string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase))
+                {
+                    switch (videoStream.Codec.ToLowerInvariant())
                     {
-                        return "-hwaccel vaapi";
+                        case "avc":
+                        case "h264":
+                            return GetHwaccelType(state, encodingOptions, "h264");
+                        case "hevc":
+                        case "h265":
+                            if (IsColorDepth10)
+                            {
+                                if (encodingOptions.EnableDecodingColorDepth10)
+                                {
+                                    return GetHwaccelType(state, encodingOptions, "hevc");
+                                }
+
+                                return null;
+                            }
+
+                            return GetHwaccelType(state, encodingOptions, "hevc");
+                        case "mpeg2video":
+                            return GetHwaccelType(state, encodingOptions, "mpeg2video");
+                        case "vc1":
+                            return GetHwaccelType(state, encodingOptions, "vc1");
+                        case "vp8":
+                            return GetHwaccelType(state, encodingOptions, "vp8");
+                        case "vp9":
+                            if (IsColorDepth10)
+                            {
+                                if (encodingOptions.EnableDecodingColorDepth10)
+                                {
+                                    return GetHwaccelType(state, encodingOptions, "vp9");
+                                }
+
+                                return null;
+                            }
+
+                            return GetHwaccelType(state, encodingOptions, "vp9");
                     }
                 }
             }
 
+            var whichCodec = videoStream.Codec.ToLowerInvariant();
+            switch (whichCodec)
+            {
+                case "avc":
+                    whichCodec = "h264";
+                    break;
+                case "h265":
+                    whichCodec = "hevc";
+                    break;
+            }
+
             // Avoid a second attempt if no hardware acceleration is being used
-            encodingOptions.HardwareDecodingCodecs = Array.Empty<string>();
+            encodingOptions.HardwareDecodingCodecs = encodingOptions.HardwareDecodingCodecs.Where(val => val != whichCodec).ToArray();
 
             // leave blank so ffmpeg will decide
             return null;
         }
 
+        /// <summary>
+        /// Gets a hwaccel type to use as a hardware decoder(dxva/vaapi) depending on the system
+        /// </summary>
+        public string GetHwaccelType(EncodingJobInfo state, EncodingOptions options, string videoCodec)
+        {
+            var IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT;
+            var IsNewWindows = Environment.OSVersion.Version.Major > 6 || (Environment.OSVersion.Version.Major == 6 && Environment.OSVersion.Version.Minor > 1);
+            var IsDxvaSupported = _mediaEncoder.SupportsHwaccel("dxva2") || _mediaEncoder.SupportsHwaccel("d3d11va");
+
+            if ((IsDxvaSupported || IsVaapiSupported(state)) && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase))
+            {
+                if (!IsWindows)
+                {
+                    return "-hwaccel vaapi ";
+                }
+                else if (IsWindows && IsNewWindows)
+                {
+                    return "-hwaccel d3d11va ";
+                }
+                else if (IsWindows && !IsNewWindows)
+                {
+                    return "-hwaccel dxva2 ";
+                }
+            }
+
+            return null;
+        }
+
         public string GetSubtitleEmbedArguments(EncodingJobInfo state)
         {
             if (state.SubtitleStream == null || state.SubtitleDeliveryMethod != SubtitleDeliveryMethod.Embed)

MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs

@@ -26,6 +26,13 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// <value>The encoder path.</value>
         string EncoderPath { get; }
 
+        /// <summary>
+        /// Supportses the encoder.
+        /// </summary>
+        /// <param name="encoder">The encoder.</param>
+        /// <returns><c>true</c> if XXXX, <c>false</c> otherwise.</returns>
+        bool SupportsEncoder(string encoder);
+
         /// <summary>
         /// Supportses the decoder.
         /// </summary>
@@ -33,6 +40,13 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// <returns><c>true</c> if XXXX, <c>false</c> otherwise.</returns>
         bool SupportsDecoder(string decoder);
 
+        /// <summary>
+        /// Supportses the hwaccel.
+        /// </summary>
+        /// <param name="hwaccel">The hwaccel.</param>
+        /// <returns><c>true</c> if XXXX, <c>false</c> otherwise.</returns>
+        bool SupportsHwaccel(string hwaccel);
+
         /// <summary>
         /// Extracts the audio image.
         /// </summary>
@@ -98,7 +112,6 @@ namespace MediaBrowser.Controller.MediaEncoding
         void SetFFmpegPath();
 
         void UpdateEncoderPath(string path, string pathType);
-        bool SupportsEncoder(string encoder);
 
         IEnumerable<string> GetPrimaryPlaylistVobFiles(string path, IIsoMount isoMount, uint? titleNumber);
     }

MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs

@@ -14,23 +14,38 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
         private static readonly string[] requiredDecoders = new[]
         {
+            "h264",
+            "hevc",
             "mpeg2video",
+            "mpeg4",
+            "msmpeg4",
+            "dts",
+            "ac3",
+            "aac",
+            "mp3",
             "h264_qsv",
             "hevc_qsv",
             "mpeg2_qsv",
-            "mpeg2_mmal",
-            "mpeg4_mmal",
             "vc1_qsv",
-            "vc1_mmal",
+            "vp8_qsv",
+            "vp9_qsv",
             "h264_cuvid",
             "hevc_cuvid",
-            "dts",
-            "ac3",
-            "aac",
-            "mp3",
-            "h264",
+            "mpeg2_cuvid",
+            "vc1_cuvid",
+            "mpeg4_cuvid",
+            "vp8_cuvid",
+            "vp9_cuvid",
             "h264_mmal",
-            "hevc"
+            "mpeg2_mmal",
+            "mpeg4_mmal",
+            "vc1_mmal",
+            "h264_mediacodec",
+            "hevc_mediacodec",
+            "mpeg2_mediacodec",
+            "mpeg4_mediacodec",
+            "vp8_mediacodec",
+            "vp9_mediacodec"
         };
 
         private static readonly string[] requiredEncoders = new[]
@@ -43,22 +58,22 @@ namespace MediaBrowser.MediaEncoding.Encoder
             "libvpx-vp9",
             "aac",
             "libfdk_aac",
+            "ac3",
             "libmp3lame",
             "libopus",
             "libvorbis",
             "srt",
-            "h264_nvenc",
-            "hevc_nvenc",
+            "h264_amf",
+            "hevc_amf",
             "h264_qsv",
             "hevc_qsv",
-            "h264_omx",
-            "hevc_omx",
+            "h264_nvenc",
+            "hevc_nvenc",
             "h264_vaapi",
             "hevc_vaapi",
-            "h264_v4l2m2m",
-            "ac3",
-            "h264_amf",
-            "hevc_amf"
+            "h264_omx",
+            "hevc_omx",
+            "h264_v4l2m2m"
         };
 
         // Try and use the individual library versions to determine a FFmpeg version
@@ -159,6 +174,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
         public IEnumerable<string> GetEncoders() => GetCodecs(Codec.Encoder);
 
+        public IEnumerable<string> GetHwaccels() => GetHwaccelTypes();
+
         /// <summary>
         /// Using the output from "ffmpeg -version" work out the FFmpeg version.
         /// For pre-built binaries the first line should contain a string like "ffmpeg version x.y", which is easy
@@ -218,6 +235,32 @@ namespace MediaBrowser.MediaEncoding.Encoder
             Decoder
         }
 
+        private IEnumerable<string> GetHwaccelTypes()
+        {
+            string output = null;
+            try
+            {
+                output = GetProcessOutput(_encoderPath, "-hwaccels");
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error detecting available hwaccel types");
+            }
+
+            if (string.IsNullOrWhiteSpace(output))
+            {
+                return Enumerable.Empty<string>();
+            }
+
+            var found = output.Split(new char[] {'\r','\n'},StringSplitOptions.RemoveEmptyEntries).Distinct().ToList();
+
+            found.RemoveAt(0);
+
+            _logger.LogInformation("Available hwaccel types: {Types}", found);
+
+            return found;
+        }
+
         private IEnumerable<string> GetCodecs(Codec codec)
         {
             string codecstr = codec == Codec.Encoder ? "encoders" : "decoders";

MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs

@@ -111,6 +111,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
                 SetAvailableDecoders(validator.GetDecoders());
                 SetAvailableEncoders(validator.GetEncoders());
+                SetAvailableHwaccels(validator.GetHwaccels());
             }
 
             _logger.LogInformation("FFmpeg: {EncoderLocation}: {FfmpegPath}", EncoderLocation, _ffmpegPath ?? string.Empty);
@@ -257,6 +258,13 @@ namespace MediaBrowser.MediaEncoding.Encoder
             //_logger.Info("Supported decoders: {0}", string.Join(",", list.ToArray()));
         }
 
+        private List<string> _hwaccels = new List<string>();
+        public void SetAvailableHwaccels(IEnumerable<string> list)
+        {
+            _hwaccels = list.ToList();
+            //_logger.Info("Supported hwaccels: {0}", string.Join(",", list.ToArray()));
+        }
+
         public bool SupportsEncoder(string encoder)
         {
             return _encoders.Contains(encoder, StringComparer.OrdinalIgnoreCase);
@@ -267,6 +275,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
             return _decoders.Contains(decoder, StringComparer.OrdinalIgnoreCase);
         }
 
+        public bool SupportsHwaccel(string hwaccel)
+        {
+            return _hwaccels.Contains(hwaccel, StringComparer.OrdinalIgnoreCase);
+        }
+
         public bool CanEncodeToAudioCodec(string codec)
         {
             if (string.Equals(codec, "opus", StringComparison.OrdinalIgnoreCase))

MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs

@@ -737,7 +737,7 @@ namespace MediaBrowser.MediaEncoding.Subtitles
                 var charset = CharsetDetector.DetectFromStream(stream).Detected?.EncodingName;
 
                 // UTF16 is automatically converted to UTF8 by FFmpeg, do not specify a character encoding
-                if ((path.EndsWith(".ass") || path.EndsWith(".ssa"))
+                if ((path.EndsWith(".ass") || path.EndsWith(".ssa") || path.EndsWith(".srt"))
                     && (string.Equals(charset, "utf-16le", StringComparison.OrdinalIgnoreCase)
                         || string.Equals(charset, "utf-16be", StringComparison.OrdinalIgnoreCase)))
                 {

MediaBrowser.Model/Configuration/EncodingOptions.cs

@@ -25,6 +25,7 @@ namespace MediaBrowser.Model.Configuration
         public int H265Crf { get; set; }
         public string EncoderPreset { get; set; }
         public string DeinterlaceMethod { get; set; }
+        public bool EnableDecodingColorDepth10 { get; set; }
         public bool EnableHardwareEncoding { get; set; }
         public bool EnableSubtitleExtraction { get; set; }
 
@@ -41,6 +42,7 @@ namespace MediaBrowser.Model.Configuration
             H264Crf = 23;
             H265Crf = 28;
             DeinterlaceMethod = "yadif";
+            EnableDecodingColorDepth10 = true;
             EnableHardwareEncoding = true;
             EnableSubtitleExtraction = true;
             HardwareDecodingCodecs = new string[] { "h264", "vc1" };