Merge pull request #2503 from nyanmisaka/vaapi

Fix various bugs in HWA subtitle burn-in

(cherry picked from commit 9aefb41512e8e2655c9c894180907d8ae60ae30e)
Signed-off-by: Joshua M. Boniface <joshua@boniface.me>

MediaBrowser.Api/Playback/Hls/DynamicHlsService.cs

@@ -927,61 +927,69 @@ namespace MediaBrowser.Api.Playback.Hls
             }
             else
             {
+                var gopArg = string.Empty;
                 var keyFrameArg = string.Format(
                     CultureInfo.InvariantCulture,
                     " -force_key_frames:0 \"expr:gte(t,{0}+n_forced*{1})\"",
                     GetStartNumber(state) * state.SegmentLength,
                     state.SegmentLength);
-                if (state.TargetFramerate.HasValue)
+
+                var framerate = state.VideoStream?.RealFrameRate;
+
+                if (framerate != null && framerate.HasValue)
                 {
                     // This is to make sure keyframe interval is limited to our segment,
                     // as forcing keyframes is not enough.
                     // Example: we encoded half of desired length, then codec detected
                     // scene cut and inserted a keyframe; next forced keyframe would
-                    // be created outside of segment, which breaks seeking.
-                    keyFrameArg += string.Format(
+                    // be created outside of segment, which breaks seeking
+                    // -sc_threshold 0 is used to prevent the hardware encoder from post processing to break the set keyframe
+                    gopArg = string.Format(
                         CultureInfo.InvariantCulture,
-                        " -g {0} -keyint_min {0}",
-                        (int)(state.SegmentLength * state.TargetFramerate)
+                        " -g {0} -keyint_min {0} -sc_threshold 0",
+                        Math.Ceiling(state.SegmentLength * framerate.Value)
                     );
                 }
 
-                var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
-
                 args += " " + EncodingHelper.GetVideoQualityParam(state, codec, encodingOptions, GetDefaultEncoderPreset());
 
-                // Unable to force key frames to h264_qsv transcode
-                if (string.Equals(codec, "h264_qsv", StringComparison.OrdinalIgnoreCase))
+                // Unable to force key frames using these hw encoders, set key frames by GOP
+                if (string.Equals(codec, "h264_qsv", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(codec, "h264_nvenc", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(codec, "h264_amf", StringComparison.OrdinalIgnoreCase))
                 {
-                    Logger.LogInformation("Bug Workaround: Disabling force_key_frames for h264_qsv");
+                    args += " " + gopArg;
                 }
                 else
                 {
-                    args += " " + keyFrameArg;
+                    args += " " + keyFrameArg + gopArg;
                 }
 
                 //args += " -mixed-refs 0 -refs 3 -x264opts b_pyramid=0:weightb=0:weightp=0";
 
+                var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
+
+                // This is for graphical subs
+                if (hasGraphicalSubs)
+                {
+                    args += EncodingHelper.GetGraphicalSubtitleParam(state, encodingOptions, codec);
+                }
                 // Add resolution params, if specified
-                if (!hasGraphicalSubs)
+                else
                 {
-                    args += EncodingHelper.GetOutputSizeParam(state, encodingOptions, codec, true);
+                    args += EncodingHelper.GetOutputSizeParam(state, encodingOptions, codec);
                 }
 
-                // This is for internal graphical subs
-                if (hasGraphicalSubs)
+                // -start_at_zero is necessary to use with -ss when seeking,
+                // otherwise the target position cannot be determined.
+                if (!(state.SubtitleStream != null && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream))
                 {
-                    args += EncodingHelper.GetGraphicalSubtitleParam(state, encodingOptions, codec);
+                    args += " -start_at_zero";
                 }
 
                 //args += " -flags -global_header";
             }
 
-            if (args.IndexOf("-copyts", StringComparison.OrdinalIgnoreCase) == -1)
-            {
-                args += " -copyts";
-            }
-
             if (!string.IsNullOrEmpty(state.OutputVideoSync))
             {
                 args += " -vsync " + state.OutputVideoSync;
@@ -1024,7 +1032,7 @@ namespace MediaBrowser.Api.Playback.Hls
             }
 
             return string.Format(
-                "{0} {1} -map_metadata -1 -map_chapters -1 -threads {2} {3} {4} {5} -f hls -max_delay 5000000 -avoid_negative_ts disabled -start_at_zero -hls_time {6} -individual_header_trailer 0 -hls_segment_type {7} -start_number {8} -hls_segment_filename \"{9}\" -hls_playlist_type vod -hls_list_size 0 -y \"{10}\"",
+                "{0} {1} -map_metadata -1 -map_chapters -1 -threads {2} {3} {4} {5} -copyts -avoid_negative_ts disabled -f hls -max_delay 5000000 -hls_time {6} -individual_header_trailer 0 -hls_segment_type {7} -start_number {8} -hls_segment_filename \"{9}\" -hls_playlist_type vod -hls_list_size 0 -y \"{10}\"",
                 inputModifier,
                 EncodingHelper.GetInputArgument(state, encodingOptions),
                 threads,

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -460,16 +460,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             if (state.IsVideoRequest
                 && string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase))
             {
-                var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
-                var hwOutputFormat = "vaapi";
-
-                if (hasGraphicalSubs)
-                {
-                    hwOutputFormat = "yuv420p";
-                }
-
-                arg.Append("-hwaccel vaapi -hwaccel_output_format ")
-                    .Append(hwOutputFormat)
+                arg.Append("-hwaccel vaapi -hwaccel_output_format vaapi")
                     .Append(" -vaapi_device ")
                     .Append(encodingOptions.VaapiDevice)
                     .Append(' ');
@@ -480,20 +471,26 @@ namespace MediaBrowser.Controller.MediaEncoding
             {
                 var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions);
                 var outputVideoCodec = GetVideoEncoder(state, encodingOptions);
+				
+                var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
 
-                if (encodingOptions.EnableHardwareEncoding && outputVideoCodec.Contains("qsv", StringComparison.OrdinalIgnoreCase))
+                if (!hasTextSubs)
                 {
-                    if (!string.IsNullOrEmpty(videoDecoder) && videoDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase))
-                    {
-                        arg.Append("-hwaccel qsv ");
-                    }
-                    else
+                    // While using QSV encoder
+                    if ((outputVideoCodec ?? string.Empty).IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1)
                     {
-                        arg.Append("-init_hw_device qsv=hw -filter_hw_device hw ");
+                        // While using QSV decoder
+                        if ((videoDecoder ?? string.Empty).IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1)
+                        {
+                            arg.Append("-hwaccel qsv ");
+                        }
+                        // While using SW decoder
+                        else
+                        {
+                            arg.Append("-init_hw_device qsv=hw -filter_hw_device hw ");
+                        }
                     }
                 }
-
-                arg.Append(videoDecoder + " ");
             }
 
             arg.Append("-i ")
@@ -503,17 +500,6 @@ namespace MediaBrowser.Controller.MediaEncoding
                 && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode
                 && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream)
             {
-                if (state.VideoStream != null && state.VideoStream.Width.HasValue)
-                {
-                    // This is hacky but not sure how to get the exact subtitle resolution
-                    int height = Convert.ToInt32(state.VideoStream.Width.Value / 16.0 * 9.0);
-
-                    arg.Append(" -canvas_size ")
-                        .Append(state.VideoStream.Width.Value.ToString(CultureInfo.InvariantCulture))
-                        .Append(':')
-                        .Append(height.ToString(CultureInfo.InvariantCulture));
-                }
-
                 var subtitlePath = state.SubtitleStream.Path;
 
                 if (string.Equals(Path.GetExtension(subtitlePath), ".sub", StringComparison.OrdinalIgnoreCase))
@@ -1546,9 +1532,12 @@ namespace MediaBrowser.Controller.MediaEncoding
         }
 
         /// <summary>
-        /// Gets the internal graphical subtitle param.
+        /// Gets the graphical subtitle param.
         /// </summary>
-        public string GetGraphicalSubtitleParam(EncodingJobInfo state, EncodingOptions options, string outputVideoCodec)
+        public string GetGraphicalSubtitleParam(
+            EncodingJobInfo state,
+            EncodingOptions options,
+            string outputVideoCodec)
         {
             var outputSizeParam = string.Empty;
 
@@ -1562,53 +1551,77 @@ namespace MediaBrowser.Controller.MediaEncoding
             {
                 outputSizeParam = GetOutputSizeParam(state, options, outputVideoCodec).TrimEnd('"');
 
-                if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
+                var index = outputSizeParam.IndexOf("hwdownload", StringComparison.OrdinalIgnoreCase);
+                if (index != -1)
                 {
-                    var index = outputSizeParam.IndexOf("format", StringComparison.OrdinalIgnoreCase);
-                    if (index != -1)
-                    {
-                        outputSizeParam = "," + outputSizeParam.Substring(index);
-                    }
+                    outputSizeParam = "," + outputSizeParam.Substring(index);
                 }
                 else
                 {
-                    var index = outputSizeParam.IndexOf("scale", StringComparison.OrdinalIgnoreCase);
+                    index = outputSizeParam.IndexOf("format", StringComparison.OrdinalIgnoreCase);
                     if (index != -1)
                     {
                         outputSizeParam = "," + outputSizeParam.Substring(index);
                     }
-                }
-            }
-
-            if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase)
-                && outputSizeParam.Length == 0)
-            {
-                outputSizeParam = ",format=nv12|vaapi,hwupload";
-
-                // Add parameters to use VAAPI with burn-in subttiles (GH issue #642)
-                if (state.SubtitleStream != null
-                    && state.SubtitleStream.IsTextSubtitleStream
-                    && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode) {
-                    outputSizeParam += ",hwmap=mode=read+write+direct";
+                    else
+                    {
+                        index = outputSizeParam.IndexOf("yadif", StringComparison.OrdinalIgnoreCase);
+                        if (index != -1)
+                        {
+                            outputSizeParam = "," + outputSizeParam.Substring(index);
+                        }
+                        else
+                        {
+                            index = outputSizeParam.IndexOf("scale", StringComparison.OrdinalIgnoreCase);
+                            if (index != -1)
+                            {
+                                outputSizeParam = "," + outputSizeParam.Substring(index);
+                            }
+                        }
+                    }
                 }
             }
 
             var videoSizeParam = string.Empty;
+            var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, options);
 
             // Setup subtitle scaling
             if (state.VideoStream != null && state.VideoStream.Width.HasValue && state.VideoStream.Height.HasValue)
             {
+                // force_original_aspect_ratio=decrease
+                // Enable decreasing output video width or height if necessary to keep the original aspect ratio
                 videoSizeParam = string.Format(
                     CultureInfo.InvariantCulture,
-                    "scale={0}:{1}",
+                    "scale={0}:{1}:force_original_aspect_ratio=decrease",
                     state.VideoStream.Width.Value,
                     state.VideoStream.Height.Value);
 
-                //For QSV, feed it into hardware encoder now
+                // For QSV, feed it into hardware encoder now
                 if (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase))
                 {
                     videoSizeParam += ",hwupload=extra_hw_frames=64";
                 }
+
+                // For VAAPI and CUVID decoder
+                // these encoders cannot automatically adjust the size of graphical subtitles to fit the output video,
+                // thus needs to be manually adjusted.
+                if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)
+                    || (videoDecoder ?? string.Empty).IndexOf("cuvid", StringComparison.OrdinalIgnoreCase) != -1)
+                {
+                    var videoStream = state.VideoStream;
+                    var inputWidth = videoStream?.Width;
+                    var inputHeight = videoStream?.Height;
+                    var (width, height) = GetFixedOutputSize(inputWidth, inputHeight, request.Width, request.Height, request.MaxWidth, request.MaxHeight);
+
+                    if (width.HasValue && height.HasValue)
+                    {
+                        videoSizeParam = string.Format(
+                        CultureInfo.InvariantCulture,
+                        "scale={0}:{1}:force_original_aspect_ratio=decrease",
+                        width.Value,
+                        height.Value);
+                    }
+                }
             }
 
             var mapPrefix = state.SubtitleStream.IsExternal ?
@@ -1619,12 +1632,34 @@ namespace MediaBrowser.Controller.MediaEncoding
                 ? 0
                 : state.SubtitleStream.Index;
 
-            var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, options);
-
             // Setup default filtergraph utilizing FFMpeg overlay() and FFMpeg scale() (see the return of this function for index reference)
             var retStr = " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay{3}\"";
 
-            if (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase))
+            // When the input may or may not be hardware VAAPI decodable
+            if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && options.EnableHardwareEncoding)
+            {
+                /*
+                    [base]: HW scaling video to OutputSize
+                    [sub]: SW scaling subtitle to FixedOutputSize
+                    [base][sub]: SW overlay
+                */
+                outputSizeParam = outputSizeParam.TrimStart(',');
+                retStr = " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3},hwdownload[base];[base][sub]overlay,format=nv12,hwupload\"";
+            }
+
+            // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first
+            else if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && !options.EnableHardwareEncoding)
+            {
+                /*
+                    [base]: SW scaling video to OutputSize
+                    [sub]: SW scaling subtitle to FixedOutputSize
+                    [base][sub]: SW overlay
+                */
+                outputSizeParam = outputSizeParam.TrimStart(',');
+                retStr = " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay\"";
+            }
+
+            else if (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase))
             {
                 /*
                     QSV in FFMpeg can now setup hardware overlay for transcodes.
@@ -1688,7 +1723,8 @@ namespace MediaBrowser.Controller.MediaEncoding
             return (Convert.ToInt32(outputWidth), Convert.ToInt32(outputHeight));
         }
 
-        public List<string> GetScalingFilters(int? videoWidth,
+        public List<string> GetScalingFilters(EncodingJobInfo state,
+            int? videoWidth,
             int? videoHeight,
             Video3DFormat? threedFormat,
             string videoDecoder,
@@ -1707,7 +1743,9 @@ namespace MediaBrowser.Controller.MediaEncoding
                 requestedMaxWidth,
                 requestedMaxHeight);
 
-            if ((string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) || string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase))
+            var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
+
+            if (string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) || (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) && !hasTextSubs)
                 && width.HasValue
                 && height.HasValue)
             {
@@ -1737,7 +1775,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                     filters.Add(string.Format(CultureInfo.InvariantCulture, "scale_{0}=format=nv12", vaapi_or_qsv));
                 }
             }
-            else if ((videoDecoder ?? string.Empty).IndexOf("_cuvid", StringComparison.OrdinalIgnoreCase) != -1
+            else if ((videoDecoder ?? string.Empty).IndexOf("cuvid", StringComparison.OrdinalIgnoreCase) != -1
                 && width.HasValue
                 && height.HasValue)
             {
@@ -1941,8 +1979,7 @@ namespace MediaBrowser.Controller.MediaEncoding
         public string GetOutputSizeParam(
             EncodingJobInfo state,
             EncodingOptions options,
-            string outputVideoCodec,
-            bool allowTimeStampCopy = true)
+            string outputVideoCodec)
         {
             // http://sonnati.wordpress.com/2012/10/19/ffmpeg-the-swiss-army-knife-of-internet-streaming-part-vi/
 
@@ -1951,42 +1988,56 @@ namespace MediaBrowser.Controller.MediaEncoding
             var videoStream = state.VideoStream;
             var filters = new List<string>();
 
+            var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, options);
+            var inputWidth = videoStream?.Width;
+            var inputHeight = videoStream?.Height;
+            var threeDFormat = state.MediaSource.Video3DFormat;
+
+            var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
+
+            // When the input may or may not be hardware VAAPI decodable
+            if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && options.EnableHardwareEncoding)
+            {
+                filters.Add("format=nv12|vaapi");
+                filters.Add("hwupload");
+            }
+
+            // When the input may or may not be hardware QSV decodable            
+            else if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) && options.EnableHardwareEncoding)
+            {
+                if (!hasTextSubs)
+                {
+                    filters.Add("format=nv12|qsv");
+                    filters.Add("hwupload=extra_hw_frames=64");
+                }
+            }
+
             // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first
-            var hwType = options.HardwareAccelerationType ?? string.Empty;
-            if (string.Equals(hwType, "vaapi", StringComparison.OrdinalIgnoreCase) && !options.EnableHardwareEncoding )
+            else if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && !options.EnableHardwareEncoding)
             {
-                filters.Add("hwdownload");
+                var codec = videoStream.Codec.ToLowerInvariant();
+                var pixelFormat = videoStream.PixelFormat.ToLowerInvariant();
 
-                // If transcoding from 10 bit, transform colour spaces too
-                if (!string.IsNullOrEmpty(videoStream.PixelFormat)
-                    && videoStream.PixelFormat.IndexOf("p10", StringComparison.OrdinalIgnoreCase) != -1
-                    && string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase))
+                // Assert 10-bit hardware VAAPI decodable
+                if ((pixelFormat ?? string.Empty).IndexOf("p10", StringComparison.OrdinalIgnoreCase) != -1
+                    && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)
+                        || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase)
+                        || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase)))
                 {
+                    filters.Add("hwdownload");
                     filters.Add("format=p010le");
                     filters.Add("format=nv12");
                 }
-                else
+
+                // Assert 8-bit hardware VAAPI decodable
+                else if ((pixelFormat ?? string.Empty).IndexOf("p10", StringComparison.OrdinalIgnoreCase) == -1)
                 {
+                    filters.Add("hwdownload");
                     filters.Add("format=nv12");
                 }
             }
 
-            if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
-            {
-                filters.Add("format=nv12|vaapi");
-                filters.Add("hwupload");
-            }
-
-            var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, options);
-
-            // If we are software decoding, and hardware encoding
-            if (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase)
-                && (string.IsNullOrEmpty(videoDecoder) || !videoDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase)))
-            {
-                filters.Add("format=nv12|qsv");
-                filters.Add("hwupload=extra_hw_frames=64");
-            }
-
+            // Add hardware deinterlace filter before scaling filter
             if (state.DeInterlace("h264", true))
             {
                 if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
@@ -1995,12 +2046,18 @@ namespace MediaBrowser.Controller.MediaEncoding
                 }
                 else if (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase))
                 {
-                    filters.Add(string.Format(CultureInfo.InvariantCulture, "deinterlace_qsv"));
+                    if (!hasTextSubs)
+                    {
+                        filters.Add(string.Format(CultureInfo.InvariantCulture, "deinterlace_qsv"));
+                    }
                 }
             }
 
-            if ((state.DeInterlace("h264", true) || state.DeInterlace("h265", true) || state.DeInterlace("hevc", true))
-                && !string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
+            // Add software deinterlace filter before scaling filter
+            if (((state.DeInterlace("h264", true) || state.DeInterlace("h265", true) || state.DeInterlace("hevc", true))
+                && !string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase)
+                && !string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase))
+                    || (hasTextSubs && state.DeInterlace("h264", true) && string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase)))
             {
                 var inputFramerate = videoStream?.RealFrameRate;
 
@@ -2015,11 +2072,21 @@ namespace MediaBrowser.Controller.MediaEncoding
                 }
             }
 
-            var inputWidth = videoStream?.Width;
-            var inputHeight = videoStream?.Height;
-            var threeDFormat = state.MediaSource.Video3DFormat;
+            // Add scaling filter: scale_*=format=nv12 or scale_*=w=*:h=*:format=nv12 or scale=expr
+            filters.AddRange(GetScalingFilters(state, inputWidth, inputHeight, threeDFormat, videoDecoder, outputVideoCodec, request.Width, request.Height, request.MaxWidth, request.MaxHeight));
 
-            filters.AddRange(GetScalingFilters(inputWidth, inputHeight, threeDFormat, videoDecoder, outputVideoCodec, request.Width, request.Height, request.MaxWidth, request.MaxHeight));
+            // Add parameters to use VAAPI with burn-in text subttiles (GH issue #642)
+            if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && options.EnableHardwareEncoding)
+            {
+                if (state.SubtitleStream != null
+                    && state.SubtitleStream.IsTextSubtitleStream
+                    && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode)
+                {
+                    // Test passed on Intel and AMD gfx
+                    filters.Add("hwmap=mode=read+write");
+                    filters.Add("format=nv12");
+                }
+            }
 
             var output = string.Empty;
 
@@ -2037,11 +2104,6 @@ namespace MediaBrowser.Controller.MediaEncoding
                 {
                     filters.Add("hwmap");
                 }
-
-                if (allowTimeStampCopy)
-                {
-                    output += " -copyts";
-                }
             }
 
             if (filters.Count > 0)
@@ -2218,7 +2280,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             {
                 inputModifier += " " + videoDecoder;
 
-                if ((videoDecoder ?? string.Empty).IndexOf("_cuvid", StringComparison.OrdinalIgnoreCase) != -1)
+                if ((videoDecoder ?? string.Empty).IndexOf("cuvid", StringComparison.OrdinalIgnoreCase) != -1)
                 {
                     var videoStream = state.VideoStream;
                     var inputWidth = videoStream?.Width;
@@ -2227,7 +2289,7 @@ namespace MediaBrowser.Controller.MediaEncoding
 
                     var (width, height) = GetFixedOutputSize(inputWidth, inputHeight, request.Width, request.Height, request.MaxWidth, request.MaxHeight);
 
-                    if ((videoDecoder ?? string.Empty).IndexOf("_cuvid", StringComparison.OrdinalIgnoreCase) != -1
+                    if ((videoDecoder ?? string.Empty).IndexOf("cuvid", StringComparison.OrdinalIgnoreCase) != -1
                         && width.HasValue
                         && height.HasValue)
                     {
@@ -2525,6 +2587,12 @@ namespace MediaBrowser.Controller.MediaEncoding
                         case "h264":
                             if (_mediaEncoder.SupportsDecoder("h264_cuvid") && encodingOptions.HardwareDecodingCodecs.Contains("h264", StringComparer.OrdinalIgnoreCase))
                             {
+                                // cuvid decoder does not support 10-bit input
+                                if ((videoStream.BitDepth ?? 8) > 8)
+                                {
+                                    encodingOptions.HardwareDecodingCodecs = Array.Empty<string>();
+                                    return null;
+                                }
                                 return "-c:v h264_cuvid ";
                             }
                             break;
@@ -2772,14 +2840,27 @@ namespace MediaBrowser.Controller.MediaEncoding
                 var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
 
                 var hasCopyTs = false;
+
                 // Add resolution params, if specified
                 if (!hasGraphicalSubs)
                 {
                     var outputSizeParam = GetOutputSizeParam(state, encodingOptions, videoCodec);
+
                     args += outputSizeParam;
+
                     hasCopyTs = outputSizeParam.IndexOf("copyts", StringComparison.OrdinalIgnoreCase) != -1;
                 }
 
+                // This is for graphical subs
+                if (hasGraphicalSubs)
+                {
+                    var graphicalSubtitleParam = GetGraphicalSubtitleParam(state, encodingOptions, videoCodec);
+
+                    args += graphicalSubtitleParam;
+
+                    hasCopyTs = graphicalSubtitleParam.IndexOf("copyts", StringComparison.OrdinalIgnoreCase) != -1;
+                }
+
                 if (state.RunTimeTicks.HasValue && state.BaseRequest.CopyTimestamps)
                 {
                     if (!hasCopyTs)
@@ -2787,13 +2868,12 @@ namespace MediaBrowser.Controller.MediaEncoding
                         args += " -copyts";
                     }
 
-                    args += " -avoid_negative_ts disabled -start_at_zero";
-                }
+                    args += " -avoid_negative_ts disabled";
 
-                // This is for internal graphical subs
-                if (hasGraphicalSubs)
-                {
-                    args += GetGraphicalSubtitleParam(state, encodingOptions, videoCodec);
+                    if (!(state.SubtitleStream != null && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream))
+                    {
+                        args += " -start_at_zero";
+                    }
                 }
 
                 var qualityParam = GetVideoQualityParam(state, videoCodec, encodingOptions, defaultPreset);
@@ -2899,6 +2979,5 @@ namespace MediaBrowser.Controller.MediaEncoding
                 string.Empty,
                 string.Empty).Trim();
         }
-
     }
 }

MediaBrowser.MediaEncoding/Subtitles/SubtitleEncoder.cs

@@ -731,6 +731,14 @@ namespace MediaBrowser.MediaEncoding.Subtitles
             {
                 var charset = CharsetDetector.DetectFromStream(stream).Detected?.EncodingName;
 
+                // UTF16 is automatically converted to UTF8 by FFmpeg, do not specify a character encoding
+                if ((path.EndsWith(".ass") || path.EndsWith(".ssa"))
+                    && (string.Equals(charset, "utf-16le", StringComparison.OrdinalIgnoreCase)
+                        || string.Equals(charset, "utf-16be", StringComparison.OrdinalIgnoreCase)))
+                {
+                    charset = "";
+                }
+
                 _logger.LogDebug("charset {0} detected for {Path}", charset ?? "null", path);
 
                 return charset;