add tonemapping for intel vaapi

hwdec->scale->tonemap->hwenc

hwdec->scale->tonemap->textsubs->hwenc

* grapical subs requires overlay_vaapi, but it's still in ffmpeg mailing list.

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -112,6 +112,13 @@ namespace MediaBrowser.Controller.MediaEncoding
             return _mediaEncoder.SupportsHwaccel("vaapi");
         }
 
+        private bool IsTonemappingSupported(EncodingJobInfo state, EncodingOptions options)
+        {
+            var videoStream = state.VideoStream;
+            var isColorDepth10 = IsColorDepth10(state);
+            return isColorDepth10 && _mediaEncoder.SupportsHwaccel("opencl") && options.EnableTonemapping && !string.IsNullOrEmpty(videoStream.VideoRange) && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase);
+        }
+
         /// <summary>
         /// Gets the name of the output video codec.
         /// </summary>
@@ -468,6 +475,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
             var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
             var isMacOS = RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
+            var isTonemappingSupported = IsTonemappingSupported(state, encodingOptions);
 
             if (!IsCopyCodec(outputVideoCodec))
             {
@@ -477,10 +485,24 @@ namespace MediaBrowser.Controller.MediaEncoding
                 {
                     if (isVaapiDecoder)
                     {
-                        arg.Append("-hwaccel_output_format vaapi ")
-                            .Append("-vaapi_device ")
-                            .Append(encodingOptions.VaapiDevice)
-                            .Append(' ');
+                        if (isTonemappingSupported)
+                        {
+                           arg.Append("-init_hw_device vaapi=va:")
+                                .Append(encodingOptions.VaapiDevice)
+                                .Append(' ')
+                                .Append("-init_hw_device opencl=ocl@va ")
+                                .Append("-hwaccel vaapi ")
+                                .Append("-hwaccel_device va ")
+                                .Append("-hwaccel_output_format vaapi ")
+                                .Append("-filter_hw_device ocl ");
+                        }
+                        else
+                        {
+                            arg.Append("-hwaccel_output_format vaapi ")
+                                .Append("-vaapi_device ")
+                                .Append(encodingOptions.VaapiDevice)
+                                .Append(' ');
+                        }
                     }
                     else if (!isVaapiDecoder && isVaapiEncoder)
                     {
@@ -529,13 +551,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                     && (string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase) && isNvdecHevcDecoder || isSwDecoder)
                         || (string.Equals(encodingOptions.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase) && isD3d11vaDecoder || isSwDecoder))
                 {
-                    var isColorDepth10 = IsColorDepth10(state);
-
-                    if (isColorDepth10
-                        && _mediaEncoder.SupportsHwaccel("opencl")
-                        && encodingOptions.EnableTonemapping
-                        && !string.IsNullOrEmpty(state.VideoStream.VideoRange)
-                        && state.VideoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase))
+                    if (isTonemappingSupported)
                     {
                         arg.Append("-init_hw_device opencl=ocl:")
                             .Append(encodingOptions.OpenclDevice)
@@ -1997,6 +2013,7 @@ namespace MediaBrowser.Controller.MediaEncoding
 
         public List<string> GetScalingFilters(
             EncodingJobInfo state,
+            EncodingOptions options,
             int? videoWidth,
             int? videoHeight,
             Video3DFormat? threedFormat,
@@ -2035,6 +2052,19 @@ namespace MediaBrowser.Controller.MediaEncoding
                     || state.DeInterlace("h265", true)
                     || state.DeInterlace("hevc", true);
 
+                var isTonemappingSupported = IsTonemappingSupported(state, options);
+                var isTonemappingSupportedOnVaapi = string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && !qsv_or_vaapi;
+
+                var outputPixFmt = string.Empty;
+                if (isTonemappingSupported && isTonemappingSupportedOnVaapi)
+                {
+                    outputPixFmt = "format=p010:out_range=limited";
+                }
+                else
+                {
+                    outputPixFmt = "format=nv12";
+                }
+
                 if (!videoWidth.HasValue
                     || outputWidth != videoWidth.Value
                     || !videoHeight.HasValue
@@ -2045,10 +2075,11 @@ namespace MediaBrowser.Controller.MediaEncoding
                     filters.Add(
                         string.Format(
                             CultureInfo.InvariantCulture,
-                            "{0}=w={1}:h={2}:format=nv12{3}",
+                            "{0}=w={1}:h={2}{3}{4}",
                             qsv_or_vaapi ? "vpp_qsv" : "scale_vaapi",
                             outputWidth,
                             outputHeight,
+                            ":" + outputPixFmt,
                             (qsv_or_vaapi && isDeintEnabled) ? ":deinterlace=1" : string.Empty));
                 }
                 else
@@ -2056,8 +2087,9 @@ namespace MediaBrowser.Controller.MediaEncoding
                     filters.Add(
                         string.Format(
                             CultureInfo.InvariantCulture,
-                            "{0}=format=nv12{1}",
+                            "{0}={1}{2}",
                             qsv_or_vaapi ? "vpp_qsv" : "scale_vaapi",
+                            outputPixFmt,
                             (qsv_or_vaapi && isDeintEnabled) ? ":deinterlace=1" : string.Empty));
                 }
             }
@@ -2290,6 +2322,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             var isSwDecoder = string.IsNullOrEmpty(videoDecoder);
             var isD3d11vaDecoder = videoDecoder.IndexOf("d3d11va", StringComparison.OrdinalIgnoreCase) != -1;
             var isVaapiDecoder = videoDecoder.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1;
+            var isVaapiEncoder = outputVideoCodec.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1;
             var isVaapiH264Encoder = outputVideoCodec.IndexOf("h264_vaapi", StringComparison.OrdinalIgnoreCase) != -1;
             var isVaapiHevcEncoder = outputVideoCodec.IndexOf("hevc_vaapi", StringComparison.OrdinalIgnoreCase) != -1;
             var isQsvH264Encoder = outputVideoCodec.IndexOf("h264_qsv", StringComparison.OrdinalIgnoreCase) != -1;
@@ -2300,6 +2333,10 @@ namespace MediaBrowser.Controller.MediaEncoding
             var isLibX265Encoder = outputVideoCodec.IndexOf("libx265", StringComparison.OrdinalIgnoreCase) != -1;
             var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
             var isColorDepth10 = IsColorDepth10(state);
+            var isTonemappingSupported = IsTonemappingSupported(state, options);
+            var isTonemappingSupportedOnNvenc = string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase) && isNvdecHevcDecoder || isSwDecoder;
+            var isTonemappingSupportedOnAmf = string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase) && isD3d11vaDecoder || isSwDecoder;
+            var isTonemappingSupportedOnVaapi = string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isVaapiH264Encoder || isVaapiHevcEncoder);
 
             var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
             var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
@@ -2311,18 +2348,13 @@ namespace MediaBrowser.Controller.MediaEncoding
             var isDeinterlaceH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true);
             var isDeinterlaceHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
 
-            if ((string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase) && isNvdecHevcDecoder || isSwDecoder)
-                || (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase) && isD3d11vaDecoder || isSwDecoder))
+            if (isTonemappingSupportedOnNvenc || isTonemappingSupportedOnAmf || isTonemappingSupportedOnVaapi)
             {
                 // Currently only with the use of NVENC decoder can we get a decent performance.
                 // Currently only the HEVC/H265 format is supported with NVDEC decoder.
                 // NVIDIA Pascal and Turing or higher are recommended.
                 // AMD Polaris and Vega or higher are recommended.
-                if (isColorDepth10
-                    && _mediaEncoder.SupportsHwaccel("opencl")
-                    && options.EnableTonemapping
-                    && !string.IsNullOrEmpty(videoStream.VideoRange)
-                    && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase))
+                if (isTonemappingSupported)
                 {
                     var parameters = "tonemap_opencl=format=nv12:primaries=bt709:transfer=bt709:matrix=bt709:tonemap={0}:desat={1}:threshold={2}:peak={3}";
 
@@ -2353,12 +2385,32 @@ namespace MediaBrowser.Controller.MediaEncoding
 
                         // Convert to hardware pixel format p010 when using SW decoder.
                         filters.Add("format=p010");
+
+                        // Upload the HDR10 or HLG data to the OpenCL device,
+                        // use tonemap_opencl filter for tone mapping,
+                        // and then download the SDR data to memory.
+                        filters.Add("hwupload");
+                    }
+
+                    if (isVaapiDecoder)
+                    {
+                        isScalingInAdvance = true;
+                        filters.AddRange(
+                            GetScalingFilters(
+                                state,
+                                options,
+                                inputWidth,
+                                inputHeight,
+                                threeDFormat,
+                                videoDecoder,
+                                outputVideoCodec,
+                                request.Width,
+                                request.Height,
+                                request.MaxWidth,
+                                request.MaxHeight));
+                        filters.Add("hwmap");
                     }
 
-                    // Upload the HDR10 or HLG data to the OpenCL device,
-                    // use tonemap_opencl filter for tone mapping,
-                    // and then download the SDR data to memory.
-                    filters.Add("hwupload");
                     filters.Add(
                         string.Format(
                             CultureInfo.InvariantCulture,
@@ -2369,21 +2421,30 @@ namespace MediaBrowser.Controller.MediaEncoding
                             options.TonemappingPeak,
                             options.TonemappingParam,
                             options.TonemappingRange));
-                    filters.Add("hwdownload");
 
-                    if (isLibX264Encoder
-                        || isLibX265Encoder
-                        || hasGraphicalSubs
-                        || (isNvdecHevcDecoder && isDeinterlaceHevc)
-                        || (!isNvdecHevcDecoder && isDeinterlaceH264 || isDeinterlaceHevc))
+                    if (isSwDecoder || isD3d11vaDecoder)
+                    {
+                        filters.Add("hwdownload");
+
+                        if (isLibX264Encoder
+                            || isLibX265Encoder
+                            || hasGraphicalSubs
+                            || (isNvdecHevcDecoder && isDeinterlaceHevc)
+                            || (!isNvdecHevcDecoder && isDeinterlaceH264 || isDeinterlaceHevc))
+                        {
+                            filters.Add("format=nv12");
+                        }
+                    }
+
+                    if (isVaapiDecoder)
                     {
-                        filters.Add("format=nv12");
+                        filters.Add("hwmap=derive_device=vaapi:reverse=1");
                     }
                 }
             }
 
             // When the input may or may not be hardware VAAPI decodable
-            if (isVaapiH264Encoder || isVaapiHevcEncoder)
+            if ((isVaapiH264Encoder || isVaapiHevcEncoder) && !isTonemappingSupported && !isTonemappingSupportedOnVaapi)
             {
                 filters.Add("format=nv12|vaapi");
                 filters.Add("hwupload");
@@ -2467,6 +2528,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                 filters.AddRange(
                     GetScalingFilters(
                         state,
+                        options,
                         inputWidth,
                         inputHeight,
                         threeDFormat,
@@ -2483,6 +2545,13 @@ namespace MediaBrowser.Controller.MediaEncoding
             {
                 if (hasTextSubs)
                 {
+                    // Convert hw context from ocl to va.
+                    // For tonemapping and text subs burn-in.
+                    if (isTonemappingSupported && isTonemappingSupportedOnVaapi)
+                    {
+                        filters.Add("scale_vaapi");
+                    }
+
                     // Test passed on Intel and AMD gfx
                     filters.Add("hwmap=mode=read+write");
                     filters.Add("format=nv12");