Enable software tonemap for dolby vision

This applies software tonemapx filter for dolby vision videos that have no compatability fallback.

Due to the complexity of the reshaping process, this is quite CPU-intensive. For real-time transcoding and tonemapping of 4K 60fps content, a CPU with 16 cores of Zen3-level performance is recommended.

Signed-off-by: gnattu <gnattuoc@me.com>

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -296,14 +296,12 @@ namespace MediaBrowser.Controller.MediaEncoding
             if (state.VideoStream is null
                 || !options.EnableTonemapping
                 || GetVideoColorBitDepth(state) != 10
-                || !_mediaEncoder.SupportsFilter("tonemapx")
-                || !(string.Equals(state.VideoStream?.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase) || string.Equals(state.VideoStream?.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase)))
+                || !_mediaEncoder.SupportsFilter("tonemapx"))
             {
                 return false;
             }
 
-            return state.VideoStream.VideoRange == VideoRange.HDR
-                   && state.VideoStream.VideoRangeType is VideoRangeType.HDR10 or VideoRangeType.HLG or VideoRangeType.DOVIWithHDR10 or VideoRangeType.DOVIWithHLG;
+            return state.VideoStream.VideoRange == VideoRange.HDR;
         }
 
         private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
@@ -3435,6 +3433,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
             var doDeintH2645 = doDeintH264 || doDeintHevc;
             var doToneMap = IsSwTonemapAvailable(state, options);
+            var requireDoviReshaping = doToneMap && state.VideoStream.VideoRangeType == VideoRangeType.DOVI;
 
             var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
             var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream;
@@ -3472,11 +3471,13 @@ namespace MediaBrowser.Controller.MediaEncoding
             // sw scale
             mainFilters.Add(swScaleFilter);
 
-            // sw tonemap <= TODO: finish dovi tone mapping
-
+            // sw tonemap
             if (doToneMap)
             {
-                var tonemapArgs = $"tonemapx=tonemap={options.TonemappingAlgorithm}:desat={options.TonemappingDesat}:peak={options.TonemappingPeak}:t=bt709:m=bt709:p=bt709:format={outFormat}";
+                // tonemapx requires yuv420p10 input for dovi reshaping, let ffmpeg convert the frame when necessary
+                var tonemapFormat = requireDoviReshaping ? "yuv420p" : outFormat;
+
+                var tonemapArgs = $"tonemapx=tonemap={options.TonemappingAlgorithm}:desat={options.TonemappingDesat}:peak={options.TonemappingPeak}:t=bt709:m=bt709:p=bt709:format={tonemapFormat}";
 
                 if (options.TonemappingParam != 0)
                 {
@@ -3490,6 +3491,13 @@ namespace MediaBrowser.Controller.MediaEncoding
                 }
 
                 mainFilters.Add(tonemapArgs);
+
+                // convert back to nv12 for VAAPI encoders
+                // Q: Is this still being used?
+                if (requireDoviReshaping && isVaapiEncoder)
+                {
+                    mainFilters.Add("format=" + outFormat);
+                }
             }
             else
             {

MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs

@@ -12,6 +12,7 @@ using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using AsyncKeyedLock;
+using Jellyfin.Data.Enums;
 using Jellyfin.Extensions;
 using Jellyfin.Extensions.Json;
 using Jellyfin.Extensions.Json.Converters;
@@ -665,18 +666,18 @@ namespace MediaBrowser.MediaEncoding.Encoder
                 filters.Add("thumbnail=n=" + (useLargerBatchSize ? "50" : "24"));
             }
 
-            // Use SW tonemap on HDR10/HLG video stream only when the zscale or tonemapx filter is available.
+            // Use SW tonemap on HDR video stream only when the zscale or tonemapx filter is available.
+            // Only enable Dolby Vision tonemap when tonemapx is available
             var enableHdrExtraction = false;
 
-            if (string.Equals(videoStream?.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase)
-                || string.Equals(videoStream?.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase))
+            if (videoStream?.VideoRange == VideoRange.HDR)
             {
                 if (SupportsFilter("tonemapx"))
                 {
                     enableHdrExtraction = true;
                     filters.Add("tonemapx=tonemap=bt2390:desat=0:peak=100:t=bt709:m=bt709:p=bt709:format=yuv420p");
                 }
-                else if (SupportsFilter("zscale"))
+                else if (SupportsFilter("zscale") && videoStream.VideoRangeType != VideoRangeType.DOVI)
                 {
                     enableHdrExtraction = true;
                     filters.Add("zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable:desat=0:peak=100,zscale=t=bt709:m=bt709,format=yuv420p");