fix: use Metal tonemap instead of OpenCL (#11198)

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -259,6 +259,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             return _mediaEncoder.SupportsHwaccel("videotoolbox")
                 && _mediaEncoder.SupportsFilter("yadif_videotoolbox")
                 && _mediaEncoder.SupportsFilter("overlay_videotoolbox")
+                && _mediaEncoder.SupportsFilter("tonemap_videotoolbox")
                 && _mediaEncoder.SupportsFilter("scale_vt");
         }
 
@@ -1376,6 +1377,14 @@ namespace MediaBrowser.Controller.MediaEncoding
                 return FormattableString.Invariant($" -rc_mode VBR -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}");
             }
 
+            if (string.Equals(videoCodec, "h264_videotoolbox", StringComparison.OrdinalIgnoreCase)
+                || string.Equals(videoCodec, "hevc_videotoolbox", StringComparison.OrdinalIgnoreCase))
+            {
+                // The `maxrate` and `bufsize` options can potentially lead to performance regression
+                // and even encoder hangs, especially when the value is very high.
+                return FormattableString.Invariant($" -b:v {bitrate}");
+            }
+
             return FormattableString.Invariant($" -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}");
         }
 
@@ -5020,17 +5029,16 @@ namespace MediaBrowser.Controller.MediaEncoding
             var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty;
             var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
             var isVtFullSupported = isMacOS && IsVideoToolboxFullSupported();
-            var isVtOclSupported = isVtFullSupported && IsOpenclFullSupported();
 
             // legacy videotoolbox pipeline (disable hw filters)
             if (!isVtEncoder
-                || !isVtOclSupported
+                || !isVtFullSupported
                 || !_mediaEncoder.SupportsFilter("alphasrc"))
             {
                 return GetSwVidFilterChain(state, options, vidEncoder);
             }
 
-            // preferred videotoolbox + vt/ocl filters pipeline
+            // preferred videotoolbox + metal filters pipeline
             return GetAppleVidFiltersPreferred(state, options, vidDecoder, vidEncoder);
         }
 
@@ -5054,13 +5062,23 @@ namespace MediaBrowser.Controller.MediaEncoding
             var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
             var doDeintH2645 = doDeintH264 || doDeintHevc;
             var doVtTonemap = IsVideoToolboxTonemapAvailable(state, options);
-            var doOclTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options);
+            var doMetalTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options);
 
             var scaleFormat = string.Empty;
+            // Use P010 for Metal tone mapping, otherwise force an 8bit output.
             if (!string.Equals(state.VideoStream.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase))
             {
-                // Use P010 for OpenCL tone mapping, otherwise force an 8bit output.
-                scaleFormat = doOclTonemap ? "p010le" : "nv12";
+                if (doMetalTonemap)
+                {
+                    if (!string.Equals(state.VideoStream.PixelFormat, "yuv420p10le", StringComparison.OrdinalIgnoreCase))
+                    {
+                        scaleFormat = "p010le";
+                    }
+                }
+                else
+                {
+                    scaleFormat = "nv12";
+                }
             }
 
             var hwScaleFilter = GetHwScaleFilter("vt", scaleFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
@@ -5081,12 +5099,6 @@ namespace MediaBrowser.Controller.MediaEncoding
             /* Make main filters for video stream */
             var mainFilters = new List<string>();
 
-            // Color override is only required for OpenCL where hardware surface is in use
-            if (doOclTonemap)
-            {
-                mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap));
-            }
-
             // INPUT videotoolbox/memory surface(vram/uma)
             // this will pass-through automatically if in/out format matches.
             mainFilters.Add("format=nv12|p010le|videotoolbox_vld");
@@ -5112,18 +5124,11 @@ namespace MediaBrowser.Controller.MediaEncoding
             // hw scale & vt tonemap
             mainFilters.Add(hwScaleFilter);
 
-            // ocl tonemap
-            if (doOclTonemap)
+            // Metal tonemap
+            if (doMetalTonemap)
             {
-                // map from videotoolbox to opencl via videotoolbox-opencl interop.
-                mainFilters.Add("hwmap=derive_device=opencl:mode=read");
-
-                var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12");
+                var tonemapFilter = GetHwTonemapFilter(options, "videotoolbox", "nv12");
                 mainFilters.Add(tonemapFilter);
-
-                // OUTPUT videotoolbox(nv12) surface(vram/uma)
-                // reverse-mapping via videotoolbox-opencl interop.
-                mainFilters.Add("hwmap=derive_device=videotoolbox:mode=write:reverse=1");
             }
 
             /* Make sub and overlay filters for subtitle stream */
@@ -6146,39 +6151,27 @@ namespace MediaBrowser.Controller.MediaEncoding
                                     || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
             var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
 
-            // Hardware surface only make sense when interop with OpenCL
             // VideoToolbox's Hardware surface in ffmpeg is not only slower than hwupload, but also breaks HDR in many cases.
             // For example: https://trac.ffmpeg.org/ticket/10884
-            var useOclToneMapping = !IsVideoToolboxTonemapAvailable(state, options)
-                                    && options.EnableTonemapping
-                                    && state.VideoStream is not null
-                                    && GetVideoColorBitDepth(state) == 10
-                                    && state.VideoStream.VideoRange == VideoRange.HDR
-                                    && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10
-                                        || state.VideoStream.VideoRangeType == VideoRangeType.HLG
-                                        || ((state.VideoStream.VideoRangeType == VideoRangeType.DOVI
-                                            || state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHDR10
-                                            || state.VideoStream.VideoRangeType == VideoRangeType.DOVIWithHLG)
-                                            && string.Equals(state.VideoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase)));
-
-            var useHwSurface = useOclToneMapping && IsVideoToolboxFullSupported() && _mediaEncoder.SupportsFilter("alphasrc");
+            // Disable it for now.
+            const bool UseHwSurface = false;
 
             if (is8bitSwFormatsVt)
             {
                 if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
                     || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "h264", bitDepth, useHwSurface);
+                    return GetHwaccelType(state, options, "h264", bitDepth, UseHwSurface);
                 }
 
                 if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "mpeg2video", bitDepth, useHwSurface);
+                    return GetHwaccelType(state, options, "mpeg2video", bitDepth, UseHwSurface);
                 }
 
                 if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "mpeg4", bitDepth, useHwSurface);
+                    return GetHwaccelType(state, options, "mpeg4", bitDepth, UseHwSurface);
                 }
             }
 
@@ -6187,12 +6180,12 @@ namespace MediaBrowser.Controller.MediaEncoding
                 if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
                     || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "hevc", bitDepth, useHwSurface);
+                    return GetHwaccelType(state, options, "hevc", bitDepth, UseHwSurface);
                 }
 
                 if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
                 {
-                    return GetHwaccelType(state, options, "vp9", bitDepth, useHwSurface);
+                    return GetHwaccelType(state, options, "vp9", bitDepth, UseHwSurface);
                 }
             }
 

MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs

@@ -130,6 +130,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             "yadif_videotoolbox",
             "scale_vt",
             "overlay_videotoolbox",
+            "tonemap_videotoolbox",
             // rkrga
             "scale_rkrga",
             "vpp_rkrga",