fix: code clean up

Co-authored-by: nyanmisaka <nst799610810@gmail.com>
Signed-off-by: gnattu <gnattuoc@me.com>

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -253,7 +253,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                    && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync);
         }
 
-        private bool IsVideoToolBoxFullSupported()
+        private bool IsVideoToolboxFullSupported()
         {
             return _mediaEncoder.SupportsHwaccel("videotoolbox")
                 && _mediaEncoder.SupportsFilter("yadif_videotoolbox")
@@ -4978,100 +4978,118 @@ namespace MediaBrowser.Controller.MediaEncoding
                 return (null, null, null);
             }
 
-            var swFilterChain = GetSwVidFilterChain(state, options, vidEncoder);
+            var isMacOS = OperatingSystem.IsMacOS();
+            var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty;
+            var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
+            var isVtFullSupported = isMacOS && IsVideoToolboxFullSupported();
+            var isVtOclSupported = isVtFullSupported && IsOpenclFullSupported();
 
-            if (!options.EnableHardwareEncoding)
+            // legacy videotoolbox pipeline (disable hw filters)
+            if (!isVtEncoder
+                || !isVtOclSupported
+                || !_mediaEncoder.SupportsFilter("alphasrc"))
             {
-                return swFilterChain;
+                return GetSwVidFilterChain(state, options, vidEncoder);
             }
 
-            var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true);
-            var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
-            var doDeintH2645 = doDeintH264 || doDeintHevc;
+            // preferred videotoolbox + vt/ocl filters pipeline
+            return GetAppleVidFiltersPreferred(state, options, vidDecoder, vidEncoder);
+        }
+
+        public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetAppleVidFiltersPreferred(
+            EncodingJobInfo state,
+            EncodingOptions options,
+            string vidDecoder,
+            string vidEncoder)
+        {
             var inW = state.VideoStream?.Width;
             var inH = state.VideoStream?.Height;
             var reqW = state.BaseRequest.Width;
             var reqH = state.BaseRequest.Height;
             var reqMaxW = state.BaseRequest.MaxWidth;
             var reqMaxH = state.BaseRequest.MaxHeight;
-            var mainFilters = new List<string>();
+            var threeDFormat = state.MediaSource.Video3DFormat;
+
+            var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
+
+            var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true);
+            var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
+            var doDeintH2645 = doDeintH264 || doDeintHevc;
+            var doVtTonemap = IsVideoToolboxTonemapAvailable(state, options);
+            var doOclTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options);
+            var doTonemap = doVtTonemap || doOclTonemap;
+
             var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
             var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream;
             var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream;
             var hasAssSubs = hasSubs
-                             && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
-                                 || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
-            // VideoToolbox is special. It does not use a separate tone mapping filter like others.
-            // Instead, it performs both tone mapping and scaling in a single filter.
-            var useVtToneMapping = IsVideoToolboxTonemapAvailable(state, options);
-            // Use OpenCL tone mapping as a fallback
-            var useOclToneMapping = !useVtToneMapping && IsHwTonemapAvailable(state, options);
-            // Fallback to software filters if we are using filters not supported by hardware yet.
-            // OpenCL won't work without proper VT support as the hwmap interop required will not be present there
-            var useHardwareFilters = IsVideoToolBoxFullSupported();
-
-            if (!useHardwareFilters)
+                && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
+
+            // FIXME: scale_vt lacks of format option for the time being.
+            // hwdownload/hwmap to sw requires setting a format explicitly.
+            if (!isVtEncoder)
             {
-                return swFilterChain;
+                // should not happen.
+                return (null, null, null);
             }
 
-            if (!(useVtToneMapping || useOclToneMapping || hasSubs || doDeintH2645))
+            /* Make main filters for video stream */
+            var mainFilters = new List<string>();
+
+            if (!(doTonemap || hasSubs || doDeintH2645))
             {
                 // Dummy action to return empty filters when nothing to do.
                 return (mainFilters, mainFilters, mainFilters);
             }
 
-            // Override the color when doing OpenCL Tone mapping, where we are using hardware surface output.
-            if (useOclToneMapping)
+            // Color override is only required for OpenCL where hardware surface is in use
+            if (doOclTonemap)
             {
-                mainFilters.Add(GetOverwriteColorPropertiesParam(state, true));
+                mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap));
             }
 
-            // With OpenCL tone mapping, we are using native hwmap and there's no need to specify a format for the main stream.
-            // However, for other cases, we have to specify the format for the main stream when we are doing subtitle burn-in.
-            // This is because the default upload option is not always processable with VideoToolbox.
-            // Most notably, yuv420p should be replaced by nv12.
-            else if (hasSubs)
-            {
-                var is8Bit = string.Equals("yuv420p", state.VideoStream.PixelFormat, StringComparison.OrdinalIgnoreCase)
-                                        || string.Equals("yuvj420p", state.VideoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
-                var is10Bit = string.Equals("yuv420p10le", state.VideoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
+            // INPUT videotoolbox/memory surface(vram/uma)
+            // this will pass-through automatically if in/out format matches.
+            mainFilters.Add("format=nv12|p010le|videotoolbox_vld");
+            mainFilters.Add("hwupload=derive_device=videotoolbox");
 
-                if (is8Bit)
-                {
-                    mainFilters.Add("format=nv12");
-                }
-                else if (is10Bit)
-                {
-                    mainFilters.Add("format=p010");
-                }
+            // hw deint
+            if (doDeintH2645)
+            {
+                var deintFilter = GetHwDeinterlaceFilter(state, options, "videotoolbox");
+                mainFilters.Add(deintFilter);
             }
 
-            mainFilters.Add("hwupload");
             var hwScaleFilter = GetHwScaleFilter("vt", null, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
-            if (useVtToneMapping)
+            if (doVtTonemap)
             {
-                const string TonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709";
+                const string VtTonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709";
+
+                // scale_vt can handle scaling & tonemapping in one shot, just like vpp_qsv.
                 hwScaleFilter = string.IsNullOrEmpty(hwScaleFilter)
-                    ? "scale_vt=" + TonemapArgs
-                    : hwScaleFilter + ":" + TonemapArgs;
+                    ? "scale_vt=" + VtTonemapArgs
+                    : hwScaleFilter + ":" + VtTonemapArgs;
             }
 
+            // hw scale & vt tonemap
             mainFilters.Add(hwScaleFilter);
 
-            if (useOclToneMapping)
+            // ocl tonemap
+            if (doOclTonemap)
             {
-                mainFilters.Add("hwmap=derive_device=opencl");
-                mainFilters.Add(GetHwTonemapFilter(options, "opencl", "nv12"));
-                mainFilters.Add("hwmap=derive_device=videotoolbox:reverse=1");
-            }
+                // map from videotoolbox to opencl via videotoolbox-opencl interop.
+                mainFilters.Add("hwmap=derive_device=opencl:mode=read");
 
-            if (doDeintH2645)
-            {
-                var deintFilter = GetHwDeinterlaceFilter(state, options, "videotoolbox");
-                mainFilters.Add(deintFilter);
+                var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12");
+                mainFilters.Add(tonemapFilter);
+
+                // OUTPUT videotoolbox(nv12) surface(vram/uma)
+                // reverse-mapping via videotoolbox-opencl interop.
+                mainFilters.Add("hwmap=derive_device=videotoolbox:mode=write:reverse=1");
             }
 
+            /* Make sub and overlay filters for subtitle stream */
             var subFilters = new List<string>();
             var overlayFilters = new List<string>();
 
@@ -6094,10 +6112,17 @@ namespace MediaBrowser.Controller.MediaEncoding
             // Hardware surface only make sense when interop with OpenCL
             // VideoToolbox's Hardware surface in ffmpeg is not only slower than hwupload, but also breaks HDR in many cases.
             // For example: https://trac.ffmpeg.org/ticket/10884
-            var useOclToneMapping = !IsVideoToolboxTonemapAvailable(state, options) &&
-                                    options.EnableTonemapping &&
-                                    state.VideoStream.VideoRangeType == VideoRangeType.DOVI;
-            var useHwSurface = useOclToneMapping && IsVideoToolBoxFullSupported() && _mediaEncoder.SupportsFilter("alphasrc");
+            var useOclToneMapping = !IsVideoToolboxTonemapAvailable(state, options)
+                                    && options.EnableTonemapping
+                                    && state.VideoStream is not null
+                                    && GetVideoColorBitDepth(state) == 10
+                                    && state.VideoStream.VideoRange == VideoRange.HDR
+                                    && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10
+                                        || state.VideoStream.VideoRangeType == VideoRangeType.HLG
+                                        || (state.VideoStream.VideoRangeType == VideoRangeType.DOVI
+                                            && string.Equals(state.VideoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase)));
+
+            var useHwSurface = useOclToneMapping && IsVideoToolboxFullSupported() && _mediaEncoder.SupportsFilter("alphasrc");
 
             if (is8bitSwFormatsVt)
             {