Backport pull request #15819 from jellyfin/release-10.11.z

Fix the use of HWA in unsupported H.264 Hi422P/Hi444PP

Original-merge: 4c587776d6263698bd0e00b56c06f14d46c4c2ec

Merged-by: crobibero <cody@robibe.ro>

Backported-by: Bond_009 <bond.009@outlook.com>

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -6359,6 +6359,21 @@ namespace MediaBrowser.Controller.MediaEncoding
                     }
                 }
 
+                // Block unsupported H.264 Hi422P and Hi444PP profiles, which can be encoded with 4:2:0 pixel format
+                if (string.Equals(videoStream.Codec, "h264", StringComparison.OrdinalIgnoreCase))
+                {
+                    if (videoStream.Profile.Contains("4:2:2", StringComparison.OrdinalIgnoreCase)
+                        || videoStream.Profile.Contains("4:4:4", StringComparison.OrdinalIgnoreCase))
+                    {
+                        // VideoToolbox on Apple Silicon has H.264 Hi444PP and theoretically also has Hi422P
+                        if (!(hardwareAccelerationType == HardwareAccelerationType.videotoolbox
+                              && RuntimeInformation.OSArchitecture.Equals(Architecture.Arm64)))
+                        {
+                            return null;
+                        }
+                    }
+                }
+
                 var decoder = hardwareAccelerationType switch
                 {
                     HardwareAccelerationType.vaapi => GetVaapiVidDecoder(state, options, videoStream, bitDepth),