Merge pull request #2821 from nyanmisaka/mpeg4

Fix MPEG4 broken on VAAPI

(cherry picked from commit 84dba64644a1dda4c3c8dcb6fbb4dc02109ee12d)
Signed-off-by: Joshua M. Boniface <joshua@boniface.me>

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -78,8 +78,7 @@ namespace MediaBrowser.Controller.MediaEncoding
 
                 if (!string.IsNullOrEmpty(hwType)
                     && encodingOptions.EnableHardwareEncoding
-                    && codecMap.ContainsKey(hwType)
-                    && CheckVaapi(state, hwType, encodingOptions))
+                    && codecMap.ContainsKey(hwType))
                 {
                     var preferredEncoder = codecMap[hwType];
 
@@ -93,23 +92,6 @@ namespace MediaBrowser.Controller.MediaEncoding
             return defaultEncoder;
         }
 
-        private bool CheckVaapi(EncodingJobInfo state, string hwType, EncodingOptions encodingOptions)
-        {
-            if (!string.Equals(hwType, "vaapi", StringComparison.OrdinalIgnoreCase))
-            {
-                // No vaapi requested, return OK.
-                return true;
-            }
-
-            if (string.IsNullOrEmpty(encodingOptions.VaapiDevice))
-            {
-                // No device specified, return OK.
-                return true;
-            }
-
-            return IsVaapiSupported(state);
-        }
-
         private bool IsVaapiSupported(EncodingJobInfo state)
         {
             var videoStream = state.VideoStream;
@@ -1605,7 +1587,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                 // For VAAPI and CUVID decoder
                 // these encoders cannot automatically adjust the size of graphical subtitles to fit the output video,
                 // thus needs to be manually adjusted.
-                if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)
+                if ((IsVaapiSupported(state) && string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase))
                     || (videoDecoder ?? string.Empty).IndexOf("cuvid", StringComparison.OrdinalIgnoreCase) != -1)
                 {
                     var videoStream = state.VideoStream;
@@ -1648,7 +1630,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             }
 
             // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first
-            else if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)
+            else if (IsVaapiSupported(state) && string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)
                 && string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase))
             {
                 /*
@@ -2014,19 +1996,29 @@ namespace MediaBrowser.Controller.MediaEncoding
             }
 
             // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first
-            else if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)
+            else if (IsVaapiSupported(state) && string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)
                 && string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase))
             {
                 var codec = videoStream.Codec.ToLowerInvariant();
-                var pixelFormat = videoStream.PixelFormat.ToLowerInvariant();
-
-                // Assert hardware VAAPI decodable (Except h264 10-bit and higher color depth)
-				// TODO: a propery way to detect hardware capabilities and falling back when transcoding is failed
-                if ((pixelFormat ?? string.Empty).IndexOf("p10", StringComparison.OrdinalIgnoreCase) == -1
-                    || ((pixelFormat ?? string.Empty).IndexOf("p10", StringComparison.OrdinalIgnoreCase) != -1
-                        && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)
-                            || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase)
-                            || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase))))
+                var isColorDepth10 = !string.IsNullOrEmpty(videoStream.Profile) && (videoStream.Profile.Contains("Main 10", StringComparison.OrdinalIgnoreCase)
+                    || videoStream.Profile.Contains("High 10", StringComparison.OrdinalIgnoreCase));
+
+                // Assert 10-bit hardware VAAPI decodable
+                if (isColorDepth10 && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase)))
+                {
+                    /*
+                        Download data from GPU to CPU as p010le format.
+                        Colorspace conversion is unnecessary here as libx264 will handle it.
+                        If this step is missing, it will fail on AMD but not on intel.
+                    */
+                    filters.Add("hwdownload");
+                    filters.Add("format=p010le");
+                }
+
+                // Assert 8-bit hardware VAAPI decodable
+                else if (!isColorDepth10)
                 {
                     filters.Add("hwdownload");
                     filters.Add("format=nv12");