Merge pull request #2785 from nyanmisaka/mpge4-profile15

Fix MPEG4 packback error regression on vaapi

(cherry picked from commit b16b58bc570b27b709dec7251f677c927c84c925)
Signed-off-by: Joshua M. Boniface <joshua@boniface.me>

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -1636,7 +1636,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             var retStr = " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay{3}\"";
 
             // When the input may or may not be hardware VAAPI decodable
-            if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && options.EnableHardwareEncoding)
+            if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
             {
                 /*
                     [base]: HW scaling video to OutputSize
@@ -1648,7 +1648,8 @@ namespace MediaBrowser.Controller.MediaEncoding
             }
 
             // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first
-            else if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && !options.EnableHardwareEncoding)
+            else if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)
+                && string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase))
             {
                 /*
                     [base]: SW scaling video to OutputSize
@@ -1996,14 +1997,14 @@ namespace MediaBrowser.Controller.MediaEncoding
             var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
 
             // When the input may or may not be hardware VAAPI decodable
-            if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && options.EnableHardwareEncoding)
+            if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
             {
                 filters.Add("format=nv12|vaapi");
                 filters.Add("hwupload");
             }
 
             // When the input may or may not be hardware QSV decodable            
-            else if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) && options.EnableHardwareEncoding)
+            else if (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase))
             {
                 if (!hasTextSubs)
                 {
@@ -2013,24 +2014,19 @@ namespace MediaBrowser.Controller.MediaEncoding
             }
 
             // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first
-            else if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && !options.EnableHardwareEncoding)
+            else if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)
+                && string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase))
             {
                 var codec = videoStream.Codec.ToLowerInvariant();
                 var pixelFormat = videoStream.PixelFormat.ToLowerInvariant();
 
-                // Assert 10-bit hardware VAAPI decodable
-                if ((pixelFormat ?? string.Empty).IndexOf("p10", StringComparison.OrdinalIgnoreCase) != -1
-                    && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)
-                        || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase)
-                        || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase)))
-                {
-                    filters.Add("hwdownload");
-                    filters.Add("format=p010le");
-                    filters.Add("format=nv12");
-                }
-
-                // Assert 8-bit hardware VAAPI decodable
-                else if ((pixelFormat ?? string.Empty).IndexOf("p10", StringComparison.OrdinalIgnoreCase) == -1)
+                // Assert hardware VAAPI decodable (Except h264 10-bit and higher color depth)
+				// TODO: a propery way to detect hardware capabilities and falling back when transcoding is failed
+                if ((pixelFormat ?? string.Empty).IndexOf("p10", StringComparison.OrdinalIgnoreCase) == -1
+                    || ((pixelFormat ?? string.Empty).IndexOf("p10", StringComparison.OrdinalIgnoreCase) != -1
+                        && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)
+                            || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase)
+                            || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase))))
                 {
                     filters.Add("hwdownload");
                     filters.Add("format=nv12");
@@ -2076,7 +2072,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             filters.AddRange(GetScalingFilters(state, inputWidth, inputHeight, threeDFormat, videoDecoder, outputVideoCodec, request.Width, request.Height, request.MaxWidth, request.MaxHeight));
 
             // Add parameters to use VAAPI with burn-in text subttiles (GH issue #642)
-            if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && options.EnableHardwareEncoding)
+            if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
             {
                 if (state.SubtitleStream != null
                     && state.SubtitleStream.IsTextSubtitleStream