Merge pull request #6053 from jellyfin/cuda-fix

Fix the 'No decoder surfaces left' error on Cuda

(cherry picked from commit 9b82b370955f700fc3a81330eeacb3a317894c1a)
Signed-off-by: Joshua M. Boniface <joshua@boniface.me>

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -598,7 +598,8 @@ namespace MediaBrowser.Controller.MediaEncoding
                     && string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)
                     && isNvdecDecoder)
                 {
-                    arg.Append("-hwaccel_output_format cuda ");
+                    // Fix for 'No decoder surfaces left' error. https://trac.ffmpeg.org/ticket/7562
+                    arg.Append("-hwaccel_output_format cuda -extra_hw_frames 3 -autorotate 0 ");
                 }
 
                 if (state.IsVideoRequest
@@ -1072,7 +1073,6 @@ namespace MediaBrowser.Controller.MediaEncoding
             else if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) // h264 (h264_nvenc)
                      || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase)) // hevc (hevc_nvenc)
             {
-                // following preset will be deprecated in ffmpeg 4.4, use p1~p7 instead.
                 switch (encodingOptions.EncoderPreset)
                 {
                     case "veryslow":
@@ -1253,7 +1253,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             }
 
             if (string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase)
-                && profile.Contains("constrainedbaseline", StringComparison.OrdinalIgnoreCase))
+                && profile.Contains("baseline", StringComparison.OrdinalIgnoreCase))
             {
                 profile = "constrained_baseline";
             }