Backport pull request #11830 from jellyfin/release-10.9.z

Fix the IOSurf error in QSV transcoding

Original-merge: 5e7514243c71f68afe5e4403c0ee702f63b94b60

Merged-by: crobibero <cody@robibe.ro>

Backported-by: Joshua M. Boniface <joshua@boniface.me>

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -4301,6 +4301,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                 {
                     // map from qsv to vaapi.
                     mainFilters.Add("hwmap=derive_device=vaapi");
+                    mainFilters.Add("format=vaapi");
                 }
 
                 var tonemapFilter = GetHwTonemapFilter(options, "vaapi", "nv12");
@@ -4310,6 +4311,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                 {
                     // map from vaapi to qsv.
                     mainFilters.Add("hwmap=derive_device=qsv");
+                    mainFilters.Add("format=qsv");
                 }
             }