[VA-API] Fix filter order (#714)

ffmpeg is very picky about the filters to be used when using VA-API,
because most of them are incompatible. This is particularly evident when
burning-in subtitles.

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -1438,6 +1438,11 @@ namespace MediaBrowser.Controller.MediaEncoding
             if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase) && outputSizeParam.Length == 0)
             {
                 outputSizeParam = ",format=nv12|vaapi,hwupload";
+
+                // Add parameters to use VAAPI with burn-in subttiles (GH issue #642)
+                if (state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode) {
+                    outputSizeParam += ",hwmap=mode=read+write+direct";
+                }
             }
 
             var videoSizeParam = string.Empty;
@@ -1741,6 +1746,12 @@ namespace MediaBrowser.Controller.MediaEncoding
 
                 filters.Add(subParam);
 
+                // Ensure proper filters are passed to ffmpeg in case of hardware acceleration via VA-API
+                // Reference: https://trac.ffmpeg.org/wiki/Hardware/VAAPI
+                if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase))
+                {
+                    filters.Add("hwmap");
+                }
                 if (allowTimeStampCopy)
                 {
                     output += " -copyts";