add extracting attachments for ffmpeg to burn subs

CONTRIBUTORS.md

@@ -76,6 +76,7 @@
  - [mitchfizz05](https://github.com/mitchfizz05)
  - [MrTimscampi](https://github.com/MrTimscampi)
  - [n8225](https://github.com/n8225)
+ - [Nalsai](https://github.com/Nalsai)
  - [Narfinger](https://github.com/Narfinger)
  - [NathanPickard](https://github.com/NathanPickard)
  - [neilsb](https://github.com/neilsb)

Jellyfin.Api/Helpers/TranscodingJobHelper.cs

@@ -18,6 +18,7 @@ using MediaBrowser.Controller.Library;
 using MediaBrowser.Controller.MediaEncoding;
 using MediaBrowser.Controller.Net;
 using MediaBrowser.Controller.Session;
+using MediaBrowser.Model.Dlna;
 using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.MediaInfo;
@@ -42,6 +43,8 @@ namespace Jellyfin.Api.Helpers
         /// </summary>
         private static readonly Dictionary<string, SemaphoreSlim> _transcodingLocks = new Dictionary<string, SemaphoreSlim>();
 
+        private readonly IAttachmentExtractor _attachmentExtractor;
+        private readonly IApplicationPaths _appPaths;
         private readonly IAuthorizationContext _authorizationContext;
         private readonly EncodingHelper _encodingHelper;
         private readonly IFileSystem _fileSystem;
@@ -55,6 +58,8 @@ namespace Jellyfin.Api.Helpers
         /// <summary>
         /// Initializes a new instance of the <see cref="TranscodingJobHelper"/> class.
         /// </summary>
+        /// <param name="attachmentExtractor">Instance of the <see cref="IAttachmentExtractor"/> interface.</param>
+        /// <param name="appPaths">Instance of the <see cref="IApplicationPaths"/> interface.</param>
         /// <param name="logger">Instance of the <see cref="ILogger{TranscodingJobHelpers}"/> interface.</param>
         /// <param name="mediaSourceManager">Instance of the <see cref="IMediaSourceManager"/> interface.</param>
         /// <param name="fileSystem">Instance of the <see cref="IFileSystem"/> interface.</param>
@@ -65,6 +70,8 @@ namespace Jellyfin.Api.Helpers
         /// <param name="encodingHelper">Instance of <see cref="EncodingHelper"/>.</param>
         /// <param name="loggerFactory">Instance of the <see cref="ILoggerFactory"/> interface.</param>
         public TranscodingJobHelper(
+            IAttachmentExtractor attachmentExtractor,
+            IApplicationPaths appPaths,
             ILogger<TranscodingJobHelper> logger,
             IMediaSourceManager mediaSourceManager,
             IFileSystem fileSystem,
@@ -75,6 +82,8 @@ namespace Jellyfin.Api.Helpers
             EncodingHelper encodingHelper,
             ILoggerFactory loggerFactory)
         {
+            _attachmentExtractor = attachmentExtractor;
+            _appPaths = appPaths;
             _logger = logger;
             _mediaSourceManager = mediaSourceManager;
             _fileSystem = fileSystem;
@@ -513,6 +522,13 @@ namespace Jellyfin.Api.Helpers
                 throw new ArgumentException("FFmpeg path not set.");
             }
 
+            // If subtitles get burned in fonts may need to be extracted from the media file
+            if (state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode)
+            {
+                var attachmentPath = Path.Combine(_appPaths.CachePath, "attachments", state.MediaSource.Id);
+                await _attachmentExtractor.ExtractAllAttachments(state.MediaPath, state.MediaSource, attachmentPath, CancellationToken.None).ConfigureAwait(false);
+            }
+
             var process = new Process
             {
                 StartInfo = new ProcessStartInfo

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -12,6 +12,7 @@ using System.Text.RegularExpressions;
 using System.Threading;
 using Jellyfin.Data.Enums;
 using Jellyfin.Extensions;
+using MediaBrowser.Common.Configuration;
 using MediaBrowser.Model.Configuration;
 using MediaBrowser.Model.Dlna;
 using MediaBrowser.Model.Dto;
@@ -28,7 +29,7 @@ namespace MediaBrowser.Controller.MediaEncoding
         private const string VideotoolboxAlias = "vt";
         private const string OpenclAlias = "ocl";
         private const string CudaAlias = "cu";
-
+        private readonly IApplicationPaths _appPaths;
         private readonly IMediaEncoder _mediaEncoder;
         private readonly ISubtitleEncoder _subtitleEncoder;
 
@@ -51,9 +52,11 @@ namespace MediaBrowser.Controller.MediaEncoding
         };
 
         public EncodingHelper(
+            IApplicationPaths appPaths,
             IMediaEncoder mediaEncoder,
             ISubtitleEncoder subtitleEncoder)
         {
+            _appPaths = appPaths;
             _mediaEncoder = mediaEncoder;
             _subtitleEncoder = subtitleEncoder;
         }
@@ -1080,6 +1083,12 @@ namespace MediaBrowser.Controller.MediaEncoding
             var alphaParam = enableAlpha ? ":alpha=1" : string.Empty;
             var sub2videoParam = enableSub2video ? ":sub2video=1" : string.Empty;
 
+            var fontPath = Path.Combine(_appPaths.CachePath, "attachments", state.MediaSource.Id);
+            var fontParam = string.Format(
+                CultureInfo.InvariantCulture,
+                ":fontsdir={0}",
+                _mediaEncoder.EscapeSubtitleFilterPath(fontPath));
+
             // TODO
             // var fallbackFontPath = Path.Combine(_appPaths.ProgramDataPath, "fonts", "DroidSansFallback.ttf");
             // string fallbackFontParam = string.Empty;
@@ -1120,11 +1129,12 @@ namespace MediaBrowser.Controller.MediaEncoding
                 // TODO: Perhaps also use original_size=1920x800 ??
                 return string.Format(
                     CultureInfo.InvariantCulture,
-                    "subtitles=f='{0}'{1}{2}{3}{4}",
+                    "subtitles=f='{0}'{1}{2}{3}{4}{5}",
                     _mediaEncoder.EscapeSubtitleFilterPath(subtitlePath),
                     charsetParam,
                     alphaParam,
                     sub2videoParam,
+                    fontParam,
                     // fallbackFontParam,
                     setPtsParam);
             }
@@ -1133,11 +1143,12 @@ namespace MediaBrowser.Controller.MediaEncoding
 
             return string.Format(
                 CultureInfo.InvariantCulture,
-                "subtitles='{0}:si={1}{2}{3}'{4}",
+                "subtitles='{0}:si={1}{2}{3}{4}'{5}",
                 _mediaEncoder.EscapeSubtitleFilterPath(mediaPath),
                 state.InternalSubtitleStreamOffset.ToString(CultureInfo.InvariantCulture),
                 alphaParam,
                 sub2videoParam,
+                fontParam,
                 // fallbackFontParam,
                 setPtsParam);
         }

MediaBrowser.Controller/MediaEncoding/IAttachmentExtractor.cs

@@ -6,6 +6,7 @@ using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
 using MediaBrowser.Controller.Entities;
+using MediaBrowser.Model.Dto;
 using MediaBrowser.Model.Entities;
 
 namespace MediaBrowser.Controller.MediaEncoding
@@ -17,5 +18,10 @@ namespace MediaBrowser.Controller.MediaEncoding
             string mediaSourceId,
             int attachmentStreamIndex,
             CancellationToken cancellationToken);
+        Task ExtractAllAttachments(
+            string inputFile,
+            MediaSourceInfo mediaSource,
+            string outputPath,
+            CancellationToken cancellationToken);
     }
 }

MediaBrowser.MediaEncoding/Attachments/AttachmentExtractor.cs

@@ -83,6 +83,130 @@ namespace MediaBrowser.MediaEncoding.Attachments
             return (mediaAttachment, attachmentStream);
         }
 
+        public async Task ExtractAllAttachments(
+            string inputFile,
+            MediaSourceInfo mediaSource,
+            string outputPath,
+            CancellationToken cancellationToken)
+        {
+            var semaphore = _semaphoreLocks.GetOrAdd(outputPath, key => new SemaphoreSlim(1, 1));
+
+            await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
+
+            try
+            {
+                if (!Directory.Exists(outputPath))
+                {
+                    await ExtractAllAttachmentsInternal(
+                        _mediaEncoder.GetInputArgument(inputFile, mediaSource),
+                        outputPath,
+                        cancellationToken).ConfigureAwait(false);
+                }
+            }
+            finally
+            {
+                semaphore.Release();
+            }
+        }
+
+        private async Task ExtractAllAttachmentsInternal(
+            string inputPath,
+            string outputPath,
+            CancellationToken cancellationToken)
+        {
+            if (string.IsNullOrEmpty(inputPath))
+            {
+                throw new ArgumentNullException(nameof(inputPath));
+            }
+
+            if (string.IsNullOrEmpty(outputPath))
+            {
+                throw new ArgumentNullException(nameof(outputPath));
+            }
+
+            Directory.CreateDirectory(outputPath);
+
+            var processArgs = string.Format(
+                CultureInfo.InvariantCulture,
+                "-dump_attachment:t \"\" -i {0} -t 0 -f null null",
+                inputPath);
+
+            int exitCode;
+
+            using (var process = new Process
+                {
+                    StartInfo = new ProcessStartInfo
+                    {
+                        Arguments = processArgs,
+                        FileName = _mediaEncoder.EncoderPath,
+                        UseShellExecute = false,
+                        CreateNoWindow = true,
+                        WindowStyle = ProcessWindowStyle.Hidden,
+                        WorkingDirectory = outputPath,
+                        ErrorDialog = false
+                    },
+                    EnableRaisingEvents = true
+                })
+            {
+                _logger.LogInformation("{File} {Arguments}", process.StartInfo.FileName, process.StartInfo.Arguments);
+
+                process.Start();
+
+                var ranToCompletion = await ProcessExtensions.WaitForExitAsync(process, cancellationToken).ConfigureAwait(false);
+
+                if (!ranToCompletion)
+                {
+                    try
+                    {
+                        _logger.LogWarning("Killing ffmpeg attachment extraction process");
+                        process.Kill();
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(ex, "Error killing attachment extraction process");
+                    }
+                }
+
+                exitCode = ranToCompletion ? process.ExitCode : -1;
+            }
+
+            var failed = false;
+
+            if (exitCode != 0)
+            {
+                failed = true;
+
+                _logger.LogWarning("Deleting extracted attachments {Path} due to failure: {ExitCode}", outputPath, exitCode);
+                try
+                {
+                    if (Directory.Exists(outputPath))
+                    {
+                        Directory.Delete(outputPath);
+                    }
+                }
+                catch (IOException ex)
+                {
+                    _logger.LogError(ex, "Error deleting extracted attachments {Path}", outputPath);
+                }
+            }
+            else if (!Directory.Exists(outputPath))
+            {
+                failed = true;
+            }
+
+            if (failed)
+            {
+                _logger.LogError("ffmpeg attachment extraction failed for {InputPath} to {OutputPath}", inputPath, outputPath);
+
+                throw new InvalidOperationException(
+                    string.Format(CultureInfo.InvariantCulture, "ffmpeg attachment extraction failed for {0} to {1}", inputPath, outputPath));
+            }
+            else
+            {
+                _logger.LogInformation("ffmpeg attachment extraction completed for {Path} to {Path}", inputPath, outputPath);
+            }
+        }
+
         private async Task<Stream> GetAttachmentStream(
             MediaSourceInfo mediaSource,
             MediaAttachment mediaAttachment,