Add support for non-jpg image extractions

MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs

@@ -95,9 +95,10 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// <param name="mediaSource">Media source information.</param>
         /// <param name="imageStream">Media stream information.</param>
         /// <param name="imageStreamIndex">Index of the stream to extract from.</param>
+        /// <param name="outputExtension">The extension of the file to write.</param>
         /// <param name="cancellationToken">CancellationToken to use for operation.</param>
         /// <returns>Location of video image.</returns>
-        Task<string> ExtractVideoImage(string inputFile, string container, MediaSourceInfo mediaSource, MediaStream imageStream, int? imageStreamIndex, CancellationToken cancellationToken);
+        Task<string> ExtractVideoImage(string inputFile, string container, MediaSourceInfo mediaSource, MediaStream imageStream, int? imageStreamIndex, string outputExtension, CancellationToken cancellationToken);
 
         /// <summary>
         /// Extracts the video images on interval.

MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs

@@ -468,17 +468,17 @@ namespace MediaBrowser.MediaEncoding.Encoder
                 Protocol = MediaProtocol.File
             };
 
-            return ExtractImage(path, null, null, imageStreamIndex, mediaSource, true, null, null, cancellationToken);
+            return ExtractImage(path, null, null, imageStreamIndex, mediaSource, true, null, null, "jpg", cancellationToken);
         }
 
         public Task<string> ExtractVideoImage(string inputFile, string container, MediaSourceInfo mediaSource, MediaStream videoStream, Video3DFormat? threedFormat, TimeSpan? offset, CancellationToken cancellationToken)
         {
-            return ExtractImage(inputFile, container, videoStream, null, mediaSource, false, threedFormat, offset, cancellationToken);
+            return ExtractImage(inputFile, container, videoStream, null, mediaSource, false, threedFormat, offset, "jpg", cancellationToken);
         }
 
-        public Task<string> ExtractVideoImage(string inputFile, string container, MediaSourceInfo mediaSource, MediaStream imageStream, int? imageStreamIndex, CancellationToken cancellationToken)
+        public Task<string> ExtractVideoImage(string inputFile, string container, MediaSourceInfo mediaSource, MediaStream imageStream, int? imageStreamIndex, string outputExtension, CancellationToken cancellationToken)
         {
-            return ExtractImage(inputFile, container, imageStream, imageStreamIndex, mediaSource, false, null, null, cancellationToken);
+            return ExtractImage(inputFile, container, imageStream, imageStreamIndex, mediaSource, false, null, null, outputExtension, cancellationToken);
         }
 
         private async Task<string> ExtractImage(
@@ -490,6 +490,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
             bool isAudio,
             Video3DFormat? threedFormat,
             TimeSpan? offset,
+            string outputExtension,
             CancellationToken cancellationToken)
         {
             var inputArgument = GetInputArgument(inputFile, mediaSource);
@@ -499,7 +500,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
                 // The failure of HDR extraction usually occurs when using custom ffmpeg that does not contain the zscale filter.
                 try
                 {
-                    return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, true, true, cancellationToken).ConfigureAwait(false);
+                    return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, true, true, outputExtension, cancellationToken).ConfigureAwait(false);
                 }
                 catch (ArgumentException)
                 {
@@ -512,7 +513,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
                 try
                 {
-                    return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, false, true, cancellationToken).ConfigureAwait(false);
+                    return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, false, true, outputExtension, cancellationToken).ConfigureAwait(false);
                 }
                 catch (ArgumentException)
                 {
@@ -525,7 +526,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
                 try
                 {
-                    return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, true, false, cancellationToken).ConfigureAwait(false);
+                    return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, true, false, outputExtension, cancellationToken).ConfigureAwait(false);
                 }
                 catch (ArgumentException)
                 {
@@ -537,17 +538,17 @@ namespace MediaBrowser.MediaEncoding.Encoder
                 }
             }
 
-            return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, false, false, cancellationToken).ConfigureAwait(false);
+            return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, false, false, outputExtension, cancellationToken).ConfigureAwait(false);
         }
 
-        private async Task<string> ExtractImageInternal(string inputPath, string container, MediaStream videoStream, int? imageStreamIndex, Video3DFormat? threedFormat, TimeSpan? offset, bool useIFrame, bool allowTonemap, CancellationToken cancellationToken)
+        private async Task<string> ExtractImageInternal(string inputPath, string container, MediaStream videoStream, int? imageStreamIndex, Video3DFormat? threedFormat, TimeSpan? offset, bool useIFrame, bool allowTonemap, string outputExtension, CancellationToken cancellationToken)
         {
             if (string.IsNullOrEmpty(inputPath))
             {
                 throw new ArgumentNullException(nameof(inputPath));
             }
 
-            var tempExtractPath = Path.Combine(_configurationManager.ApplicationPaths.TempDirectory, Guid.NewGuid() + ".jpg");
+            var tempExtractPath = Path.Combine(_configurationManager.ApplicationPaths.TempDirectory, Guid.NewGuid() + "." + outputExtension);
             Directory.CreateDirectory(Path.GetDirectoryName(tempExtractPath));
 
             // apply some filters to thumbnail extracted below (below) crop any black lines that we made and get the correct ar.

MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs

@@ -582,7 +582,8 @@ namespace MediaBrowser.MediaEncoding.Probing
         /// <returns>MediaAttachments.</returns>
         private MediaAttachment GetMediaAttachment(MediaStreamInfo streamInfo)
         {
-            if (!string.Equals(streamInfo.CodecType, "attachment", StringComparison.OrdinalIgnoreCase))
+            if (!string.Equals(streamInfo.CodecType, "attachment", StringComparison.OrdinalIgnoreCase) &&
+                !(streamInfo.Disposition != null && streamInfo.Disposition.GetValueOrDefault("attached_pic") == 1))
             {
                 return null;
             }
@@ -735,15 +736,14 @@ namespace MediaBrowser.MediaEncoding.Probing
                 else if (string.Equals(stream.Codec, "mjpeg", StringComparison.OrdinalIgnoreCase))
                 {
                     // How to differentiate between video and embedded image?
-                    // check disposition, alternately: presence of codec tag, also embedded images have high (unusual) framerates
-                    if ((streamInfo.Disposition != null && streamInfo.Disposition.GetValueOrDefault("attached_pic") == 1) ||
-                        string.IsNullOrWhiteSpace(stream.CodecTag))
+                    // The only difference I've seen thus far is presence of codec tag, also embedded images have high (unusual) framerates
+                    if (!string.IsNullOrWhiteSpace(stream.CodecTag))
                     {
-                        stream.Type = MediaStreamType.EmbeddedImage;
+                        stream.Type = MediaStreamType.Video;
                     }
                     else
                     {
-                        stream.Type = MediaStreamType.Video;
+                        stream.Type = MediaStreamType.EmbeddedImage;
                     }
                 }
                 else
@@ -812,12 +812,6 @@ namespace MediaBrowser.MediaEncoding.Probing
                 {
                     stream.ColorPrimaries = streamInfo.ColorPrimaries;
                 }
-
-                // workaround for mkv attached_pics losing filename due to being classified as video based on codec
-                if (stream.Type == MediaStreamType.EmbeddedImage && streamInfo.Tags != null && string.IsNullOrEmpty(stream.Comment))
-                {
-                    stream.Comment = GetDictionaryValue(streamInfo.Tags, "filename");
-                }
             }
             else
             {

MediaBrowser.Providers/MediaInfo/EmbeddedImageProvider.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.IO;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
@@ -15,6 +16,7 @@ using MediaBrowser.Model.Drawing;
 using MediaBrowser.Model.Dto;
 using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.MediaInfo;
+using MediaBrowser.Model.Net;
 using Microsoft.Extensions.Logging;
 
 namespace MediaBrowser.Providers.MediaInfo
@@ -115,25 +117,49 @@ namespace MediaBrowser.Providers.MediaInfo
                 Protocol = item.PathProtocol ?? MediaProtocol.File,
             };
 
-            string[] imageFileNames;
-            switch (type)
+            string[] imageFileNames = type switch
             {
-                case ImageType.Backdrop:
-                    imageFileNames = _backdropImageFileNames;
-                    break;
-                case ImageType.Logo:
-                    imageFileNames = _logoImageFileNames;
-                    break;
-                case ImageType.Primary:
-                default:
-                    imageFileNames = _primaryImageFileNames;
-                    break;
+                ImageType.Primary => _primaryImageFileNames,
+                ImageType.Backdrop => _backdropImageFileNames,
+                ImageType.Logo => _logoImageFileNames,
+                _ => _primaryImageFileNames
+            };
+
+            // Try attachments first
+            var attachmentSources = item.GetMediaSources(false).SelectMany(source => source.MediaAttachments).ToList();
+            var attachmentStream = attachmentSources
+                .Where(stream => !string.IsNullOrEmpty(stream.FileName))
+                .First(stream => imageFileNames.Any(name => stream.FileName.Contains(name, StringComparison.OrdinalIgnoreCase)));
+
+            if (attachmentStream != null)
+            {
+                var extension = (string.IsNullOrEmpty(attachmentStream.MimeType) ?
+                    Path.GetExtension(attachmentStream.FileName) :
+                    MimeTypes.ToExtension(attachmentStream.MimeType)) ?? "jpg";
+
+                string extractedAttachmentPath = await _mediaEncoder.ExtractVideoImage(item.Path, item.Container, mediaSource, null, attachmentStream.Index, extension, cancellationToken).ConfigureAwait(false);
+
+                ImageFormat format = extension switch
+                {
+                    "bmp" => ImageFormat.Bmp,
+                    "gif" => ImageFormat.Gif,
+                    "jpg" => ImageFormat.Jpg,
+                    "png" => ImageFormat.Png,
+                    "webp" => ImageFormat.Webp,
+                    _ => ImageFormat.Jpg
+                };
+
+                return new DynamicImageResponse
+                {
+                    Format = format,
+                    HasImage = true,
+                    Path = extractedAttachmentPath,
+                    Protocol = MediaProtocol.File
+                };
             }
 
-            var imageStreams =
-                item.GetMediaStreams()
-                    .Where(i => i.Type == MediaStreamType.EmbeddedImage)
-                    .ToList();
+            // Fall back to EmbeddedImage streams
+            var imageStreams = item.GetMediaStreams().FindAll(i => i.Type == MediaStreamType.EmbeddedImage);
 
             if (!imageStreams.Any())
             {
@@ -160,7 +186,7 @@ namespace MediaBrowser.Providers.MediaInfo
                 }
             }
 
-            string extractedImagePath = await _mediaEncoder.ExtractVideoImage(item.Path, item.Container, mediaSource, imageStream, imageStream.Index, cancellationToken).ConfigureAwait(false);
+            string extractedImagePath = await _mediaEncoder.ExtractVideoImage(item.Path, item.Container, mediaSource, imageStream, imageStream.Index, "jpg", cancellationToken).ConfigureAwait(false);
 
             return new DynamicImageResponse
             {