Merge pull request #11250 from nyanmisaka/fix-hwa-video-rotation

Fix the broken video orientation (+-90/180)

Emby.Server.Implementations/Data/SqliteItemRepository.cs

@@ -183,7 +183,8 @@ namespace Emby.Server.Implementations.Data
             "ElPresentFlag",
             "BlPresentFlag",
             "DvBlSignalCompatibilityId",
-            "IsHearingImpaired"
+            "IsHearingImpaired",
+            "Rotation"
         };
 
         private static readonly string _mediaStreamSaveColumnsInsertQuery =
@@ -343,7 +344,7 @@ namespace Emby.Server.Implementations.Data
             base.Initialize();
 
             const string CreateMediaStreamsTableCommand
-                    = "create table if not exists mediastreams (ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, CodecTag TEXT NULL, Comment TEXT NULL, NalLengthSize TEXT NULL, IsAvc BIT NULL, Title TEXT NULL, TimeBase TEXT NULL, CodecTimeBase TEXT NULL, ColorPrimaries TEXT NULL, ColorSpace TEXT NULL, ColorTransfer TEXT NULL, DvVersionMajor INT NULL, DvVersionMinor INT NULL, DvProfile INT NULL, DvLevel INT NULL, RpuPresentFlag INT NULL, ElPresentFlag INT NULL, BlPresentFlag INT NULL, DvBlSignalCompatibilityId INT NULL, IsHearingImpaired BIT NULL, PRIMARY KEY (ItemId, StreamIndex))";
+                    = "create table if not exists mediastreams (ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, CodecTag TEXT NULL, Comment TEXT NULL, NalLengthSize TEXT NULL, IsAvc BIT NULL, Title TEXT NULL, TimeBase TEXT NULL, CodecTimeBase TEXT NULL, ColorPrimaries TEXT NULL, ColorSpace TEXT NULL, ColorTransfer TEXT NULL, DvVersionMajor INT NULL, DvVersionMinor INT NULL, DvProfile INT NULL, DvLevel INT NULL, RpuPresentFlag INT NULL, ElPresentFlag INT NULL, BlPresentFlag INT NULL, DvBlSignalCompatibilityId INT NULL, IsHearingImpaired BIT NULL, Rotation INT NULL, PRIMARY KEY (ItemId, StreamIndex))";
 
             const string CreateMediaAttachmentsTableCommand
                     = "create table if not exists mediaattachments (ItemId GUID, AttachmentIndex INT, Codec TEXT, CodecTag TEXT NULL, Comment TEXT NULL, Filename TEXT NULL, MIMEType TEXT NULL, PRIMARY KEY (ItemId, AttachmentIndex))";
@@ -538,6 +539,8 @@ namespace Emby.Server.Implementations.Data
 
                 AddColumn(connection, "MediaStreams", "IsHearingImpaired", "BIT", existingColumnNames);
 
+                AddColumn(connection, "MediaStreams", "Rotation", "INT", existingColumnNames);
+
                 connection.Execute(string.Join(';', postQueries));
 
                 transaction.Commit();
@@ -5483,6 +5486,8 @@ AND Type = @InternalPersonType)");
                         statement.TryBind("@DvBlSignalCompatibilityId" + index, stream.DvBlSignalCompatibilityId);
 
                         statement.TryBind("@IsHearingImpaired" + index, stream.IsHearingImpaired);
+
+                        statement.TryBind("@Rotation" + index, stream.Rotation);
                     }
 
                     statement.ExecuteNonQuery();
@@ -5694,6 +5699,11 @@ AND Type = @InternalPersonType)");
 
             item.IsHearingImpaired = reader.TryGetBoolean(43, out var result) && result;
 
+            if (reader.TryGetInt32(44, out var rotation))
+            {
+                item.Rotation = rotation;
+            }
+
             if (item.Type is MediaStreamType.Audio or MediaStreamType.Subtitle)
             {
                 item.LocalizedDefault = _localization.GetLocalizedString("Default");

Jellyfin.Api/Helpers/DynamicHlsHelper.cs

@@ -198,6 +198,17 @@ public class DynamicHlsHelper
             AddSubtitles(state, subtitleStreams, builder, _httpContextAccessor.HttpContext.User);
         }
 
+        // Video rotation metadata is only supported in fMP4 remuxing
+        if (state.VideoStream is not null
+            && state.VideoRequest is not null
+            && (state.VideoStream?.Rotation ?? 0) != 0
+            && EncodingHelper.IsCopyCodec(state.OutputVideoCodec)
+            && !string.IsNullOrWhiteSpace(state.Request.SegmentContainer)
+            && !string.Equals(state.Request.SegmentContainer, "mp4", StringComparison.OrdinalIgnoreCase))
+        {
+            playlistUrl += "&AllowVideoStreamCopy=false";
+        }
+
         var basicPlaylist = AppendPlaylist(builder, state, playlistUrl, totalBitrate, subtitleGroup);
 
         if (state.VideoStream is not null && state.VideoRequest is not null)

MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs

@@ -33,6 +33,11 @@ namespace MediaBrowser.Controller.MediaEncoding
         /// <summary>
         /// The overlay_vulkan_framesync.
         /// </summary>
-        OverlayVulkanFrameSync = 5
+        OverlayVulkanFrameSync = 5,
+
+        /// <summary>
+        /// The transpose_opencl_reversal.
+        /// </summary>
+        TransposeOpenclReversal = 6
     }
 }

MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs

@@ -112,25 +112,31 @@ namespace MediaBrowser.MediaEncoding.Encoder
             "yadif_cuda",
             "tonemap_cuda",
             "overlay_cuda",
+            "transpose_cuda",
             "hwupload_cuda",
             // opencl
             "scale_opencl",
             "tonemap_opencl",
             "overlay_opencl",
+            "transpose_opencl",
             // vaapi
             "scale_vaapi",
             "deinterlace_vaapi",
             "tonemap_vaapi",
             "procamp_vaapi",
             "overlay_vaapi",
+            "transpose_vaapi",
             "hwupload_vaapi",
             // vulkan
             "libplacebo",
             "scale_vulkan",
             "overlay_vulkan",
+            "transpose_vulkan",
+            "flip_vulkan",
             // videotoolbox
             "yadif_videotoolbox",
             "scale_vt",
+            "transpose_vt",
             "overlay_videotoolbox",
             "tonemap_videotoolbox",
             // rkrga
@@ -146,7 +152,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
             { 2, new string[] { "tonemap_opencl", "bt2390" } },
             { 3, new string[] { "overlay_opencl", "Action to take when encountering EOF from secondary input" } },
             { 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } },
-            { 5, new string[] { "overlay_vulkan", "Action to take when encountering EOF from secondary input" } }
+            { 5, new string[] { "overlay_vulkan", "Action to take when encountering EOF from secondary input" } },
+            { 6, new string[] { "transpose_opencl", "rotate by half-turn" } }
         };
 
         // These are the library versions that corresponds to our minimum ffmpeg version 4.4 according to the version table below

MediaBrowser.MediaEncoding/Probing/MediaStreamInfoSideData.cs

@@ -69,5 +69,12 @@ namespace MediaBrowser.MediaEncoding.Probing
         /// <value>The DvBlSignalCompatibilityId.</value>
         [JsonPropertyName("dv_bl_signal_compatibility_id")]
         public int? DvBlSignalCompatibilityId { get; set; }
+
+        /// <summary>
+        /// Gets or sets the Rotation in degrees.
+        /// </summary>
+        /// <value>The Rotation.</value>
+        [JsonPropertyName("rotation")]
+        public int? Rotation { get; set; }
     }
 }

MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs

@@ -892,8 +892,12 @@ namespace MediaBrowser.MediaEncoding.Probing
                             stream.ElPresentFlag = data.ElPresentFlag;
                             stream.BlPresentFlag = data.BlPresentFlag;
                             stream.DvBlSignalCompatibilityId = data.DvBlSignalCompatibilityId;
+                        }
 
-                            break;
+                        // Parse video rotation metadata from side_data
+                        else if (string.Equals(data.SideDataType, "Display Matrix", StringComparison.OrdinalIgnoreCase))
+                        {
+                            stream.Rotation = data.Rotation;
                         }
                     }
                 }

MediaBrowser.Model/Entities/MediaStream.cs

@@ -123,6 +123,12 @@ namespace MediaBrowser.Model.Entities
         /// <value>The Dolby Vision bl signal compatibility id.</value>
         public int? DvBlSignalCompatibilityId { get; set; }
 
+        /// <summary>
+        /// Gets or sets the Rotation in degrees.
+        /// </summary>
+        /// <value>The video rotation.</value>
+        public int? Rotation { get; set; }
+
         /// <summary>
         /// Gets or sets the comment.
         /// </summary>

tests/Jellyfin.MediaEncoding.Tests/Probing/ProbeResultNormalizerTests.cs

@@ -84,6 +84,7 @@ namespace Jellyfin.MediaEncoding.Tests.Probing
             Assert.Equal(0, res.VideoStream.ElPresentFlag);
             Assert.Equal(1, res.VideoStream.BlPresentFlag);
             Assert.Equal(0, res.VideoStream.DvBlSignalCompatibilityId);
+            Assert.Equal(-180, res.VideoStream.Rotation);
 
             var audio1 = res.MediaStreams[1];
             Assert.Equal("eac3", audio1.Codec);

tests/Jellyfin.MediaEncoding.Tests/Test Data/Probing/video_metadata.json

@@ -59,6 +59,10 @@
                     "el_present_flag": 0,
                     "bl_present_flag": 1,
                     "dv_bl_signal_compatibility_id": 0
+                },
+                {
+                    "side_data_type": "Display Matrix",
+                    "rotation": -180
                 }
             ]
         },