capture key frame info

MediaBrowser.Api/Playback/BaseStreamingService.cs

@@ -1705,6 +1705,102 @@ namespace MediaBrowser.Api.Playback
             {
                 state.OutputAudioCodec = "copy";
             }
+
+            if (string.Equals(state.OutputVideoCodec, "copy", StringComparison.OrdinalIgnoreCase))
+            {
+                var segmentLength = GetSegmentLength(state);
+                if (segmentLength.HasValue)
+                {
+                    state.SegmentLength = segmentLength.Value;
+                }
+            }
+        }
+
+        private int? GetSegmentLength(StreamState state)
+        {
+            var stream = state.VideoStream;
+
+            if (stream == null)
+            {
+                return null;
+            }
+
+            var frames = stream.KeyFrames;
+
+            if (frames == null || frames.Count < 2)
+            {
+                return null;
+            }
+
+            Logger.Debug("Found keyframes at {0}", string.Join(",", frames.ToArray()));
+
+            var intervals = new List<int>();
+            for (var i = 1; i < frames.Count; i++)
+            {
+                var start = frames[i - 1];
+                var end = frames[i];
+                intervals.Add(end - start);
+            }
+
+            Logger.Debug("Found keyframes intervals {0}", string.Join(",", intervals.ToArray()));
+
+            var results = new List<Tuple<int, int>>();
+
+            for (var i = 1; i <= 10; i++)
+            {
+                var idealMs = i*1000;
+
+                if (intervals.Max() < idealMs - 1000)
+                {
+                    break;
+                }
+
+                var segments = PredictStreamCopySegments(intervals, idealMs);
+                var variance = segments.Select(s => Math.Abs(idealMs - s)).Sum();
+
+                results.Add(new Tuple<int, int>(i, variance));
+            }
+
+            if (results.Count == 0)
+            {
+                return null;
+            }
+
+            return results.OrderBy(i => i.Item2).ThenBy(i => i.Item1).Select(i => i.Item1).First();
+        }
+
+        private List<int> PredictStreamCopySegments(List<int> intervals, int idealMs)
+        {
+            var segments = new List<int>();
+            var currentLength = 0;
+
+            foreach (var interval in intervals)
+            {
+                if (currentLength == 0 || (currentLength + interval) <= idealMs)
+                {
+                    currentLength += interval;
+                }
+
+                else
+                {
+                    // The segment will either be above or below the ideal. 
+                    // Need to figure out which is preferable
+                    var offset1 = Math.Abs(idealMs - currentLength);
+                    var offset2 = Math.Abs(idealMs - (currentLength + interval));
+
+                    if (offset1 <= offset2)
+                    {
+                        segments.Add(currentLength);
+                        currentLength = interval;
+                    }
+                    else
+                    {
+                        currentLength += interval;
+                    }
+                }
+            }
+            Logger.Debug("Predicted actual segment lengths for length {0}: {1}", idealMs, string.Join(",", segments.ToArray()));
+            return segments;
         }
 
         private void AttachMediaSourceInfo(StreamState state,

MediaBrowser.Api/Playback/Progressive/VideoService.cs

@@ -5,7 +5,6 @@ using MediaBrowser.Controller.Devices;
 using MediaBrowser.Controller.Dlna;
 using MediaBrowser.Controller.Drawing;
 using MediaBrowser.Controller.Library;
-using MediaBrowser.Controller.LiveTv;
 using MediaBrowser.Controller.MediaEncoding;
 using MediaBrowser.Model.IO;
 using ServiceStack;

MediaBrowser.Controller/MediaEncoding/MediaInfoRequest.cs

@@ -15,6 +15,7 @@ namespace MediaBrowser.Controller.MediaEncoding
         public IIsoMount MountedIso { get; set; }
         public VideoType VideoType { get; set; }
         public List<string> PlayableStreamFileNames { get; set; }
+        public bool ExtractKeyFrameInterval { get; set; }
 
         public MediaInfoRequest()
         {

MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs

@@ -1,4 +1,3 @@
-using System.Collections.Generic;
 using MediaBrowser.Common.IO;
 using MediaBrowser.Controller.Channels;
 using MediaBrowser.Controller.Configuration;
@@ -14,6 +13,7 @@ using MediaBrowser.Model.Logging;
 using MediaBrowser.Model.MediaInfo;
 using MediaBrowser.Model.Serialization;
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
 using System.IO;
@@ -75,7 +75,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
         protected readonly Func<ISubtitleEncoder> SubtitleEncoder;
         protected readonly Func<IMediaSourceManager> MediaSourceManager;
 
-        private List<Process> _runningProcesses = new List<Process>(); 
+        private readonly List<Process> _runningProcesses = new List<Process>();
 
         public MediaEncoder(ILogger logger, IJsonSerializer jsonSerializer, string ffMpegPath, string ffProbePath, string version, IServerConfigurationManager configurationManager, IFileSystem fileSystem, ILiveTvManager liveTvManager, IIsoManager isoManager, ILibraryManager libraryManager, IChannelManager channelManager, ISessionManager sessionManager, Func<ISubtitleEncoder> subtitleEncoder, Func<IMediaSourceManager> mediaSourceManager)
         {
@@ -116,7 +116,9 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
             var inputFiles = MediaEncoderHelpers.GetInputArgument(request.InputPath, request.Protocol, request.MountedIso, request.PlayableStreamFileNames);
 
-            return GetMediaInfoInternal(GetInputArgument(inputFiles, request.Protocol), request.InputPath, request.Protocol, extractChapters,
+            var extractKeyFrameInterval = request.ExtractKeyFrameInterval && request.Protocol == MediaProtocol.File && request.VideoType == VideoType.VideoFile;
+
+            return GetMediaInfoInternal(GetInputArgument(inputFiles, request.Protocol), request.InputPath, request.Protocol, extractChapters, extractKeyFrameInterval,
                 GetProbeSizeArgument(inputFiles, request.Protocol), request.MediaType == DlnaProfileType.Audio, cancellationToken);
         }
 
@@ -150,12 +152,17 @@ namespace MediaBrowser.MediaEncoding.Encoder
         /// <param name="primaryPath">The primary path.</param>
         /// <param name="protocol">The protocol.</param>
         /// <param name="extractChapters">if set to <c>true</c> [extract chapters].</param>
+        /// <param name="extractKeyFrameInterval">if set to <c>true</c> [extract key frame interval].</param>
         /// <param name="probeSizeArgument">The probe size argument.</param>
         /// <param name="isAudio">if set to <c>true</c> [is audio].</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns>Task{MediaInfoResult}.</returns>
         /// <exception cref="System.ApplicationException"></exception>
-        private async Task<Model.MediaInfo.MediaInfo> GetMediaInfoInternal(string inputPath, string primaryPath, MediaProtocol protocol, bool extractChapters,
+        private async Task<Model.MediaInfo.MediaInfo> GetMediaInfoInternal(string inputPath,
+            string primaryPath,
+            MediaProtocol protocol,
+            bool extractChapters,
+            bool extractKeyFrameInterval,
             string probeSizeArgument,
             bool isAudio,
             CancellationToken cancellationToken)
@@ -174,6 +181,7 @@ namespace MediaBrowser.MediaEncoding.Encoder
                     // Must consume both or ffmpeg may hang due to deadlocks. See comments below.   
                     RedirectStandardOutput = true,
                     RedirectStandardError = true,
+                    RedirectStandardInput = true,
                     FileName = FFProbePath,
                     Arguments = string.Format(args,
                     probeSizeArgument, inputPath).Trim(),
@@ -187,12 +195,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
             _logger.Debug("{0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments);
 
-            process.Exited += ProcessExited;
-
             await _ffProbeResourcePool.WaitAsync(cancellationToken).ConfigureAwait(false);
 
-            InternalMediaInfoResult result;
-
             try
             {
                 StartProcess(process);
@@ -210,19 +214,55 @@ namespace MediaBrowser.MediaEncoding.Encoder
             {
                 process.BeginErrorReadLine();
 
-                result = _jsonSerializer.DeserializeFromStream<InternalMediaInfoResult>(process.StandardOutput.BaseStream);
+                var result = _jsonSerializer.DeserializeFromStream<InternalMediaInfoResult>(process.StandardOutput.BaseStream);
+
+                if (result != null)
+                {
+                    if (result.streams != null)
+                    {
+                        // Normalize aspect ratio if invalid
+                        foreach (var stream in result.streams)
+                        {
+                            if (string.Equals(stream.display_aspect_ratio, "0:1", StringComparison.OrdinalIgnoreCase))
+                            {
+                                stream.display_aspect_ratio = string.Empty;
+                            }
+                            if (string.Equals(stream.sample_aspect_ratio, "0:1", StringComparison.OrdinalIgnoreCase))
+                            {
+                                stream.sample_aspect_ratio = string.Empty;
+                            }
+                        }
+                    }
+
+                    var mediaInfo = new ProbeResultNormalizer(_logger, FileSystem).GetMediaInfo(result, isAudio, primaryPath, protocol);
+
+                    if (extractKeyFrameInterval && mediaInfo.RunTimeTicks.HasValue)
+                    {
+                        foreach (var stream in mediaInfo.MediaStreams.Where(i => i.Type == MediaStreamType.Video)
+                            .ToList())
+                        {
+                            try
+                            {
+                                stream.KeyFrames = await GetKeyFrames(inputPath, stream.Index, cancellationToken)
+                                            .ConfigureAwait(false);
+                            }
+                            catch (OperationCanceledException)
+                            {
+                                
+                            }
+                            catch (Exception ex)
+                            {
+                                _logger.ErrorException("Error getting key frame interval", ex);
+                            }
+                        }
+                    }
+
+                    return mediaInfo;
+                }
             }
             catch
             {
-                // Hate having to do this
-                try
-                {
-                    process.Kill();
-                }
-                catch (Exception ex1)
-                {
-                    _logger.ErrorException("Error killing ffprobe", ex1);
-                }
+                StopProcess(process, 100, true);
 
                 throw;
             }
@@ -231,30 +271,108 @@ namespace MediaBrowser.MediaEncoding.Encoder
                 _ffProbeResourcePool.Release();
             }
 
-            if (result == null)
+            throw new ApplicationException(string.Format("FFProbe failed for {0}", inputPath));
+        }
+
+        private async Task<List<int>> GetKeyFrames(string inputPath, int videoStreamIndex, CancellationToken cancellationToken)
+        {
+            const string args = "-i {0} -select_streams v:{1} -show_frames -print_format compact";
+
+            var process = new Process
             {
-                throw new ApplicationException(string.Format("FFProbe failed for {0}", inputPath));
+                StartInfo = new ProcessStartInfo
+                {
+                    CreateNoWindow = true,
+                    UseShellExecute = false,
+
+                    // Must consume both or ffmpeg may hang due to deadlocks. See comments below.   
+                    RedirectStandardOutput = true,
+                    RedirectStandardError = true,
+                    RedirectStandardInput = true,
+                    FileName = FFProbePath,
+                    Arguments = string.Format(args, inputPath, videoStreamIndex.ToString(CultureInfo.InvariantCulture)).Trim(),
+
+                    WindowStyle = ProcessWindowStyle.Hidden,
+                    ErrorDialog = false
+                },
+
+                EnableRaisingEvents = true
+            };
+
+            _logger.Debug("{0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments);
+
+            StartProcess(process);
+
+            var lines = new List<int>();
+            var outputCancellationSource = new CancellationTokenSource(4000);
+
+            try
+            {
+                process.BeginErrorReadLine();
+
+                var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(outputCancellationSource.Token, cancellationToken);
+
+                await StartReadingOutput(process.StandardOutput.BaseStream, lines, 120000, outputCancellationSource, linkedCancellationTokenSource.Token).ConfigureAwait(false);
+            }
+            catch (OperationCanceledException)
+            {
+                if (cancellationToken.IsCancellationRequested)
+                {
+                    throw;
+                }
+            }
+            finally
+            {
+                StopProcess(process, 100, true);
             }
 
-            cancellationToken.ThrowIfCancellationRequested();
+            return lines;
+        }
 
-            if (result.streams != null)
+        private async Task StartReadingOutput(Stream source, List<int> lines, int timeoutMs, CancellationTokenSource cancellationTokenSource, CancellationToken cancellationToken)
+        {
+            try
             {
-                // Normalize aspect ratio if invalid
-                foreach (var stream in result.streams)
+                using (var reader = new StreamReader(source))
                 {
-                    if (string.Equals(stream.display_aspect_ratio, "0:1", StringComparison.OrdinalIgnoreCase))
-                    {
-                        stream.display_aspect_ratio = string.Empty;
-                    }
-                    if (string.Equals(stream.sample_aspect_ratio, "0:1", StringComparison.OrdinalIgnoreCase))
+                    while (!reader.EndOfStream)
                     {
-                        stream.sample_aspect_ratio = string.Empty;
+                        cancellationToken.ThrowIfCancellationRequested();
+
+                        var line = await reader.ReadLineAsync().ConfigureAwait(false);
+
+                        var values = (line ?? string.Empty).Split('|')
+                            .Where(i => !string.IsNullOrWhiteSpace(i))
+                            .Select(i => i.Split('='))
+                            .Where(i => i.Length == 2)
+                            .ToDictionary(i => i[0], i => i[1]);
+
+                        string pktDts;
+                        int frameMs;
+                        if (values.TryGetValue("pkt_dts", out pktDts) && int.TryParse(pktDts, NumberStyles.Any, CultureInfo.InvariantCulture, out frameMs))
+                        {
+                            string keyFrame;
+                            if (values.TryGetValue("key_frame", out keyFrame) && string.Equals(keyFrame, "1", StringComparison.OrdinalIgnoreCase))
+                            {
+                                lines.Add(frameMs);
+                            }
+
+                            if (frameMs > timeoutMs)
+                            {
+                                cancellationTokenSource.Cancel();
+                            }
+                        }
                     }
                 }
             }
-
-            return new ProbeResultNormalizer(_logger, FileSystem).GetMediaInfo(result, isAudio, primaryPath, protocol);
+            catch (OperationCanceledException)
+            {
+                throw;
+            }
+            catch (Exception ex)
+            {
+                _logger.ErrorException("Error reading ffprobe output", ex);
+            }
         }
 
         /// <summary>
@@ -269,7 +387,14 @@ namespace MediaBrowser.MediaEncoding.Encoder
         /// <param name="e">The <see cref="EventArgs" /> instance containing the event data.</param>
         private void ProcessExited(object sender, EventArgs e)
         {
-            ((Process)sender).Dispose();
+            var process = (Process) sender;
+
+            lock (_runningProcesses)
+            {
+                _runningProcesses.Remove(process);
+            }
+
+            process.Dispose();
         }
 
         public Task<Stream> ExtractAudioImage(string path, CancellationToken cancellationToken)
@@ -574,6 +699,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
 
         private void StartProcess(Process process)
         {
+            process.Exited += ProcessExited;
+
             process.Start();
 
             lock (_runningProcesses)
@@ -587,27 +714,36 @@ namespace MediaBrowser.MediaEncoding.Encoder
             {
                 _logger.Info("Killing ffmpeg process");
 
-                process.StandardInput.WriteLine("q");
+                try
+                {
+                    process.StandardInput.WriteLine("q");
+                }
+                catch (Exception)
+                {
+                    _logger.Error("Error sending q command to process");
+                }
 
-                if (!process.WaitForExit(1000))
+                try
                 {
-                    if (enableForceKill)
+                    if (process.WaitForExit(waitTimeMs))
                     {
-                        process.Kill();
+                        return;
                     }
                 }
+                catch (Exception ex)
+                {
+                    _logger.Error("Error in WaitForExit", ex);
+                }
+
+                if (enableForceKill)
+                {
+                    process.Kill();
+                }
             }
             catch (Exception ex)
             {
                 _logger.ErrorException("Error killing process", ex);
             }
-            finally
-            {
-                lock (_runningProcesses)
-                {
-                    _runningProcesses.Remove(process);
-                }
-            }
         }
 
         private void StopProcesses()

MediaBrowser.Model/Entities/MediaStream.cs

@@ -1,4 +1,5 @@
-using MediaBrowser.Model.Dlna;
+using System.Collections.Generic;
+using MediaBrowser.Model.Dlna;
 using MediaBrowser.Model.Extensions;
 using System.Diagnostics;
 
@@ -58,6 +59,12 @@ namespace MediaBrowser.Model.Entities
         /// <value>The length of the packet.</value>
         public int? PacketLength { get; set; }
 
+        /// <summary>
+        /// Gets or sets the key frames.
+        /// </summary>
+        /// <value>The key frames.</value>
+        public List<int> KeyFrames { get; set; }
+        
         /// <summary>
         /// Gets or sets the channels.
         /// </summary>

MediaBrowser.Providers/MediaInfo/FFProbeVideoInfo.cs

@@ -130,7 +130,7 @@ namespace MediaBrowser.Providers.MediaInfo
             return ItemUpdateType.MetadataImport;
         }
 
-        private const string SchemaVersion = "2";
+        private const string SchemaVersion = "3";
 
         private async Task<Model.MediaInfo.MediaInfo> GetMediaInfo(Video item,
             IIsoMount isoMount,
@@ -145,7 +145,7 @@ namespace MediaBrowser.Providers.MediaInfo
 
             try
             {
-                return _json.DeserializeFromFile<Model.MediaInfo.MediaInfo>(cachePath);
+                //return _json.DeserializeFromFile<Model.MediaInfo.MediaInfo>(cachePath);
             }
             catch (FileNotFoundException)
             {
@@ -167,7 +167,8 @@ namespace MediaBrowser.Providers.MediaInfo
                 VideoType = item.VideoType,
                 MediaType = DlnaProfileType.Video,
                 InputPath = item.Path,
-                Protocol = protocol
+                Protocol = protocol,
+                ExtractKeyFrameInterval = true
 
             }, cancellationToken).ConfigureAwait(false);
 

MediaBrowser.Server.Implementations/Persistence/SqliteMediaStreamsRepository.cs

@@ -1,4 +1,5 @@
-using MediaBrowser.Controller.Persistence;
+using System.Globalization;
+using MediaBrowser.Controller.Persistence;
 using MediaBrowser.Model.Entities;
 using MediaBrowser.Model.Logging;
 using System;
@@ -40,7 +41,7 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
             // Add PixelFormat column
 
-            createTableCommand += "(ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, IsCabac BIT NULL, PRIMARY KEY (ItemId, StreamIndex))";
+            createTableCommand += "(ItemId GUID, StreamIndex INT, StreamType TEXT, Codec TEXT, Language TEXT, ChannelLayout TEXT, Profile TEXT, AspectRatio TEXT, Path TEXT, IsInterlaced BIT, BitRate INT NULL, Channels INT NULL, SampleRate INT NULL, IsDefault BIT, IsForced BIT, IsExternal BIT, Height INT NULL, Width INT NULL, AverageFrameRate FLOAT NULL, RealFrameRate FLOAT NULL, Level FLOAT NULL, PixelFormat TEXT, BitDepth INT NULL, IsAnamorphic BIT NULL, RefFrames INT NULL, IsCabac BIT NULL, KeyFrames TEXT NULL, PRIMARY KEY (ItemId, StreamIndex))";
 
             string[] queries = {
 
@@ -61,6 +62,7 @@ namespace MediaBrowser.Server.Implementations.Persistence
             AddIsAnamorphicColumn();
             AddIsCabacColumn();
             AddRefFramesCommand();
+            AddKeyFramesCommand();
 
             PrepareStatements();
 
@@ -160,6 +162,37 @@ namespace MediaBrowser.Server.Implementations.Persistence
             _connection.RunQueries(new[] { builder.ToString() }, _logger);
         }
 
+        private void AddKeyFramesCommand()
+        {
+            using (var cmd = _connection.CreateCommand())
+            {
+                cmd.CommandText = "PRAGMA table_info(mediastreams)";
+
+                using (var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess | CommandBehavior.SingleResult))
+                {
+                    while (reader.Read())
+                    {
+                        if (!reader.IsDBNull(1))
+                        {
+                            var name = reader.GetString(1);
+
+                            if (string.Equals(name, "KeyFrames", StringComparison.OrdinalIgnoreCase))
+                            {
+                                return;
+                            }
+                        }
+                    }
+                }
+            }
+
+            var builder = new StringBuilder();
+
+            builder.AppendLine("alter table mediastreams");
+            builder.AppendLine("add column KeyFrames TEXT NULL");
+
+            _connection.RunQueries(new[] { builder.ToString() }, _logger);
+        }
+
         private void AddIsCabacColumn()
         {
             using (var cmd = _connection.CreateCommand())
@@ -249,6 +282,7 @@ namespace MediaBrowser.Server.Implementations.Persistence
             "BitDepth",
             "IsAnamorphic",
             "RefFrames",
+            "KeyFrames",
             "IsCabac"
         };
 
@@ -430,7 +464,12 @@ namespace MediaBrowser.Server.Implementations.Persistence
 
             if (!reader.IsDBNull(25))
             {
-                item.IsCabac = reader.GetBoolean(25);
+                item.KeyFrames = reader.GetString(25).Split(',').Where(i => !string.IsNullOrWhiteSpace(i)).Select(i => int.Parse(i, CultureInfo.InvariantCulture)).ToList();
+            }
+
+            if (!reader.IsDBNull(26))
+            {
+                item.IsCabac = reader.GetBoolean(26);
             }
 
             return item;
@@ -498,7 +537,15 @@ namespace MediaBrowser.Server.Implementations.Persistence
                     _saveStreamCommand.GetParameter(22).Value = stream.BitDepth;
                     _saveStreamCommand.GetParameter(23).Value = stream.IsAnamorphic;
                     _saveStreamCommand.GetParameter(24).Value = stream.RefFrames;
-                    _saveStreamCommand.GetParameter(25).Value = stream.IsCabac;
+                    if (stream.KeyFrames != null)
+                    {
+                        _saveStreamCommand.GetParameter(25).Value = string.Join(",", stream.KeyFrames.Select(i => i.ToString(CultureInfo.InvariantCulture)).ToArray());
+                    }
+                    else
+                    {
+                        _saveStreamCommand.GetParameter(25).Value = null;
+                    }
+                    _saveStreamCommand.GetParameter(26).Value = stream.IsCabac;
 
                     _saveStreamCommand.Transaction = transaction;
                     _saveStreamCommand.ExecuteNonQuery();