Merge pull request #2875 from MediaBrowser/dev

Dev

Emby.Server.Implementations/Data/SqliteItemRepository.cs

@@ -4254,6 +4254,54 @@ namespace Emby.Server.Implementations.Data
                 }
             }
 
+            if (!string.IsNullOrWhiteSpace(query.HasNoAudioTrackWithLanguage))
+            {
+                whereClauses.Add("((select language from MediaStreams where MediaStreams.ItemId=A.Guid and MediaStreams.StreamType='Audio' and MediaStreams.Language=@HasNoAudioTrackWithLanguage limit 1) is null)");
+                if (statement != null)
+                {
+                    statement.TryBind("@HasNoAudioTrackWithLanguage", query.HasNoAudioTrackWithLanguage);
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(query.HasNoInternalSubtitleTrackWithLanguage))
+            {
+                whereClauses.Add("((select language from MediaStreams where MediaStreams.ItemId=A.Guid and MediaStreams.StreamType='Subtitle' and MediaStreams.IsExternal=0 and MediaStreams.Language=@HasNoInternalSubtitleTrackWithLanguage limit 1) is null)");
+                if (statement != null)
+                {
+                    statement.TryBind("@HasNoInternalSubtitleTrackWithLanguage", query.HasNoInternalSubtitleTrackWithLanguage);
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(query.HasNoExternalSubtitleTrackWithLanguage))
+            {
+                whereClauses.Add("((select language from MediaStreams where MediaStreams.ItemId=A.Guid and MediaStreams.StreamType='Subtitle' and MediaStreams.IsExternal=1 and MediaStreams.Language=@HasNoExternalSubtitleTrackWithLanguage limit 1) is null)");
+                if (statement != null)
+                {
+                    statement.TryBind("@HasNoExternalSubtitleTrackWithLanguage", query.HasNoExternalSubtitleTrackWithLanguage);
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(query.HasNoSubtitleTrackWithLanguage))
+            {
+                whereClauses.Add("((select language from MediaStreams where MediaStreams.ItemId=A.Guid and MediaStreams.StreamType='Subtitle' and MediaStreams.Language=@HasNoSubtitleTrackWithLanguage limit 1) is null)");
+                if (statement != null)
+                {
+                    statement.TryBind("@HasNoSubtitleTrackWithLanguage", query.HasNoSubtitleTrackWithLanguage);
+                }
+            }
+
+            if (query.HasChapterImages.HasValue)
+            {
+                if (query.HasChapterImages.Value)
+                {
+                    whereClauses.Add("((select imagepath from Chapters2 where Chapters2.ItemId=A.Guid and imagepath not null limit 1) not null)");
+                }
+                else
+                {
+                    whereClauses.Add("((select imagepath from Chapters2 where Chapters2.ItemId=A.Guid and imagepath not null limit 1) is null)");
+                }
+            }
+
             if (query.HasDeadParentId.HasValue && query.HasDeadParentId.Value)
             {
                 whereClauses.Add("ParentId NOT NULL AND ParentId NOT IN (select guid from TypedBaseItems)");

Emby.Server.Implementations/Library/LibraryManager.cs

@@ -620,37 +620,12 @@ namespace Emby.Server.Implementations.Library
             return ResolveItem(args, resolvers);
         }
 
-        private readonly List<string> _ignoredPaths = new List<string>();
-
-        public void RegisterIgnoredPath(string path)
-        {
-            lock (_ignoredPaths)
-            {
-                _ignoredPaths.Add(path);
-            }
-        }
-        public void UnRegisterIgnoredPath(string path)
-        {
-            lock (_ignoredPaths)
-            {
-                _ignoredPaths.Remove(path);
-            }
-        }
-
         public bool IgnoreFile(FileSystemMetadata file, BaseItem parent)
         {
             if (EntityResolutionIgnoreRules.Any(r => r.ShouldIgnore(file, parent)))
             {
                 return true;
             }
-
-            //lock (_ignoredPaths)
-            {
-                if (_ignoredPaths.Contains(file.FullName, StringComparer.OrdinalIgnoreCase))
-                {
-                    return true;
-                }
-            }
             return false;
         }
 

Emby.Server.Implementations/LiveTv/RefreshChannelsScheduledTask.cs

@@ -51,7 +51,7 @@ namespace Emby.Server.Implementations.LiveTv
             return new[] { 
             
                 // Every so often
-                new TaskTriggerInfo { Type = TaskTriggerInfo.TriggerInterval, IntervalTicks = TimeSpan.FromHours(12).Ticks}
+                new TaskTriggerInfo { Type = TaskTriggerInfo.TriggerInterval, IntervalTicks = TimeSpan.FromHours(24).Ticks}
             };
         }
 

Emby.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunHttpStream.cs

@@ -26,10 +26,7 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts.HdHomerun
         private readonly CancellationTokenSource _liveStreamCancellationTokenSource = new CancellationTokenSource();
         private readonly TaskCompletionSource<bool> _liveStreamTaskCompletionSource = new TaskCompletionSource<bool>();
 
-        private readonly MulticastStream _multicastStream;
-
         private readonly string _tempFilePath;
-        private bool _enableFileBuffer = false;
 
         public HdHomerunHttpStream(MediaSourceInfo mediaSource, string originalStreamId, IFileSystem fileSystem, IHttpClient httpClient, ILogger logger, IServerApplicationPaths appPaths, IServerApplicationHost appHost, IEnvironmentInfo environment)
             : base(mediaSource, environment, fileSystem)
@@ -39,7 +36,6 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts.HdHomerun
             _appHost = appHost;
             OriginalStreamId = originalStreamId;
 
-            _multicastStream = new MulticastStream(_logger);
             _tempFilePath = Path.Combine(appPaths.TranscodingTempPath, UniqueId + ".ts");
         }
 
@@ -63,6 +59,9 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts.HdHomerun
 
             OpenedMediaSource.Path = _appHost.GetLocalApiUrl("127.0.0.1") + "/LiveTv/LiveStreamFiles/" + UniqueId + "/stream.ts";
             OpenedMediaSource.Protocol = MediaProtocol.Http;
+
+            //OpenedMediaSource.Path = _tempFilePath;
+            //OpenedMediaSource.Protocol = MediaProtocol.File;
             //OpenedMediaSource.SupportsDirectPlay = false;
             //OpenedMediaSource.SupportsDirectStream = true;
             //OpenedMediaSource.SupportsTranscoding = true;
@@ -107,19 +106,10 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts.HdHomerun
                             {
                                 _logger.Info("Beginning multicastStream.CopyUntilCancelled");
 
-                                if (_enableFileBuffer)
-                                {
-                                    FileSystem.CreateDirectory(FileSystem.GetDirectoryName(_tempFilePath));
-                                    using (var fileStream = FileSystem.GetFileStream(_tempFilePath, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, FileOpenOptions.None))
-                                    {
-                                        StreamHelper.CopyTo(response.Content, fileStream, 81920, () => Resolve(openTaskCompletionSource), cancellationToken);
-                                    }
-                                }
-                                else
+                                FileSystem.CreateDirectory(FileSystem.GetDirectoryName(_tempFilePath));
+                                using (var fileStream = FileSystem.GetFileStream(_tempFilePath, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, FileOpenOptions.None))
                                 {
-                                    Resolve(openTaskCompletionSource);
-
-                                    await _multicastStream.CopyUntilCancelled(response.Content, null, cancellationToken).ConfigureAwait(false);
+                                    StreamHelper.CopyTo(response.Content, fileStream, 81920, () => Resolve(openTaskCompletionSource), cancellationToken);
                                 }
                             }
                         }
@@ -158,7 +148,33 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts.HdHomerun
 
         public Task CopyToAsync(Stream stream, CancellationToken cancellationToken)
         {
-            return _multicastStream.CopyToAsync(stream, cancellationToken);
+            return CopyFileTo(_tempFilePath, stream, cancellationToken);
+        }
+
+        protected async Task CopyFileTo(string path, Stream outputStream, CancellationToken cancellationToken)
+        {
+            long startPosition = -20000;
+
+            _logger.Info("Live stream starting position is {0} bytes", startPosition.ToString(CultureInfo.InvariantCulture));
+
+            var allowAsync = false;//Environment.OperatingSystem != MediaBrowser.Model.System.OperatingSystem.Windows;
+            // use non-async filestream along with read due to https://github.com/dotnet/corefx/issues/6039
+
+            using (var inputStream = (FileStream)GetInputStream(path, allowAsync))
+            {
+                if (startPosition > 0)
+                {
+                    inputStream.Seek(-20000, SeekOrigin.End);
+                }
+
+                while (!cancellationToken.IsCancellationRequested)
+                {
+                    StreamHelper.CopyTo(inputStream, outputStream, 81920, cancellationToken);
+
+                    //var position = fs.Position;
+                    //_logger.Debug("Streamed {0} bytes to position {1} from file {2}", bytesRead, position, path);
+                }
+            }
         }
     }
 }

Emby.Server.Implementations/LiveTv/TunerHosts/HdHomerun/HdHomerunUdpStream.cs

@@ -34,8 +34,6 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts.HdHomerun
         private readonly INetworkManager _networkManager;
 
         private readonly string _tempFilePath;
-        private bool _enableFileBuffer = false;
-        private readonly MulticastStream _multicastStream;
 
         public HdHomerunUdpStream(MediaSourceInfo mediaSource, string originalStreamId, IHdHomerunChannelCommands channelCommands, int numTuners, IFileSystem fileSystem, IHttpClient httpClient, ILogger logger, IServerApplicationPaths appPaths, IServerApplicationHost appHost, ISocketFactory socketFactory, INetworkManager networkManager, IEnvironmentInfo environment)
             : base(mediaSource, environment, fileSystem)
@@ -48,7 +46,6 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts.HdHomerun
             _channelCommands = channelCommands;
             _numTuners = numTuners;
             _tempFilePath = Path.Combine(appPaths.TranscodingTempPath, UniqueId + ".ts");
-            _multicastStream = new MulticastStream(_logger);
         }
 
         protected override async Task OpenInternal(CancellationToken openCancellationToken)
@@ -126,17 +123,10 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts.HdHomerun
 
                                 if (!cancellationToken.IsCancellationRequested)
                                 {
-                                    if (_enableFileBuffer)
+                                    FileSystem.CreateDirectory(FileSystem.GetDirectoryName(_tempFilePath));
+                                    using (var fileStream = FileSystem.GetFileStream(_tempFilePath, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, FileOpenOptions.None))
                                     {
-                                        FileSystem.CreateDirectory(FileSystem.GetDirectoryName(_tempFilePath));
-                                        using (var fileStream = FileSystem.GetFileStream(_tempFilePath, FileOpenMode.Create, FileAccessMode.Write, FileShareMode.Read, FileOpenOptions.None))
-                                        {
-                                            CopyTo(udpClient, fileStream, openTaskCompletionSource, cancellationToken);
-                                        }
-                                    }
-                                    else
-                                    {
-                                        await _multicastStream.CopyUntilCancelled(new UdpClientStream(udpClient), () => Resolve(openTaskCompletionSource), cancellationToken).ConfigureAwait(false);
+                                        CopyTo(udpClient, fileStream, openTaskCompletionSource, cancellationToken);
                                     }
                                 }
                             }
@@ -178,49 +168,33 @@ namespace Emby.Server.Implementations.LiveTv.TunerHosts.HdHomerun
            });
         }
 
-        public async Task CopyToAsync(Stream outputStream, CancellationToken cancellationToken)
+        public Task CopyToAsync(Stream stream, CancellationToken cancellationToken)
         {
-            if (!_enableFileBuffer)
-            {
-                await _multicastStream.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false);
-                return;
-            }
-
-            var path = _tempFilePath;
+            return CopyFileTo(_tempFilePath, stream, cancellationToken);
+        }
 
+        protected async Task CopyFileTo(string path, Stream outputStream, CancellationToken cancellationToken)
+        {
             long startPosition = -20000;
-            if (startPosition < 0)
-            {
-                var length = FileSystem.GetFileInfo(path).Length;
-                startPosition = Math.Max(length - startPosition, 0);
-            }
 
             _logger.Info("Live stream starting position is {0} bytes", startPosition.ToString(CultureInfo.InvariantCulture));
 
-            var allowAsync = Environment.OperatingSystem != MediaBrowser.Model.System.OperatingSystem.Windows;
+            var allowAsync = false;//Environment.OperatingSystem != MediaBrowser.Model.System.OperatingSystem.Windows;
             // use non-async filestream along with read due to https://github.com/dotnet/corefx/issues/6039
 
-            using (var inputStream = GetInputStream(path, startPosition, allowAsync))
+            using (var inputStream = (FileStream)GetInputStream(path, allowAsync))
             {
                 if (startPosition > 0)
                 {
-                    inputStream.Position = startPosition;
+                    inputStream.Seek(-20000, SeekOrigin.End);
                 }
 
                 while (!cancellationToken.IsCancellationRequested)
                 {
-                    long bytesRead;
-
                     StreamHelper.CopyTo(inputStream, outputStream, 81920, cancellationToken);
-                    bytesRead = 1;
 
                     //var position = fs.Position;
                     //_logger.Debug("Streamed {0} bytes to position {1} from file {2}", bytesRead, position, path);
-
-                    if (bytesRead == 0)
-                    {
-                        await Task.Delay(100, cancellationToken).ConfigureAwait(false);
-                    }
                 }
             }
         }

Emby.Server.Implementations/ScheduledTasks/ChapterImagesTask.cs

@@ -58,8 +58,8 @@ namespace Emby.Server.Implementations.ScheduledTasks
         /// </summary>
         public IEnumerable<TaskTriggerInfo> GetDefaultTriggers()
         {
-            return new[] { 
-            
+            return new[] {
+
                 new TaskTriggerInfo
                 {
                     Type = TaskTriggerInfo.TriggerDaily,
@@ -88,6 +88,12 @@ namespace Emby.Server.Implementations.ScheduledTasks
                 IsFolder = false,
                 Recursive = true,
                 DtoOptions = new DtoOptions(false)
+                {
+                    EnableImages = false
+                },
+                SourceTypes = new SourceType[] { SourceType.Library },
+                HasChapterImages = false,
+                IsVirtualItem = false
 
             })
                 .OfType<Video>()

Emby.Server.Implementations/Services/SwaggerService.cs

@@ -193,13 +193,11 @@ namespace Emby.Server.Implementations.Services
                     summary = info.Summary,
                     produces = new[]
                     {
-                        "application/json",
-                        "application/xml"
+                        "application/json"
                     },
                     consumes = new[]
                     {
-                        "application/json",
-                        "application/xml"
+                        "application/json"
                     },
                     operationId = info.RequestType.Name,
                     tags = new string[] { },

MediaBrowser.Controller/Entities/InternalItemsQuery.cs

@@ -160,6 +160,7 @@ namespace MediaBrowser.Controller.Entities
         public bool ForceDirect { get; set; }
         public Dictionary<string, string> ExcludeProviderIds { get; set; }
         public bool EnableGroupByMetadataKey { get; set; }
+        public bool? HasChapterImages { get; set; }
 
         public Tuple<string, SortOrder>[] OrderBy { get; set; }
 
@@ -169,6 +170,10 @@ namespace MediaBrowser.Controller.Entities
 
         public DtoOptions DtoOptions { get; set; }
         public int MinSimilarityScore { get; set; }
+        public string HasNoAudioTrackWithLanguage { get; set; }
+        public string HasNoInternalSubtitleTrackWithLanguage { get; set; }
+        public string HasNoExternalSubtitleTrackWithLanguage { get; set; }
+        public string HasNoSubtitleTrackWithLanguage { get; set; }
 
         public InternalItemsQuery()
         {

MediaBrowser.Controller/Library/ILibraryManager.cs

@@ -563,8 +563,6 @@ namespace MediaBrowser.Controller.Library
         QueryResult<Tuple<BaseItem, ItemCounts>> GetAlbumArtists(InternalItemsQuery query);
         QueryResult<Tuple<BaseItem, ItemCounts>> GetAllArtists(InternalItemsQuery query);
 
-        void RegisterIgnoredPath(string path);
-        void UnRegisterIgnoredPath(string path);
         int GetCount(InternalItemsQuery query);
     }
 }

MediaBrowser.Controller/LiveTv/LiveStream.cs

@@ -51,7 +51,7 @@ namespace MediaBrowser.Controller.LiveTv
             return Task.FromResult(true);
         }
 
-        protected Stream GetInputStream(string path, long startPosition, bool allowAsyncFileRead)
+        protected Stream GetInputStream(string path, bool allowAsyncFileRead)
         {
             var fileOpenOptions = FileOpenOptions.SequentialScan;
 

MediaBrowser.Providers/MediaInfo/SubtitleDownloader.cs

@@ -32,16 +32,40 @@ namespace MediaBrowser.Providers.MediaInfo
             bool requirePerfectMatch,
             IEnumerable<string> languages,
             CancellationToken cancellationToken)
+        {
+            var downloadedLanguages = new List<string>();
+
+            foreach (var lang in languages)
+            {
+                var downloaded = await DownloadSubtitles(video, mediaStreams, skipIfEmbeddedSubtitlesPresent,
+                    skipIfAudioTrackMatches, requirePerfectMatch, lang, cancellationToken).ConfigureAwait(false);
+
+                if (downloaded)
+                {
+                    downloadedLanguages.Add(lang);
+                }
+            }
+
+            return downloadedLanguages;
+        }
+
+        public Task<bool> DownloadSubtitles(Video video,
+            List<MediaStream> mediaStreams,
+            bool skipIfEmbeddedSubtitlesPresent,
+            bool skipIfAudioTrackMatches,
+            bool requirePerfectMatch,
+            string lang,
+            CancellationToken cancellationToken)
         {
             if (video.LocationType != LocationType.FileSystem ||
                 video.VideoType != VideoType.VideoFile)
             {
-                return new List<string>();
+                return Task.FromResult(false);
             }
 
             if (!video.IsCompleteMedia)
             {
-                return new List<string>();
+                return Task.FromResult(false);
             }
 
             VideoContentType mediaType;
@@ -57,30 +81,11 @@ namespace MediaBrowser.Providers.MediaInfo
             else
             {
                 // These are the only supported types
-                return new List<string>();
+                return Task.FromResult(false);
             }
 
-            var downloadedLanguages = new List<string>();
-
-            foreach (var lang in languages)
-            {
-                try
-                {
-                    var downloaded = await DownloadSubtitles(video, mediaStreams, skipIfEmbeddedSubtitlesPresent, skipIfAudioTrackMatches, requirePerfectMatch, lang, mediaType, cancellationToken)
-                        .ConfigureAwait(false);
-
-                    if (downloaded)
-                    {
-                        downloadedLanguages.Add(lang);
-                    }
-                }
-                catch (Exception ex)
-                {
-                    _logger.ErrorException("Error downloading subtitles", ex);
-                }
-            }
-
-            return downloadedLanguages;
+            return DownloadSubtitles(video, mediaStreams, skipIfEmbeddedSubtitlesPresent, skipIfAudioTrackMatches,
+                requirePerfectMatch, lang, mediaType, cancellationToken);
         }
 
         private async Task<bool> DownloadSubtitles(Video video,

MediaBrowser.Providers/MediaInfo/SubtitleScheduledTask.cs

@@ -80,17 +80,44 @@ namespace MediaBrowser.Providers.MediaInfo
                 return;
             }
 
-            var videos = _libraryManager.GetItemList(new InternalItemsQuery
+            var dict = new Dictionary<Guid, BaseItem>();
+
+            foreach (var lang in options.DownloadLanguages)
             {
-                MediaTypes = new string[] { MediaType.Video },
-                IsVirtualItem = false,
-                IncludeItemTypes = types.ToArray(types.Count),
-                DtoOptions = new DtoOptions(true)
+                var query = new InternalItemsQuery
+                {
+                    MediaTypes = new string[] {MediaType.Video},
+                    IsVirtualItem = false,
+                    IncludeItemTypes = types.ToArray(types.Count),
+                    DtoOptions = new DtoOptions(true),
+                    SourceTypes = new[] {SourceType.Library}
+                };
+
+                if (options.SkipIfAudioTrackMatches)
+                {
+                    query.HasNoAudioTrackWithLanguage = lang;
+                }
+
+                if (options.SkipIfEmbeddedSubtitlesPresent)
+                {
+                    // Exclude if it already has any subtitles of the same language
+                    query.HasNoSubtitleTrackWithLanguage = lang;
+                }
+                else
+                {
+                    // Exclude if it already has external subtitles of the same language
+                    query.HasNoExternalSubtitleTrackWithLanguage = lang;
+                }
 
-            }).OfType<Video>()
-                .Where(i => i.LocationType != LocationType.Remote)
-                .ToList();
+                var videosByLanguage = _libraryManager.GetItemList(query);
 
+                foreach (var video in videosByLanguage)
+                {
+                    dict[video.Id] = video;
+                }
+            }
+
+            var videos = dict.Values.ToList();
             if (videos.Count == 0)
             {
                 return;
@@ -100,9 +127,11 @@ namespace MediaBrowser.Providers.MediaInfo
 
             foreach (var video in videos)
             {
+                cancellationToken.ThrowIfCancellationRequested();
+
                 try
                 {
-                    await DownloadSubtitles(video, options, cancellationToken).ConfigureAwait(false);
+                    await DownloadSubtitles(video as Video, options, cancellationToken).ConfigureAwait(false);
                 }
                 catch (Exception ex)
                 {

SharedVersion.cs

@@ -1,3 +1,3 @@
 using System.Reflection;
 
-[assembly: AssemblyVersion("3.2.30.13")]
+[assembly: AssemblyVersion("3.2.30.14")]