separate metadata refresh from validation

MediaBrowser.Controller/Entities/Audio/MusicArtist.cs

@@ -1,4 +1,5 @@
-using MediaBrowser.Model.Configuration;
+using MediaBrowser.Controller.Providers;
+using MediaBrowser.Model.Configuration;
 using MediaBrowser.Model.Dto;
 using MediaBrowser.Model.Entities;
 using System;
@@ -49,7 +50,7 @@ namespace MediaBrowser.Controller.Entities.Audio
         }
 
         private readonly Task _cachedTask = Task.FromResult(true);
-        protected override Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
+        protected override Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions)
         {
             if (IsAccessedByName)
             {
@@ -57,7 +58,7 @@ namespace MediaBrowser.Controller.Entities.Audio
                 return _cachedTask;
             }
 
-            return base.ValidateChildrenInternal(progress, cancellationToken, recursive, forceRefreshMetadata);
+            return base.ValidateChildrenInternal(progress, cancellationToken, recursive, refreshChildMetadata, refreshOptions);
         }
 
         public override string GetClientTypeName()

MediaBrowser.Controller/Entities/BaseItem.cs

@@ -472,75 +472,30 @@ namespace MediaBrowser.Controller.Entities
         /// <returns>List{Video}.</returns>
         private IEnumerable<Trailer> LoadLocalTrailers(List<FileSystemInfo> fileSystemChildren)
         {
-            return new List<Trailer>();
-            //ItemResolveArgs resolveArgs;
-
-            //try
-            //{
-            //    resolveArgs = ResolveArgs;
-
-            //    if (!resolveArgs.IsDirectory)
-            //    {
-            //        return new List<Trailer>();
-            //    }
-            //}
-            //catch (IOException ex)
-            //{
-            //    Logger.ErrorException("Error getting ResolveArgs for {0}", ex, Path);
-            //    return new List<Trailer>();
-            //}
-
-            //var files = new List<FileSystemInfo>();
-
-            //var folder = resolveArgs.GetFileSystemEntryByName(TrailerFolderName);
-
-            //// Path doesn't exist. No biggie
-            //if (folder != null)
-            //{
-            //    try
-            //    {
-            //        files.AddRange(new DirectoryInfo(folder.FullName).EnumerateFiles());
-            //    }
-            //    catch (IOException ex)
-            //    {
-            //        Logger.ErrorException("Error loading trailers for {0}", ex, Name);
-            //    }
-            //}
-
-            //// Support xbmc trailers (-trailer suffix on video file names)
-            //files.AddRange(resolveArgs.FileSystemChildren.Where(i =>
-            //{
-            //    try
-            //    {
-            //        if ((i.Attributes & FileAttributes.Directory) != FileAttributes.Directory)
-            //        {
-            //            if (System.IO.Path.GetFileNameWithoutExtension(i.Name).EndsWith(XbmcTrailerFileSuffix, StringComparison.OrdinalIgnoreCase) && !string.Equals(Path, i.FullName, StringComparison.OrdinalIgnoreCase))
-            //            {
-            //                return true;
-            //            }
-            //        }
-            //    }
-            //    catch (IOException ex)
-            //    {
-            //        Logger.ErrorException("Error accessing path {0}", ex, i.FullName);
-            //    }
-
-            //    return false;
-            //}));
-
-            //return LibraryManager.ResolvePaths<Trailer>(files, null).Select(video =>
-            //{
-            //    // Try to retrieve it from the db. If we don't find it, use the resolved version
-            //    var dbItem = LibraryManager.GetItemById(video.Id) as Trailer;
-
-            //    if (dbItem != null)
-            //    {
-            //        video = dbItem;
-            //    }
-
-            //    return video;
-
-            //}).ToList();
+            var files = fileSystemChildren.OfType<DirectoryInfo>()
+                .Where(i => string.Equals(i.Name, TrailerFolderName, StringComparison.OrdinalIgnoreCase))
+                .SelectMany(i => i.EnumerateFiles("*", SearchOption.TopDirectoryOnly))
+                .ToList();
+
+            // Support plex/xbmc convention
+            files.AddRange(fileSystemChildren.OfType<FileInfo>()
+                .Where(i => System.IO.Path.GetFileNameWithoutExtension(i.Name).EndsWith(XbmcTrailerFileSuffix, StringComparison.OrdinalIgnoreCase) && !string.Equals(Path, i.FullName, StringComparison.OrdinalIgnoreCase))
+                );
+
+            return LibraryManager.ResolvePaths<Trailer>(files, null).Select(video =>
+            {
+                // Try to retrieve it from the db. If we don't find it, use the resolved version
+                var dbItem = LibraryManager.GetItemById(video.Id) as Trailer;
+
+                if (dbItem != null)
+                {
+                    video = dbItem;
+                }
+
+                return video;
+
+                // Sort them so that the list can be easily compared for changes
+            }).OrderBy(i => i.Path).ToList();
         }
 
         /// <summary>
@@ -656,19 +611,8 @@ namespace MediaBrowser.Controller.Entities
                 }
             }
             
-            if (themeSongsChanged)
-            {
-                Logger.Debug("Theme songs have changed for {0}", Path);
-                options.ForceSave = true;
-            }
-            if (themeVideosChanged)
-            {
-                Logger.Debug("Theme videos have changed for {0}", Path);
-                options.ForceSave = true;
-            }
-            if (localTrailersChanged)
+            if (themeSongsChanged || themeVideosChanged || localTrailersChanged)
             {
-                Logger.Debug("Local trailers have changed for {0}", Path);
                 options.ForceSave = true;
             }
         }

MediaBrowser.Controller/Entities/CollectionFolder.cs

@@ -7,6 +7,7 @@ using System.Linq;
 using System.Runtime.Serialization;
 using System.Threading;
 using System.Threading.Tasks;
+using MediaBrowser.Controller.Providers;
 
 namespace MediaBrowser.Controller.Entities
 {
@@ -116,9 +117,10 @@ namespace MediaBrowser.Controller.Entities
         /// <param name="progress">The progress.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <param name="recursive">if set to <c>true</c> [recursive].</param>
-        /// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
+        /// <param name="refreshChildMetadata">if set to <c>true</c> [refresh child metadata].</param>
+        /// <param name="refreshOptions">The refresh options.</param>
         /// <returns>Task.</returns>
-        protected override Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
+        protected override Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions)
         {
             CreateResolveArgs();
             ResetDynamicChildren();

MediaBrowser.Controller/Entities/Folder.cs

@@ -307,7 +307,17 @@ namespace MediaBrowser.Controller.Entities
         /// <param name="recursive">if set to <c>true</c> [recursive].</param>
         /// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
         /// <returns>Task.</returns>
-        public async Task ValidateChildren(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
+        public Task ValidateChildren(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
+        {
+            return ValidateChildrenWithCancellationSupport(progress, cancellationToken, recursive ?? true, true,
+
+                new MetadataRefreshOptions
+                {
+                    ReplaceAllMetadata = forceRefreshMetadata
+                });
+        }
+
+        private async Task ValidateChildrenWithCancellationSupport(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions)
         {
             cancellationToken.ThrowIfCancellationRequested();
 
@@ -327,7 +337,7 @@ namespace MediaBrowser.Controller.Entities
 
                 var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(innerCancellationTokenSource.Token, cancellationToken);
 
-                await ValidateChildrenInternal(progress, linkedCancellationTokenSource.Token, recursive, forceRefreshMetadata).ConfigureAwait(false);
+                await ValidateChildrenInternal(progress, linkedCancellationTokenSource.Token, recursive, refreshChildMetadata, refreshOptions).ConfigureAwait(false);
             }
             catch (OperationCanceledException ex)
             {
@@ -352,15 +362,15 @@ namespace MediaBrowser.Controller.Entities
         }
 
         /// <summary>
-        /// Compare our current children (presumably just read from the repo) with the current state of the file system and adjust for any changes
-        /// ***Currently does not contain logic to maintain items that are unavailable in the file system***
+        /// Validates the children internal.
         /// </summary>
         /// <param name="progress">The progress.</param>
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <param name="recursive">if set to <c>true</c> [recursive].</param>
-        /// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
+        /// <param name="refreshChildMetadata">if set to <c>true</c> [refresh child metadata].</param>
+        /// <param name="refreshOptions">The refresh options.</param>
         /// <returns>Task.</returns>
-        protected async virtual Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
+        protected async virtual Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions)
         {
             var locationType = LocationType;
 
@@ -401,33 +411,23 @@ namespace MediaBrowser.Controller.Entities
 
                     if (currentChildren.TryGetValue(child.Id, out currentChild))
                     {
-                        //existing item - check if it has changed
-                        if (currentChild.HasChanged(child))
+                        var currentChildLocationType = currentChild.LocationType;
+                        if (currentChildLocationType != LocationType.Remote &&
+                            currentChildLocationType != LocationType.Virtual)
                         {
-                            var currentChildLocationType = currentChild.LocationType;
-                            if (currentChildLocationType != LocationType.Remote &&
-                                currentChildLocationType != LocationType.Virtual)
-                            {
-                                currentChild.DateModified = child.DateModified;
-                            }
-
-                            currentChild.IsInMixedFolder = child.IsInMixedFolder;
-                            validChildren.Add(currentChild);
-                        }
-                        else
-                        {
-                            validChildren.Add(currentChild);
+                            currentChild.DateModified = child.DateModified;
                         }
 
+                        currentChild.IsInMixedFolder = child.IsInMixedFolder;
                         currentChild.IsOffline = false;
                     }
                     else
                     {
                         //brand new item - needs to be added
                         newItems.Add(child);
-
-                        validChildren.Add(child);
                     }
+
+                    validChildren.Add(currentChild);
                 }
 
                 // If any items were added or removed....
@@ -435,7 +435,6 @@ namespace MediaBrowser.Controller.Entities
                 {
                     // That's all the new and changed ones - now see if there are any that are missing
                     var itemsRemoved = currentChildren.Values.Except(validChildren).ToList();
-
                     var actualRemovals = new List<BaseItem>();
 
                     foreach (var item in itemsRemoved)
@@ -450,7 +449,6 @@ namespace MediaBrowser.Controller.Entities
                         else if (!string.IsNullOrEmpty(item.Path) && IsPathOffline(item.Path))
                         {
                             item.IsOffline = true;
-
                             validChildren.Add(item);
                         }
                         else
@@ -486,78 +484,135 @@ namespace MediaBrowser.Controller.Entities
 
             cancellationToken.ThrowIfCancellationRequested();
 
-            await RefreshChildren(validChildren, progress, cancellationToken, recursive, forceRefreshMetadata).ConfigureAwait(false);
+            if (recursive)
+            {
+                await ValidateSubFolders(validChildren.OfType<Folder>().ToList(), progress, cancellationToken).ConfigureAwait(false);
+            }
+
+            progress.Report(20);
+
+            if (refreshChildMetadata)
+            {
+                var container = this as IMetadataContainer;
+
+                var innerProgress = new ActionableProgress<double>();
+
+                innerProgress.RegisterAction(p => progress.Report((.80 * p) + 20));
+
+                if (container != null)
+                {
+                    await container.RefreshAllMetadata(refreshOptions, innerProgress, cancellationToken).ConfigureAwait(false);
+                }
+                else
+                {
+                    await RefreshMetadataRecursive(refreshOptions, recursive, innerProgress, cancellationToken);
+                }
+            }
 
             progress.Report(100);
         }
 
-        /// <summary>
-        /// Refreshes the children.
-        /// </summary>
-        /// <param name="children">The children.</param>
-        /// <param name="progress">The progress.</param>
-        /// <param name="cancellationToken">The cancellation token.</param>
-        /// <param name="recursive">if set to <c>true</c> [recursive].</param>
-        /// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
-        /// <returns>Task.</returns>
-        private async Task RefreshChildren(IList<BaseItem> children, IProgress<double> progress, CancellationToken cancellationToken, bool? recursive, bool forceRefreshMetadata = false)
+        private async Task RefreshMetadataRecursive(MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            var list = children;
+            var children = ActualChildren.ToList();
 
-            var percentages = new Dictionary<Guid, double>(list.Count);
+            var percentages = new Dictionary<Guid, double>(children.Count);
 
             var tasks = new List<Task>();
 
-            foreach (var tuple in list)
+            foreach (var child in children)
             {
-                if (tasks.Count > 10)
+                if (tasks.Count > 3)
                 {
                     await Task.WhenAll(tasks).ConfigureAwait(false);
+                    tasks.Clear();
                 }
 
-                tasks.Add(RefreshChild(tuple, progress, percentages, list.Count, cancellationToken, recursive, forceRefreshMetadata));
-            }
+                cancellationToken.ThrowIfCancellationRequested();
+                var innerProgress = new ActionableProgress<double>();
+                
+                // Avoid implicitly captured closure
+                var currentChild = child;
+                innerProgress.RegisterAction(p =>
+                {
+                    lock (percentages)
+                    {
+                        percentages[currentChild.Id] = p / 100;
 
-            cancellationToken.ThrowIfCancellationRequested();
+                        var percent = percentages.Values.Sum();
+                        percent /= children.Count;
+                        percent *= 100;
+                        progress.Report(percent);
+                    }
+                });
+
+                if (child.IsFolder)
+                {
+                    await RefreshChildMetadata(child, refreshOptions, recursive, innerProgress, cancellationToken)
+                      .ConfigureAwait(false);
+                }
+                else
+                {
+                    tasks.Add(RefreshChildMetadata(child, refreshOptions, recursive, innerProgress, cancellationToken));
+                }
+            }
 
             await Task.WhenAll(tasks).ConfigureAwait(false);
+            progress.Report(100);
         }
 
-        private async Task RefreshChild(BaseItem item, IProgress<double> progress, Dictionary<Guid, double> percentages, int childCount, CancellationToken cancellationToken, bool? recursive, bool forceRefreshMetadata = false)
+        private async Task RefreshChildMetadata(BaseItem child, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
         {
-            cancellationToken.ThrowIfCancellationRequested();
+            var container = child as IMetadataContainer;
 
-            var child = item;
-            try
+            if (container != null)
             {
-                //refresh it
-                await child.RefreshMetadata(new MetadataRefreshOptions
-                {
-                    ReplaceAllMetadata = forceRefreshMetadata
-
-                }, cancellationToken).ConfigureAwait(false);
+                await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false);
             }
-            catch (IOException ex)
+            else
             {
-                Logger.ErrorException("Error refreshing {0}", ex, child.Path ?? child.Name);
+                await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false);
+
+                if (recursive)
+                {
+                    var folder = child as Folder;
+
+                    if (folder != null)
+                    {
+                        await folder.RefreshMetadataRecursive(refreshOptions, true, progress, cancellationToken);
+                    }
+                }
             }
+            progress.Report(100);
+        }
+
+        /// <summary>
+        /// Refreshes the children.
+        /// </summary>
+        /// <param name="children">The children.</param>
+        /// <param name="progress">The progress.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        private async Task ValidateSubFolders(IList<Folder> children, IProgress<double> progress, CancellationToken cancellationToken)
+        {
+            var list = children;
+            var childCount = list.Count;
 
-            // Refresh children if a folder and the item changed or recursive is set to true
-            var refreshChildren = child.IsFolder;
+            var percentages = new Dictionary<Guid, double>(list.Count);
+
+            var tasks = new List<Task>();
 
-            if (refreshChildren)
+            foreach (var item in list)
             {
-                // Don't refresh children if explicitly set to false
-                if (recursive.HasValue && recursive.Value == false)
+                if (tasks.Count > 10)
                 {
-                    refreshChildren = false;
+                    await Task.WhenAll(tasks).ConfigureAwait(false);
                 }
-            }
 
-            if (refreshChildren)
-            {
                 cancellationToken.ThrowIfCancellationRequested();
 
+                var child = item;
+
                 var innerProgress = new ActionableProgress<double>();
 
                 innerProgress.RegisterAction(p =>
@@ -569,24 +624,16 @@ namespace MediaBrowser.Controller.Entities
                         var percent = percentages.Values.Sum();
                         percent /= childCount;
 
-                        progress.Report((90 * percent) + 10);
+                        progress.Report((10 * percent) + 10);
                     }
                 });
 
-                await ((Folder)child).ValidateChildren(innerProgress, cancellationToken, recursive, forceRefreshMetadata).ConfigureAwait(false);
+                tasks.Add(child.ValidateChildrenWithCancellationSupport(innerProgress, cancellationToken, true, false, null));
             }
-            else
-            {
-                lock (percentages)
-                {
-                    percentages[child.Id] = 1;
 
-                    var percent = percentages.Values.Sum();
-                    percent /= childCount;
+            cancellationToken.ThrowIfCancellationRequested();
 
-                    progress.Report((90 * percent) + 10);
-                }
-            }
+            await Task.WhenAll(tasks).ConfigureAwait(false);
         }
 
         /// <summary>
@@ -962,11 +1009,11 @@ namespace MediaBrowser.Controller.Entities
         /// <returns>Task.</returns>
         public override async Task ChangedExternally()
         {
-            await base.ChangedExternally().ConfigureAwait(false);
-
             var progress = new Progress<double>();
 
             await ValidateChildren(progress, CancellationToken.None).ConfigureAwait(false);
+
+            await base.ChangedExternally().ConfigureAwait(false);
         }
 
         /// <summary>

MediaBrowser.Controller/Entities/IMetadataContainer.cs

@@ -0,0 +1,19 @@
+using MediaBrowser.Controller.Providers;
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace MediaBrowser.Controller.Entities
+{
+    public interface IMetadataContainer
+    {
+        /// <summary>
+        /// Refreshes all metadata.
+        /// </summary>
+        /// <param name="refreshOptions">The refresh options.</param>
+        /// <param name="progress">The progress.</param>
+        /// <param name="cancellationToken">The cancellation token.</param>
+        /// <returns>Task.</returns>
+        Task RefreshAllMetadata(MetadataRefreshOptions refreshOptions, IProgress<double> progress, CancellationToken cancellationToken);
+    }
+}

MediaBrowser.Controller/Entities/User.cs

@@ -215,8 +215,9 @@ namespace MediaBrowser.Controller.Entities
 
             return RefreshMetadata(new MetadataRefreshOptions
             {
-                ForceSave = true,
-                ReplaceAllMetadata = true
+                ReplaceAllMetadata = true,
+                ImageRefreshMode = ImageRefreshMode.FullRefresh,
+                MetadataRefreshMode = MetadataRefreshMode.FullRefresh
 
             }, CancellationToken.None);
         }

MediaBrowser.Controller/MediaBrowser.Controller.csproj

@@ -103,6 +103,7 @@
     <Compile Include="Entities\IItemByName.cs" />
     <Compile Include="Entities\ILibraryItem.cs" />
     <Compile Include="Entities\ImageSourceInfo.cs" />
+    <Compile Include="Entities\IMetadataContainer.cs" />
     <Compile Include="Entities\LinkedChild.cs" />
     <Compile Include="Entities\MusicVideo.cs" />
     <Compile Include="Entities\IHasAwards.cs" />

MediaBrowser.Providers/Manager/MetadataService.cs

@@ -197,6 +197,11 @@ namespace MediaBrowser.Providers.Manager
                     .Where(i => i.HasChanged(currentItem, currentItem.DateLastSaved))
                     .ToList();
 
+                if (providersWithChanges.Count > 0)
+                {
+                    var b = true;
+                }
+
                 // If local providers are the only ones with changes, then just run those
                 if (providersWithChanges.All(i => i is ILocalMetadataProvider))
                 {

MediaBrowser.Providers/TV/SeriesMetadataService.cs

@@ -51,12 +51,15 @@ namespace MediaBrowser.Providers.TV
 
             var dateLastEpisodeAdded = item.DateLastEpisodeAdded;
 
-            item.DateLastEpisodeAdded = episodes.Select(i => i.DateCreated)
+            item.DateLastEpisodeAdded = episodes
+                .Where(i => i.LocationType != LocationType.Virtual)
+                .Select(i => i.DateCreated)
                 .OrderByDescending(i => i)
                 .FirstOrDefault();
 
             if (dateLastEpisodeAdded != item.DateLastEpisodeAdded)
             {
+                Logger.Debug("DateLastEpisodeAdded changed for {0}", item.Path);
                 updateType = updateType | ItemUpdateType.MetadataImport;
             }
 

MediaBrowser.Providers/TV/TvdbSeriesProvider.cs

@@ -1088,30 +1088,15 @@ namespace MediaBrowser.Providers.TV
             {
                 var seriesDataPath = GetSeriesDataPath(_config.ApplicationPaths, seriesId);
 
-                try
-                {
-                    var files = new DirectoryInfo(seriesDataPath).EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly)
-                        .ToList();
-
-                    var seriesXmlFilename = item.GetPreferredMetadataLanguage() + ".xml";
+                var seriesXmlFilename = item.GetPreferredMetadataLanguage() + ".xml";
 
-                    var seriesFile = files.FirstOrDefault(i => string.Equals(seriesXmlFilename, i.Name, StringComparison.OrdinalIgnoreCase));
-
-                    if (seriesFile != null && seriesFile.Exists && _fileSystem.GetLastWriteTimeUtc(seriesFile) > date)
-                    {
-                        return true;
-                    }
+                var filePath = Path.Combine(seriesDataPath, seriesXmlFilename);
 
-                    var actorsXml = files.FirstOrDefault(i => string.Equals("actors.xml", i.Name, StringComparison.OrdinalIgnoreCase));
+                var seriesFile = new FileInfo(filePath);
 
-                    if (actorsXml != null && actorsXml.Exists && _fileSystem.GetLastWriteTimeUtc(actorsXml) > date)
-                    {
-                        return true;
-                    }
-                }
-                catch (DirectoryNotFoundException)
+                if (seriesFile.Exists && _fileSystem.GetLastWriteTimeUtc(seriesFile) > date)
                 {
-                    // Don't blow up
+                    return true;
                 }
             }
 

MediaBrowser.Server.Implementations/Library/LibraryManager.cs

@@ -1110,7 +1110,6 @@ namespace MediaBrowser.Server.Implementations.Library
             cancellationToken.ThrowIfCancellationRequested();
 
             await userRootFolder.ValidateChildren(new Progress<double>(), cancellationToken, recursive: false).ConfigureAwait(false);
-            var b = true;
         }
 
         /// <summary>