Merge pull request #1067 from MediaBrowser/master

add bulk sync remove

MediaBrowser.Api/Sync/SyncService.cs

@@ -65,6 +65,16 @@ namespace MediaBrowser.Api.Sync
         public string Id { get; set; }
     }
 
+    [Route("/Sync/{TargetId}/Items", "DELETE", Summary = "Cancels items from a sync target")]
+    public class CancelItems : IReturnVoid
+    {
+        [ApiMember(Name = "TargetId", Description = "TargetId", IsRequired = false, DataType = "string", ParameterType = "query", Verb = "Items")]
+        public string TargetId { get; set; }
+
+        [ApiMember(Name = "ItemIds", Description = "ItemIds", IsRequired = false, DataType = "string", ParameterType = "query", Verb = "Items")]
+        public string ItemIds { get; set; }
+    }
+
     [Route("/Sync/Jobs", "GET", Summary = "Gets sync jobs.")]
     public class GetSyncJobs : SyncJobQuery, IReturn<QueryResult<SyncJob>>
     {
@@ -200,6 +210,15 @@ namespace MediaBrowser.Api.Sync
             return ToOptimizedResult(result);
         }
 
+        public void Delete(CancelItems request)
+        {
+            var itemIds = request.ItemIds.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries);
+
+            var task = _syncManager.CancelItems(request.TargetId, itemIds);
+
+            Task.WaitAll(task);
+        }
+
         public void Post(ReportSyncJobItemTransferred request)
         {
             var task = _syncManager.ReportSyncJobItemTransferred(request.Id);
@@ -261,7 +280,7 @@ namespace MediaBrowser.Api.Sync
                 var auth = AuthorizationContext.GetAuthorizationInfo(Request);
 
                 var authenticatedUser = _userManager.GetUserById(auth.UserId);
-                
+
                 var items = request.ItemIds.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries)
                     .Select(_libraryManager.GetItemById)
                     .Where(i => i != null);

MediaBrowser.Controller/Sync/ISyncDataProvider.cs

@@ -21,7 +21,7 @@ namespace MediaBrowser.Controller.Sync
         /// <param name="serverId">The server identifier.</param>
         /// <returns>Task&lt;List&lt;System.String&gt;&gt;.</returns>
         Task<List<string>> GetSyncJobItemIds(SyncTarget target, string serverId);
-        
+
         /// <summary>
         /// Adds the or update.
         /// </summary>
@@ -53,7 +53,7 @@ namespace MediaBrowser.Controller.Sync
         /// <param name="serverId">The server identifier.</param>
         /// <param name="itemId">The item identifier.</param>
         /// <returns>Task&lt;LocalItem&gt;.</returns>
-        Task<List<LocalItem>> GetCachedItems(SyncTarget target, string serverId, string itemId);
+        Task<List<LocalItem>> GetItems(SyncTarget target, string serverId, string itemId);
         /// <summary>
         /// Gets the cached items by synchronize job item identifier.
         /// </summary>
@@ -61,6 +61,6 @@ namespace MediaBrowser.Controller.Sync
         /// <param name="serverId">The server identifier.</param>
         /// <param name="syncJobItemId">The synchronize job item identifier.</param>
         /// <returns>Task&lt;List&lt;LocalItem&gt;&gt;.</returns>
-        Task<List<LocalItem>> GetCachedItemsBySyncJobItemId(SyncTarget target, string serverId, string syncJobItemId);
+        Task<List<LocalItem>> GetItemsBySyncJobItemId(SyncTarget target, string serverId, string syncJobItemId);
     }
 }

MediaBrowser.Controller/Sync/ISyncManager.cs

@@ -73,6 +73,14 @@ namespace MediaBrowser.Controller.Sync
         /// <returns>Task.</returns>
         Task CancelJob(string id);
 
+        /// <summary>
+        /// Cancels the items.
+        /// </summary>
+        /// <param name="targetId">The target identifier.</param>
+        /// <param name="itemIds">The item ids.</param>
+        /// <returns>Task.</returns>
+        Task CancelItems(string targetId, IEnumerable<string> itemIds);
+
         /// <summary>
         /// Adds the parts.
         /// </summary>

MediaBrowser.Model/ApiClient/IApiClient.cs

@@ -1359,7 +1359,7 @@ namespace MediaBrowser.Model.ApiClient
         /// <param name="job">The job.</param>
         /// <returns>Task.</returns>
         Task UpdateSyncJob(SyncJob job);
-        
+
         /// <summary>
         /// Gets the synchronize jobs.
         /// </summary>
@@ -1492,5 +1492,12 @@ namespace MediaBrowser.Model.ApiClient
         /// <param name="cancellationToken">The cancellation token.</param>
         /// <returns>Task&lt;LiveStreamResponse&gt;.</returns>
         Task<LiveStreamResponse> OpenLiveStream(LiveStreamRequest request, CancellationToken cancellationToken);
+        /// <summary>
+        /// Cancels the synchronize library items.
+        /// </summary>
+        /// <param name="targetId">The target identifier.</param>
+        /// <param name="itemIds">The item ids.</param>
+        /// <returns>Task.</returns>
+        Task CancelSyncLibraryItems(string targetId, IEnumerable<string> itemIds);
     }
 }

MediaBrowser.Server.Implementations/Sync/MediaSync.cs

@@ -289,7 +289,7 @@ namespace MediaBrowser.Server.Implementations.Sync
             SyncTarget target,
             CancellationToken cancellationToken)
         {
-            var localItems = await dataProvider.GetCachedItemsBySyncJobItemId(target, serverId, syncJobItemId);
+            var localItems = await dataProvider.GetItemsBySyncJobItemId(target, serverId, syncJobItemId);
 
             foreach (var localItem in localItems)
             {

MediaBrowser.Server.Implementations/Sync/SyncJobProcessor.cs

@@ -67,9 +67,10 @@ namespace MediaBrowser.Server.Implementations.Sync
             var items = (await GetItemsForSync(job.Category, job.ParentId, job.RequestedItemIds, user, job.UnwatchedOnly).ConfigureAwait(false))
                 .ToList();
 
-            var jobItems = _syncRepo.GetJobItems(new SyncJobItemQuery
+            var jobItems = _syncManager.GetJobItems(new SyncJobItemQuery
             {
-                JobId = job.Id
+                JobId = job.Id,
+                AddMetadata = false
 
             }).Items.ToList();
 
@@ -140,9 +141,10 @@ namespace MediaBrowser.Server.Implementations.Sync
                 throw new ArgumentNullException("job");
             }
 
-            var result = _syncRepo.GetJobItems(new SyncJobItemQuery
+            var result = _syncManager.GetJobItems(new SyncJobItemQuery
             {
-                JobId = job.Id
+                JobId = job.Id,
+                AddMetadata = false
             });
 
             return UpdateJobStatus(job, result.Items.ToList());
@@ -362,9 +364,10 @@ namespace MediaBrowser.Server.Implementations.Sync
             await EnsureSyncJobItems(null, cancellationToken).ConfigureAwait(false);
 
             // If it already has a converting status then is must have been aborted during conversion
-            var result = _syncRepo.GetJobItems(new SyncJobItemQuery
+            var result = _syncManager.GetJobItems(new SyncJobItemQuery
             {
-                Statuses = new SyncJobItemStatus[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting }
+                Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting },
+                AddMetadata = false
             });
 
             await SyncJobItems(result.Items, true, progress, cancellationToken).ConfigureAwait(false);
@@ -384,10 +387,11 @@ namespace MediaBrowser.Server.Implementations.Sync
             await EnsureSyncJobItems(targetId, cancellationToken).ConfigureAwait(false);
 
             // If it already has a converting status then is must have been aborted during conversion
-            var result = _syncRepo.GetJobItems(new SyncJobItemQuery
+            var result = _syncManager.GetJobItems(new SyncJobItemQuery
             {
-                Statuses = new SyncJobItemStatus[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting },
-                TargetId = targetId
+                Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting },
+                TargetId = targetId,
+                AddMetadata = false
             });
 
             await SyncJobItems(result.Items, true, progress, cancellationToken).ConfigureAwait(false);

MediaBrowser.Server.Implementations/Sync/SyncManager.cs

@@ -182,19 +182,21 @@ namespace MediaBrowser.Server.Implementations.Sync
             await processor.EnsureJobItems(job).ConfigureAwait(false);
 
             // If it already has a converting status then is must have been aborted during conversion
-            var jobItemsResult = _repo.GetJobItems(new SyncJobItemQuery
+            var jobItemsResult = GetJobItems(new SyncJobItemQuery
             {
-                Statuses = new SyncJobItemStatus[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting },
-                JobId = jobId
+                Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting },
+                JobId = jobId,
+                AddMetadata = false
             });
 
             await processor.SyncJobItems(jobItemsResult.Items, false, new Progress<double>(), CancellationToken.None)
                     .ConfigureAwait(false);
 
-            jobItemsResult = _repo.GetJobItems(new SyncJobItemQuery
+            jobItemsResult = GetJobItems(new SyncJobItemQuery
             {
-                Statuses = new SyncJobItemStatus[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting },
-                JobId = jobId
+                Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.Converting },
+                JobId = jobId,
+                AddMetadata = false
             });
 
             var returnResult = new SyncJobCreationResult
@@ -761,11 +763,13 @@ namespace MediaBrowser.Server.Implementations.Sync
                     if (jobItem.IsMarkedForRemoval)
                     {
                         // Tell the device to remove it since it has been marked for removal
+                        _logger.Debug("Adding ItemIdsToRemove {0} because IsMarkedForRemoval is set.", jobItem.ItemId);
                         response.ItemIdsToRemove.Add(jobItem.ItemId);
                     }
                     else if (user == null)
                     {
                         // Tell the device to remove it since the user is gone now
+                        _logger.Debug("Adding ItemIdsToRemove {0} because the user is no longer valid.", jobItem.ItemId);
                         response.ItemIdsToRemove.Add(jobItem.ItemId);
                     }
                     else if (job.UnwatchedOnly)
@@ -777,18 +781,22 @@ namespace MediaBrowser.Server.Implementations.Sync
                             if (libraryItem.IsPlayed(user) && libraryItem is Video)
                             {
                                 // Tell the device to remove it since it has been played
+                                _logger.Debug("Adding ItemIdsToRemove {0} because it has been marked played.", jobItem.ItemId);
                                 response.ItemIdsToRemove.Add(jobItem.ItemId);
                             }
                         }
                         else
                         {
                             // Tell the device to remove it since it's no longer available
+                            _logger.Debug("Adding ItemIdsToRemove {0} because it is no longer available.", jobItem.ItemId);
                             response.ItemIdsToRemove.Add(jobItem.ItemId);
                         }
                     }
                 }
                 else
                 {
+                    _logger.Debug("Setting status to RemovedFromDevice for {0} because it is no longer on the device.", jobItem.ItemId);
+
                     // Content is no longer on the device
                     jobItem.Status = SyncJobItemStatus.RemovedFromDevice;
                     await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false);
@@ -842,11 +850,13 @@ namespace MediaBrowser.Server.Implementations.Sync
                     if (jobItem.IsMarkedForRemoval)
                     {
                         // Tell the device to remove it since it has been marked for removal
+                        _logger.Debug("Adding ItemIdsToRemove {0} because IsMarkedForRemoval is set.", jobItem.Id);
                         response.ItemIdsToRemove.Add(jobItem.Id);
                     }
                     else if (user == null)
                     {
                         // Tell the device to remove it since the user is gone now
+                        _logger.Debug("Adding ItemIdsToRemove {0} because the user is no longer valid.", jobItem.Id);
                         response.ItemIdsToRemove.Add(jobItem.Id);
                     }
                     else if (job.UnwatchedOnly)
@@ -858,18 +868,22 @@ namespace MediaBrowser.Server.Implementations.Sync
                             if (libraryItem.IsPlayed(user) && libraryItem is Video)
                             {
                                 // Tell the device to remove it since it has been played
+                                _logger.Debug("Adding ItemIdsToRemove {0} because it has been marked played.", jobItem.Id);
                                 response.ItemIdsToRemove.Add(jobItem.Id);
                             }
                         }
                         else
                         {
                             // Tell the device to remove it since it's no longer available
+                            _logger.Debug("Adding ItemIdsToRemove {0} because it is no longer available.", jobItem.Id);
                             response.ItemIdsToRemove.Add(jobItem.Id);
                         }
                     }
                 }
                 else
                 {
+                    _logger.Debug("Setting status to RemovedFromDevice for {0} because it is no longer on the device.", jobItem.Id);
+
                     // Content is no longer on the device
                     jobItem.Status = SyncJobItemStatus.RemovedFromDevice;
                     await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false);
@@ -894,12 +908,6 @@ namespace MediaBrowser.Server.Implementations.Sync
 
             response.ItemIdsToRemove = response.ItemIdsToRemove.Distinct(StringComparer.OrdinalIgnoreCase).ToList();
 
-            var itemsOnDevice = request.LocalItemIds
-                .Except(response.ItemIdsToRemove)
-                .ToList();
-
-            SetUserAccess(request, response, itemsOnDevice);
-
             return response;
         }
 
@@ -962,16 +970,39 @@ namespace MediaBrowser.Server.Implementations.Sync
             await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false);
         }
 
+        public async Task CancelItems(string targetId, IEnumerable<string> itemIds)
+        {
+            foreach (var item in itemIds)
+            {
+                var syncJobItemResult = GetJobItems(new SyncJobItemQuery
+                {
+                    AddMetadata = false,
+                    ItemId = item,
+                    TargetId = targetId,
+                    Statuses = new[] { SyncJobItemStatus.Queued, SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Converting, SyncJobItemStatus.Synced, SyncJobItemStatus.Failed }
+                });
+
+                foreach (var jobItem in syncJobItemResult.Items)
+                {
+                    await CancelJobItem(jobItem.Id).ConfigureAwait(false);
+                }
+            }
+        }
+
         public async Task CancelJobItem(string id)
         {
             var jobItem = _repo.GetJobItem(id);
 
-            if (jobItem.Status != SyncJobItemStatus.Queued && jobItem.Status != SyncJobItemStatus.ReadyToTransfer && jobItem.Status != SyncJobItemStatus.Converting)
+            if (jobItem.Status != SyncJobItemStatus.Queued && jobItem.Status != SyncJobItemStatus.ReadyToTransfer && jobItem.Status != SyncJobItemStatus.Converting && jobItem.Status != SyncJobItemStatus.Failed && jobItem.Status != SyncJobItemStatus.Synced)
             {
                 throw new ArgumentException("Operation is not valid for this job item");
             }
 
-            jobItem.Status = SyncJobItemStatus.Cancelled;
+            if (jobItem.Status != SyncJobItemStatus.Synced)
+            {
+                jobItem.Status = SyncJobItemStatus.Cancelled;
+            }
+
             jobItem.Progress = 0;
             jobItem.IsMarkedForRemoval = true;
 
@@ -995,24 +1026,24 @@ namespace MediaBrowser.Server.Implementations.Sync
             {
                 _logger.ErrorException("Error deleting directory {0}", ex, path);
             }
+
+            //var jobItemsResult = GetJobItems(new SyncJobItemQuery
+            //{
+            //    AddMetadata = false,
+            //    JobId = jobItem.JobId,
+            //    Limit = 0,
+            //    Statuses = new[] { SyncJobItemStatus.Converting, SyncJobItemStatus.Failed, SyncJobItemStatus.Queued, SyncJobItemStatus.ReadyToTransfer, SyncJobItemStatus.Synced, SyncJobItemStatus.Transferring }
+            //});
+
+            //if (jobItemsResult.TotalRecordCount == 0)
+            //{
+            //    await CancelJob(jobItem.JobId).ConfigureAwait(false);
+            //}
         }
 
-        public async Task MarkJobItemForRemoval(string id)
+        public Task MarkJobItemForRemoval(string id)
         {
-            var jobItem = _repo.GetJobItem(id);
-
-            if (jobItem.Status != SyncJobItemStatus.Synced)
-            {
-                throw new ArgumentException("Operation is not valid for this job item");
-            }
-
-            jobItem.IsMarkedForRemoval = true;
-
-            await UpdateSyncJobItemInternal(jobItem).ConfigureAwait(false);
-
-            var processor = GetSyncJobProcessor();
-
-            await processor.UpdateJobStatus(jobItem.JobId).ConfigureAwait(false);
+            return CancelJobItem(id);
         }
 
         public async Task UnmarkJobItemForRemoval(string id)

MediaBrowser.Server.Implementations/Sync/SyncedMediaSourceProvider.cs

@@ -56,7 +56,7 @@ namespace MediaBrowser.Server.Implementations.Sync
                         var syncProvider = targetTuple.Item1;
                         var dataProvider = _syncManager.GetDataProvider(targetTuple.Item1, syncTarget);
 
-                        var localItems = await dataProvider.GetCachedItems(syncTarget, serverId, item.Id.ToString("N")).ConfigureAwait(false);
+                        var localItems = await dataProvider.GetItems(syncTarget, serverId, item.Id.ToString("N")).ConfigureAwait(false);
 
                         foreach (var localItem in localItems)
                         {

MediaBrowser.Server.Implementations/Sync/TargetDataProvider.cs

@@ -42,11 +42,6 @@ namespace MediaBrowser.Server.Implementations.Sync
             _appHost = appHost;
         }
 
-        private string GetCachePath()
-        {
-            return Path.Combine(_appPaths.DataPath, "sync", _target.Id.GetMD5().ToString("N") + ".json");
-        }
-
         private string GetRemotePath()
         {
             var parts = new List<string>
@@ -66,37 +61,17 @@ namespace MediaBrowser.Server.Implementations.Sync
             return _fileSystem.GetValidFilename(filename);
         }
 
-        private async Task CacheData(Stream stream)
-        {
-            var cachePath = GetCachePath();
-
-            await _cacheFileLock.WaitAsync().ConfigureAwait(false);
-
-            try
-            {
-                Directory.CreateDirectory(Path.GetDirectoryName(cachePath));
-                using (var fileStream = _fileSystem.GetFileStream(cachePath, FileMode.Create, FileAccess.Write, FileShare.Read, true))
-                {
-                    await stream.CopyToAsync(fileStream).ConfigureAwait(false);
-                }
-            }
-            catch (Exception ex)
-            {
-                _logger.ErrorException("Error saving sync data to {0}", ex, cachePath);
-            }
-            finally
-            {
-                _cacheFileLock.Release();
-            }
-        }
-
         private async Task EnsureData(CancellationToken cancellationToken)
         {
             if (_items == null)
             {
                 try
                 {
-                    using (var stream = await _provider.GetFile(GetRemotePath(), _target, new Progress<double>(), cancellationToken))
+                    var path = GetRemotePath();
+
+                    _logger.Debug("Getting {0} from {1}", path, _provider.Name);
+
+                    using (var stream = await _provider.GetFile(path, _target, new Progress<double>(), cancellationToken))
                     {
                         _items = _json.DeserializeFromStream<List<LocalItem>>(stream);
                     }
@@ -109,15 +84,6 @@ namespace MediaBrowser.Server.Implementations.Sync
                 {
                     _items = new List<LocalItem>();
                 }
-
-                using (var memoryStream = new MemoryStream())
-                {
-                    _json.SerializeToStream(_items, memoryStream);
-                    
-                    // Now cache it
-                    memoryStream.Position = 0;
-                    await CacheData(memoryStream).ConfigureAwait(false);
-                }
             }
         }
 
@@ -130,10 +96,6 @@ namespace MediaBrowser.Server.Implementations.Sync
                 // Save to sync provider
                 stream.Position = 0;
                 await _provider.SendFile(stream, GetRemotePath(), _target, new Progress<double>(), cancellationToken).ConfigureAwait(false);
-
-                // Now cache it
-                stream.Position = 0;
-                await CacheData(stream).ConfigureAwait(false);
             }
         }
 
@@ -204,62 +166,14 @@ namespace MediaBrowser.Server.Implementations.Sync
             return GetData(items => items.FirstOrDefault(i => string.Equals(i.Id, id, StringComparison.OrdinalIgnoreCase)));
         }
 
-        private async Task<List<LocalItem>> GetCachedData()
+        public Task<List<LocalItem>> GetItems(SyncTarget target, string serverId, string itemId)
         {
-            if (_items == null)
-            {
-                await _cacheFileLock.WaitAsync().ConfigureAwait(false);
-
-                try
-                {
-                    if (_items == null)
-                    {
-                        try
-                        {
-                            _items = _json.DeserializeFromFile<List<LocalItem>>(GetCachePath());
-                        }
-                        catch (FileNotFoundException)
-                        {
-                            _items = new List<LocalItem>();
-                        }
-                        catch (DirectoryNotFoundException)
-                        {
-                            _items = new List<LocalItem>();
-                        }
-                    }
-                }
-                finally
-                {
-                    _cacheFileLock.Release();
-                }
-            }
-
-            return _items.ToList();
+            return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.ItemId, itemId, StringComparison.OrdinalIgnoreCase)).ToList());
         }
 
-        public async Task<List<string>> GetCachedServerItemIds(SyncTarget target, string serverId)
+        public Task<List<LocalItem>> GetItemsBySyncJobItemId(SyncTarget target, string serverId, string syncJobItemId)
         {
-            var items = await GetCachedData().ConfigureAwait(false);
-
-            return items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase))
-                    .Select(i => i.ItemId)
-                    .ToList();
-        }
-
-        public async Task<List<LocalItem>> GetCachedItems(SyncTarget target, string serverId, string itemId)
-        {
-            var items = await GetCachedData().ConfigureAwait(false);
-
-            return items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.ItemId, itemId, StringComparison.OrdinalIgnoreCase))
-                    .ToList();
-        }
-
-        public async Task<List<LocalItem>> GetCachedItemsBySyncJobItemId(SyncTarget target, string serverId, string syncJobItemId)
-        {
-            var items = await GetCachedData().ConfigureAwait(false);
-
-            return items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.SyncJobItemId, syncJobItemId, StringComparison.OrdinalIgnoreCase))
-                    .ToList();
+            return GetData(items => items.Where(i => string.Equals(i.ServerId, serverId, StringComparison.OrdinalIgnoreCase) && string.Equals(i.SyncJobItemId, syncJobItemId, StringComparison.OrdinalIgnoreCase)).ToList());
         }
     }
 }