MediaSync.cs 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502
  1. using MediaBrowser.Common.Configuration;
  2. using MediaBrowser.Common.IO;
  3. using MediaBrowser.Common.Progress;
  4. using MediaBrowser.Controller;
  5. using MediaBrowser.Controller.IO;
  6. using MediaBrowser.Controller.Sync;
  7. using MediaBrowser.Model.Dto;
  8. using MediaBrowser.Model.Entities;
  9. using MediaBrowser.Model.Logging;
  10. using MediaBrowser.Model.MediaInfo;
  11. using MediaBrowser.Model.Sync;
  12. using System;
  13. using System.Collections.Generic;
  14. using System.Globalization;
  15. using System.IO;
  16. using System.Linq;
  17. using System.Security.Cryptography;
  18. using System.Text;
  19. using System.Threading;
  20. using System.Threading.Tasks;
  21. using CommonIO;
  22. using Interfaces.IO;
  23. namespace MediaBrowser.Server.Implementations.Sync
  24. {
  25. public class MediaSync
  26. {
  27. private readonly ISyncManager _syncManager;
  28. private readonly IServerApplicationHost _appHost;
  29. private readonly ILogger _logger;
  30. private readonly IFileSystem _fileSystem;
  31. private readonly IConfigurationManager _config;
  32. public const string PathSeparatorString = "/";
  33. public const char PathSeparatorChar = '/';
  34. public MediaSync(ILogger logger, ISyncManager syncManager, IServerApplicationHost appHost, IFileSystem fileSystem, IConfigurationManager config)
  35. {
  36. _logger = logger;
  37. _syncManager = syncManager;
  38. _appHost = appHost;
  39. _fileSystem = fileSystem;
  40. _config = config;
  41. }
  42. public async Task Sync(IServerSyncProvider provider,
  43. ISyncDataProvider dataProvider,
  44. SyncTarget target,
  45. IProgress<double> progress,
  46. CancellationToken cancellationToken)
  47. {
  48. var serverId = _appHost.SystemId;
  49. var serverName = _appHost.FriendlyName;
  50. await SyncData(provider, dataProvider, serverId, target, cancellationToken).ConfigureAwait(false);
  51. progress.Report(3);
  52. var innerProgress = new ActionableProgress<double>();
  53. innerProgress.RegisterAction(pct =>
  54. {
  55. var totalProgress = pct * .97;
  56. totalProgress += 1;
  57. progress.Report(totalProgress);
  58. });
  59. await GetNewMedia(provider, dataProvider, target, serverId, serverName, innerProgress, cancellationToken);
  60. // Do the data sync twice so the server knows what was removed from the device
  61. await SyncData(provider, dataProvider, serverId, target, cancellationToken).ConfigureAwait(false);
  62. progress.Report(100);
  63. }
  64. private async Task SyncData(IServerSyncProvider provider,
  65. ISyncDataProvider dataProvider,
  66. string serverId,
  67. SyncTarget target,
  68. CancellationToken cancellationToken)
  69. {
  70. var localItems = await dataProvider.GetLocalItems(target, serverId).ConfigureAwait(false);
  71. var remoteFiles = await provider.GetFiles(new FileQuery(), target, cancellationToken).ConfigureAwait(false);
  72. var remoteIds = remoteFiles.Items.Select(i => i.Id).ToList();
  73. var jobItemIds = new List<string>();
  74. foreach (var localItem in localItems)
  75. {
  76. if (remoteIds.Contains(localItem.FileId, StringComparer.OrdinalIgnoreCase))
  77. {
  78. jobItemIds.Add(localItem.SyncJobItemId);
  79. }
  80. }
  81. var result = await _syncManager.SyncData(new SyncDataRequest
  82. {
  83. TargetId = target.Id,
  84. SyncJobItemIds = jobItemIds
  85. }).ConfigureAwait(false);
  86. cancellationToken.ThrowIfCancellationRequested();
  87. foreach (var itemIdToRemove in result.ItemIdsToRemove)
  88. {
  89. try
  90. {
  91. await RemoveItem(provider, dataProvider, serverId, itemIdToRemove, target, cancellationToken).ConfigureAwait(false);
  92. }
  93. catch (Exception ex)
  94. {
  95. _logger.ErrorException("Error deleting item from device. Id: {0}", ex, itemIdToRemove);
  96. }
  97. }
  98. }
  99. private async Task GetNewMedia(IServerSyncProvider provider,
  100. ISyncDataProvider dataProvider,
  101. SyncTarget target,
  102. string serverId,
  103. string serverName,
  104. IProgress<double> progress,
  105. CancellationToken cancellationToken)
  106. {
  107. var jobItems = await _syncManager.GetReadySyncItems(target.Id).ConfigureAwait(false);
  108. var numComplete = 0;
  109. double startingPercent = 0;
  110. double percentPerItem = 1;
  111. if (jobItems.Count > 0)
  112. {
  113. percentPerItem /= jobItems.Count;
  114. }
  115. foreach (var jobItem in jobItems)
  116. {
  117. cancellationToken.ThrowIfCancellationRequested();
  118. var currentPercent = startingPercent;
  119. var innerProgress = new ActionableProgress<double>();
  120. innerProgress.RegisterAction(pct =>
  121. {
  122. var totalProgress = pct * percentPerItem;
  123. totalProgress += currentPercent;
  124. progress.Report(totalProgress);
  125. });
  126. try
  127. {
  128. await GetItem(provider, dataProvider, target, serverId, serverName, jobItem, innerProgress, cancellationToken).ConfigureAwait(false);
  129. }
  130. catch (Exception ex)
  131. {
  132. _logger.ErrorException("Error syncing item", ex);
  133. }
  134. numComplete++;
  135. startingPercent = numComplete;
  136. startingPercent /= jobItems.Count;
  137. startingPercent *= 100;
  138. progress.Report(startingPercent);
  139. }
  140. }
  141. private async Task GetItem(IServerSyncProvider provider,
  142. ISyncDataProvider dataProvider,
  143. SyncTarget target,
  144. string serverId,
  145. string serverName,
  146. SyncedItem jobItem,
  147. IProgress<double> progress,
  148. CancellationToken cancellationToken)
  149. {
  150. var libraryItem = jobItem.Item;
  151. var internalSyncJobItem = _syncManager.GetJobItem(jobItem.SyncJobItemId);
  152. var internalSyncJob = _syncManager.GetJob(jobItem.SyncJobId);
  153. var localItem = CreateLocalItem(provider, jobItem, internalSyncJob, target, libraryItem, serverId, serverName, jobItem.OriginalFileName);
  154. await _syncManager.ReportSyncJobItemTransferBeginning(internalSyncJobItem.Id);
  155. var transferSuccess = false;
  156. Exception transferException = null;
  157. var options = _config.GetSyncOptions();
  158. try
  159. {
  160. var fileTransferProgress = new ActionableProgress<double>();
  161. fileTransferProgress.RegisterAction(pct => progress.Report(pct * .92));
  162. var sendFileResult = await SendFile(provider, internalSyncJobItem.OutputPath, localItem.LocalPath.Split(PathSeparatorChar), target, options, fileTransferProgress, cancellationToken).ConfigureAwait(false);
  163. if (localItem.Item.MediaSources != null)
  164. {
  165. var mediaSource = localItem.Item.MediaSources.FirstOrDefault();
  166. if (mediaSource != null)
  167. {
  168. mediaSource.Path = sendFileResult.Path;
  169. mediaSource.Protocol = sendFileResult.Protocol;
  170. mediaSource.RequiredHttpHeaders = sendFileResult.RequiredHttpHeaders;
  171. mediaSource.SupportsTranscoding = false;
  172. }
  173. }
  174. localItem.FileId = sendFileResult.Id;
  175. // Create db record
  176. await dataProvider.AddOrUpdate(target, localItem).ConfigureAwait(false);
  177. if (localItem.Item.MediaSources != null)
  178. {
  179. var mediaSource = localItem.Item.MediaSources.FirstOrDefault();
  180. if (mediaSource != null)
  181. {
  182. await SendSubtitles(localItem, mediaSource, provider, dataProvider, target, options, cancellationToken).ConfigureAwait(false);
  183. }
  184. }
  185. progress.Report(92);
  186. transferSuccess = true;
  187. progress.Report(99);
  188. }
  189. catch (Exception ex)
  190. {
  191. _logger.ErrorException("Error transferring sync job file", ex);
  192. transferException = ex;
  193. }
  194. if (transferSuccess)
  195. {
  196. await _syncManager.ReportSyncJobItemTransferred(jobItem.SyncJobItemId).ConfigureAwait(false);
  197. }
  198. else
  199. {
  200. await _syncManager.ReportSyncJobItemTransferFailed(jobItem.SyncJobItemId).ConfigureAwait(false);
  201. throw transferException;
  202. }
  203. }
  204. private async Task SendSubtitles(LocalItem localItem, MediaSourceInfo mediaSource, IServerSyncProvider provider, ISyncDataProvider dataProvider, SyncTarget target, SyncOptions options, CancellationToken cancellationToken)
  205. {
  206. var failedSubtitles = new List<MediaStream>();
  207. var requiresSave = false;
  208. foreach (var mediaStream in mediaSource.MediaStreams
  209. .Where(i => i.Type == MediaStreamType.Subtitle && i.IsExternal)
  210. .ToList())
  211. {
  212. try
  213. {
  214. var remotePath = GetRemoteSubtitlePath(localItem, mediaStream, provider, target);
  215. var sendFileResult = await SendFile(provider, mediaStream.Path, remotePath, target, options, new Progress<double>(), cancellationToken).ConfigureAwait(false);
  216. // This is the path that will be used when talking to the provider
  217. mediaStream.ExternalId = sendFileResult.Id;
  218. // Keep track of all additional files for cleanup later.
  219. localItem.AdditionalFiles.Add(sendFileResult.Id);
  220. // This is the public path clients will use
  221. mediaStream.Path = sendFileResult.Path;
  222. requiresSave = true;
  223. }
  224. catch (Exception ex)
  225. {
  226. _logger.ErrorException("Error sending subtitle stream", ex);
  227. failedSubtitles.Add(mediaStream);
  228. }
  229. }
  230. if (failedSubtitles.Count > 0)
  231. {
  232. mediaSource.MediaStreams = mediaSource.MediaStreams.Except(failedSubtitles).ToList();
  233. requiresSave = true;
  234. }
  235. if (requiresSave)
  236. {
  237. await dataProvider.AddOrUpdate(target, localItem).ConfigureAwait(false);
  238. }
  239. }
  240. private string[] GetRemoteSubtitlePath(LocalItem item, MediaStream stream, IServerSyncProvider provider, SyncTarget target)
  241. {
  242. var filename = GetSubtitleSaveFileName(item, stream.Language, stream.IsForced) + "." + stream.Codec.ToLower();
  243. var pathParts = item.LocalPath.Split(PathSeparatorChar);
  244. var list = pathParts.Take(pathParts.Length - 1).ToList();
  245. list.Add(filename);
  246. return list.ToArray();
  247. }
  248. private string GetSubtitleSaveFileName(LocalItem item, string language, bool isForced)
  249. {
  250. var path = item.LocalPath;
  251. var name = Path.GetFileNameWithoutExtension(path);
  252. if (!string.IsNullOrWhiteSpace(language))
  253. {
  254. name += "." + language.ToLower();
  255. }
  256. if (isForced)
  257. {
  258. name += ".foreign";
  259. }
  260. return name;
  261. }
  262. private async Task RemoveItem(IServerSyncProvider provider,
  263. ISyncDataProvider dataProvider,
  264. string serverId,
  265. string syncJobItemId,
  266. SyncTarget target,
  267. CancellationToken cancellationToken)
  268. {
  269. var localItems = await dataProvider.GetItemsBySyncJobItemId(target, serverId, syncJobItemId);
  270. foreach (var localItem in localItems)
  271. {
  272. var files = localItem.AdditionalFiles.ToList();
  273. foreach (var file in files)
  274. {
  275. _logger.Debug("Removing {0} from {1}.", file, target.Name);
  276. await provider.DeleteFile(file, target, cancellationToken).ConfigureAwait(false);
  277. }
  278. _logger.Debug("Removing {0} from {1}.", localItem.FileId, target.Name);
  279. await provider.DeleteFile(localItem.FileId, target, cancellationToken).ConfigureAwait(false);
  280. await dataProvider.Delete(target, localItem.Id).ConfigureAwait(false);
  281. }
  282. }
  283. private async Task<SyncedFileInfo> SendFile(IServerSyncProvider provider, string inputPath, string[] pathParts, SyncTarget target, SyncOptions options, IProgress<double> progress, CancellationToken cancellationToken)
  284. {
  285. _logger.Debug("Sending {0} to {1}. Remote path: {2}", inputPath, provider.Name, string.Join("/", pathParts));
  286. var supportsDirectCopy = provider as ISupportsDirectCopy;
  287. if (supportsDirectCopy != null)
  288. {
  289. return await supportsDirectCopy.SendFile(inputPath, pathParts, target, progress, cancellationToken).ConfigureAwait(false);
  290. }
  291. using (var fileStream = _fileSystem.GetFileStream(inputPath, FileMode.Open, FileAccess.Read, FileShare.Read, true))
  292. {
  293. Stream stream = fileStream;
  294. if (options.UploadSpeedLimitBytes > 0 && provider is IRemoteSyncProvider)
  295. {
  296. stream = new ThrottledStream(stream, options.UploadSpeedLimitBytes);
  297. }
  298. return await provider.SendFile(stream, pathParts, target, progress, cancellationToken).ConfigureAwait(false);
  299. }
  300. }
  301. private static string GetLocalId(string jobItemId, string itemId)
  302. {
  303. var bytes = Encoding.UTF8.GetBytes(jobItemId + itemId);
  304. bytes = CreateMd5(bytes);
  305. return BitConverter.ToString(bytes, 0, bytes.Length).Replace("-", string.Empty);
  306. }
  307. private static byte[] CreateMd5(byte[] value)
  308. {
  309. using (var provider = MD5.Create())
  310. {
  311. return provider.ComputeHash(value);
  312. }
  313. }
  314. public LocalItem CreateLocalItem(IServerSyncProvider provider, SyncedItem syncedItem, SyncJob job, SyncTarget target, BaseItemDto libraryItem, string serverId, string serverName, string originalFileName)
  315. {
  316. var path = GetDirectoryPath(provider, job, syncedItem, libraryItem, serverName);
  317. path.Add(GetLocalFileName(provider, libraryItem, originalFileName));
  318. var localPath = string.Join(PathSeparatorString, path.ToArray());
  319. foreach (var mediaSource in libraryItem.MediaSources)
  320. {
  321. mediaSource.Path = localPath;
  322. mediaSource.Protocol = MediaProtocol.File;
  323. }
  324. return new LocalItem
  325. {
  326. Item = libraryItem,
  327. ItemId = libraryItem.Id,
  328. ServerId = serverId,
  329. LocalPath = localPath,
  330. Id = GetLocalId(syncedItem.SyncJobItemId, libraryItem.Id),
  331. SyncJobItemId = syncedItem.SyncJobItemId
  332. };
  333. }
  334. private List<string> GetDirectoryPath(IServerSyncProvider provider, SyncJob job, SyncedItem syncedItem, BaseItemDto item, string serverName)
  335. {
  336. var parts = new List<string>
  337. {
  338. serverName
  339. };
  340. var profileOption = _syncManager.GetProfileOptions(job.TargetId)
  341. .FirstOrDefault(i => string.Equals(i.Id, job.Profile, StringComparison.OrdinalIgnoreCase));
  342. string name;
  343. if (profileOption != null && !string.IsNullOrWhiteSpace(profileOption.Name))
  344. {
  345. name = profileOption.Name;
  346. if (job.Bitrate.HasValue)
  347. {
  348. name += "-" + job.Bitrate.Value.ToString(CultureInfo.InvariantCulture);
  349. }
  350. else
  351. {
  352. var qualityOption = _syncManager.GetQualityOptions(job.TargetId)
  353. .FirstOrDefault(i => string.Equals(i.Id, job.Quality, StringComparison.OrdinalIgnoreCase));
  354. if (qualityOption != null && !string.IsNullOrWhiteSpace(qualityOption.Name))
  355. {
  356. name += "-" + qualityOption.Name;
  357. }
  358. }
  359. }
  360. else
  361. {
  362. name = syncedItem.SyncJobName + "-" + syncedItem.SyncJobDateCreated
  363. .ToLocalTime()
  364. .ToString("g")
  365. .Replace(" ", "-");
  366. }
  367. name = GetValidFilename(provider, name);
  368. parts.Add(name);
  369. if (item.IsType("episode"))
  370. {
  371. parts.Add("TV");
  372. if (!string.IsNullOrWhiteSpace(item.SeriesName))
  373. {
  374. parts.Add(item.SeriesName);
  375. }
  376. }
  377. else if (item.IsVideo)
  378. {
  379. parts.Add("Videos");
  380. parts.Add(item.Name);
  381. }
  382. else if (item.IsAudio)
  383. {
  384. parts.Add("Music");
  385. if (!string.IsNullOrWhiteSpace(item.AlbumArtist))
  386. {
  387. parts.Add(item.AlbumArtist);
  388. }
  389. if (!string.IsNullOrWhiteSpace(item.Album))
  390. {
  391. parts.Add(item.Album);
  392. }
  393. }
  394. else if (string.Equals(item.MediaType, MediaType.Photo, StringComparison.OrdinalIgnoreCase))
  395. {
  396. parts.Add("Photos");
  397. if (!string.IsNullOrWhiteSpace(item.Album))
  398. {
  399. parts.Add(item.Album);
  400. }
  401. }
  402. return parts.Select(i => GetValidFilename(provider, i)).ToList();
  403. }
  404. private string GetLocalFileName(IServerSyncProvider provider, BaseItemDto item, string originalFileName)
  405. {
  406. var filename = originalFileName;
  407. if (string.IsNullOrWhiteSpace(filename))
  408. {
  409. filename = item.Name;
  410. }
  411. return GetValidFilename(provider, filename);
  412. }
  413. private string GetValidFilename(IServerSyncProvider provider, string filename)
  414. {
  415. // We can always add this method to the sync provider if it's really needed
  416. return _fileSystem.GetValidFilename(filename);
  417. }
  418. }
  419. }