LibraryMonitor.cs 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583
  1. #pragma warning disable CS1591
  2. using System;
  3. using System.Collections.Concurrent;
  4. using System.Collections.Generic;
  5. using System.IO;
  6. using System.Linq;
  7. using System.Threading.Tasks;
  8. using MediaBrowser.Controller.Configuration;
  9. using MediaBrowser.Controller.Entities;
  10. using MediaBrowser.Controller.Library;
  11. using MediaBrowser.Controller.Plugins;
  12. using MediaBrowser.Model.IO;
  13. using Microsoft.Extensions.Logging;
  14. namespace Emby.Server.Implementations.IO
  15. {
  16. public class LibraryMonitor : ILibraryMonitor
  17. {
  18. private readonly ILogger _logger;
  19. private readonly ILibraryManager _libraryManager;
  20. private readonly IServerConfigurationManager _configurationManager;
  21. private readonly IFileSystem _fileSystem;
  22. /// <summary>
  23. /// The file system watchers.
  24. /// </summary>
  25. private readonly ConcurrentDictionary<string, FileSystemWatcher> _fileSystemWatchers = new ConcurrentDictionary<string, FileSystemWatcher>(StringComparer.OrdinalIgnoreCase);
  26. /// <summary>
  27. /// The affected paths.
  28. /// </summary>
  29. private readonly List<FileRefresher> _activeRefreshers = new List<FileRefresher>();
  30. /// <summary>
  31. /// A dynamic list of paths that should be ignored. Added to during our own file system modifications.
  32. /// </summary>
  33. private readonly ConcurrentDictionary<string, string> _tempIgnoredPaths = new ConcurrentDictionary<string, string>(StringComparer.OrdinalIgnoreCase);
  34. /// <summary>
  35. /// Any file name ending in any of these will be ignored by the watchers.
  36. /// </summary>
  37. private static readonly HashSet<string> _alwaysIgnoreFiles = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
  38. {
  39. "small.jpg",
  40. "albumart.jpg",
  41. // WMC temp recording directories that will constantly be written to
  42. "TempRec",
  43. "TempSBE"
  44. };
  45. private static readonly string[] _alwaysIgnoreSubstrings = new string[]
  46. {
  47. // Synology
  48. "eaDir",
  49. "#recycle",
  50. ".wd_tv",
  51. ".actors"
  52. };
  53. private static readonly HashSet<string> _alwaysIgnoreExtensions = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
  54. {
  55. // thumbs.db
  56. ".db",
  57. // bts sync files
  58. ".bts",
  59. ".sync"
  60. };
  61. /// <summary>
  62. /// Add the path to our temporary ignore list. Use when writing to a path within our listening scope.
  63. /// </summary>
  64. /// <param name="path">The path.</param>
  65. private void TemporarilyIgnore(string path)
  66. {
  67. _tempIgnoredPaths[path] = path;
  68. }
  69. public void ReportFileSystemChangeBeginning(string path)
  70. {
  71. if (string.IsNullOrEmpty(path))
  72. {
  73. throw new ArgumentNullException(nameof(path));
  74. }
  75. TemporarilyIgnore(path);
  76. }
  77. public bool IsPathLocked(string path)
  78. {
  79. // This method is not used by the core but it used by auto-organize
  80. var lockedPaths = _tempIgnoredPaths.Keys.ToList();
  81. return lockedPaths.Any(i => _fileSystem.AreEqual(i, path) || _fileSystem.ContainsSubPath(i, path));
  82. }
  83. public async void ReportFileSystemChangeComplete(string path, bool refreshPath)
  84. {
  85. if (string.IsNullOrEmpty(path))
  86. {
  87. throw new ArgumentNullException(nameof(path));
  88. }
  89. // This is an arbitrary amount of time, but delay it because file system writes often trigger events long after the file was actually written to.
  90. // Seeing long delays in some situations, especially over the network, sometimes up to 45 seconds
  91. // But if we make this delay too high, we risk missing legitimate changes, such as user adding a new file, or hand-editing metadata
  92. await Task.Delay(45000).ConfigureAwait(false);
  93. _tempIgnoredPaths.TryRemove(path, out var val);
  94. if (refreshPath)
  95. {
  96. try
  97. {
  98. ReportFileSystemChanged(path);
  99. }
  100. catch (Exception ex)
  101. {
  102. _logger.LogError(ex, "Error in ReportFileSystemChanged for {path}", path);
  103. }
  104. }
  105. }
  106. /// <summary>
  107. /// Initializes a new instance of the <see cref="LibraryMonitor" /> class.
  108. /// </summary>
  109. public LibraryMonitor(
  110. ILogger<LibraryMonitor> logger,
  111. ILibraryManager libraryManager,
  112. IServerConfigurationManager configurationManager,
  113. IFileSystem fileSystem)
  114. {
  115. _libraryManager = libraryManager;
  116. _logger = logger;
  117. _configurationManager = configurationManager;
  118. _fileSystem = fileSystem;
  119. }
  120. private bool IsLibraryMonitorEnabled(BaseItem item)
  121. {
  122. if (item is BasePluginFolder)
  123. {
  124. return false;
  125. }
  126. var options = _libraryManager.GetLibraryOptions(item);
  127. if (options != null)
  128. {
  129. return options.EnableRealtimeMonitor;
  130. }
  131. return false;
  132. }
  133. public void Start()
  134. {
  135. _libraryManager.ItemAdded += OnLibraryManagerItemAdded;
  136. _libraryManager.ItemRemoved += OnLibraryManagerItemRemoved;
  137. var pathsToWatch = new List<string>();
  138. var paths = _libraryManager
  139. .RootFolder
  140. .Children
  141. .Where(IsLibraryMonitorEnabled)
  142. .OfType<Folder>()
  143. .SelectMany(f => f.PhysicalLocations)
  144. .Distinct(StringComparer.OrdinalIgnoreCase)
  145. .OrderBy(i => i)
  146. .ToList();
  147. foreach (var path in paths)
  148. {
  149. if (!ContainsParentFolder(pathsToWatch, path))
  150. {
  151. pathsToWatch.Add(path);
  152. }
  153. }
  154. foreach (var path in pathsToWatch)
  155. {
  156. StartWatchingPath(path);
  157. }
  158. }
  159. private void StartWatching(BaseItem item)
  160. {
  161. if (IsLibraryMonitorEnabled(item))
  162. {
  163. StartWatchingPath(item.Path);
  164. }
  165. }
  166. /// <summary>
  167. /// Handles the ItemRemoved event of the LibraryManager control.
  168. /// </summary>
  169. /// <param name="sender">The source of the event.</param>
  170. /// <param name="e">The <see cref="ItemChangeEventArgs"/> instance containing the event data.</param>
  171. private void OnLibraryManagerItemRemoved(object sender, ItemChangeEventArgs e)
  172. {
  173. if (e.Parent is AggregateFolder)
  174. {
  175. StopWatchingPath(e.Item.Path);
  176. }
  177. }
  178. /// <summary>
  179. /// Handles the ItemAdded event of the LibraryManager control.
  180. /// </summary>
  181. /// <param name="sender">The source of the event.</param>
  182. /// <param name="e">The <see cref="ItemChangeEventArgs"/> instance containing the event data.</param>
  183. private void OnLibraryManagerItemAdded(object sender, ItemChangeEventArgs e)
  184. {
  185. if (e.Parent is AggregateFolder)
  186. {
  187. StartWatching(e.Item);
  188. }
  189. }
  190. /// <summary>
  191. /// Examine a list of strings assumed to be file paths to see if it contains a parent of
  192. /// the provided path.
  193. /// </summary>
  194. /// <param name="lst">The LST.</param>
  195. /// <param name="path">The path.</param>
  196. /// <returns><c>true</c> if [contains parent folder] [the specified LST]; otherwise, <c>false</c>.</returns>
  197. /// <exception cref="ArgumentNullException">path</exception>
  198. private static bool ContainsParentFolder(IEnumerable<string> lst, string path)
  199. {
  200. if (string.IsNullOrEmpty(path))
  201. {
  202. throw new ArgumentNullException(nameof(path));
  203. }
  204. path = path.TrimEnd(Path.DirectorySeparatorChar);
  205. return lst.Any(str =>
  206. {
  207. // this should be a little quicker than examining each actual parent folder...
  208. var compare = str.TrimEnd(Path.DirectorySeparatorChar);
  209. return path.Equals(compare, StringComparison.OrdinalIgnoreCase) || (path.StartsWith(compare, StringComparison.OrdinalIgnoreCase) && path[compare.Length] == Path.DirectorySeparatorChar);
  210. });
  211. }
  212. /// <summary>
  213. /// Starts the watching path.
  214. /// </summary>
  215. /// <param name="path">The path.</param>
  216. private void StartWatchingPath(string path)
  217. {
  218. if (!Directory.Exists(path))
  219. {
  220. // Seeing a crash in the mono runtime due to an exception being thrown on a different thread
  221. _logger.LogInformation("Skipping realtime monitor for {Path} because the path does not exist", path);
  222. return;
  223. }
  224. // Already being watched
  225. if (_fileSystemWatchers.ContainsKey(path))
  226. {
  227. return;
  228. }
  229. // Creating a FileSystemWatcher over the LAN can take hundreds of milliseconds, so wrap it in a Task to do them all in parallel
  230. Task.Run(() =>
  231. {
  232. try
  233. {
  234. var newWatcher = new FileSystemWatcher(path, "*")
  235. {
  236. IncludeSubdirectories = true,
  237. InternalBufferSize = 65536,
  238. NotifyFilter = NotifyFilters.CreationTime |
  239. NotifyFilters.DirectoryName |
  240. NotifyFilters.FileName |
  241. NotifyFilters.LastWrite |
  242. NotifyFilters.Size |
  243. NotifyFilters.Attributes
  244. };
  245. newWatcher.Created += OnWatcherChanged;
  246. newWatcher.Deleted += OnWatcherChanged;
  247. newWatcher.Renamed += OnWatcherChanged;
  248. newWatcher.Changed += OnWatcherChanged;
  249. newWatcher.Error += OnWatcherError;
  250. if (_fileSystemWatchers.TryAdd(path, newWatcher))
  251. {
  252. newWatcher.EnableRaisingEvents = true;
  253. _logger.LogInformation("Watching directory " + path);
  254. }
  255. else
  256. {
  257. DisposeWatcher(newWatcher, false);
  258. }
  259. }
  260. catch (Exception ex)
  261. {
  262. _logger.LogError(ex, "Error watching path: {path}", path);
  263. }
  264. });
  265. }
  266. /// <summary>
  267. /// Stops the watching path.
  268. /// </summary>
  269. /// <param name="path">The path.</param>
  270. private void StopWatchingPath(string path)
  271. {
  272. if (_fileSystemWatchers.TryGetValue(path, out var watcher))
  273. {
  274. DisposeWatcher(watcher, true);
  275. }
  276. }
  277. /// <summary>
  278. /// Disposes the watcher.
  279. /// </summary>
  280. private void DisposeWatcher(FileSystemWatcher watcher, bool removeFromList)
  281. {
  282. try
  283. {
  284. using (watcher)
  285. {
  286. _logger.LogInformation("Stopping directory watching for path {Path}", watcher.Path);
  287. watcher.Created -= OnWatcherChanged;
  288. watcher.Deleted -= OnWatcherChanged;
  289. watcher.Renamed -= OnWatcherChanged;
  290. watcher.Changed -= OnWatcherChanged;
  291. watcher.Error -= OnWatcherError;
  292. watcher.EnableRaisingEvents = false;
  293. }
  294. }
  295. finally
  296. {
  297. if (removeFromList)
  298. {
  299. RemoveWatcherFromList(watcher);
  300. }
  301. }
  302. }
  303. /// <summary>
  304. /// Removes the watcher from list.
  305. /// </summary>
  306. /// <param name="watcher">The watcher.</param>
  307. private void RemoveWatcherFromList(FileSystemWatcher watcher)
  308. {
  309. _fileSystemWatchers.TryRemove(watcher.Path, out _);
  310. }
  311. /// <summary>
  312. /// Handles the Error event of the watcher control.
  313. /// </summary>
  314. /// <param name="sender">The source of the event.</param>
  315. /// <param name="e">The <see cref="ErrorEventArgs" /> instance containing the event data.</param>
  316. private void OnWatcherError(object sender, ErrorEventArgs e)
  317. {
  318. var ex = e.GetException();
  319. var dw = (FileSystemWatcher)sender;
  320. _logger.LogError(ex, "Error in Directory watcher for: {Path}", dw.Path);
  321. DisposeWatcher(dw, true);
  322. }
  323. /// <summary>
  324. /// Handles the Changed event of the watcher control.
  325. /// </summary>
  326. /// <param name="sender">The source of the event.</param>
  327. /// <param name="e">The <see cref="FileSystemEventArgs" /> instance containing the event data.</param>
  328. private void OnWatcherChanged(object sender, FileSystemEventArgs e)
  329. {
  330. try
  331. {
  332. ReportFileSystemChanged(e.FullPath);
  333. }
  334. catch (Exception ex)
  335. {
  336. _logger.LogError(ex, "Exception in ReportFileSystemChanged. Path: {FullPath}", e.FullPath);
  337. }
  338. }
  339. public void ReportFileSystemChanged(string path)
  340. {
  341. if (string.IsNullOrEmpty(path))
  342. {
  343. throw new ArgumentNullException(nameof(path));
  344. }
  345. var filename = Path.GetFileName(path);
  346. var monitorPath = !string.IsNullOrEmpty(filename) &&
  347. !_alwaysIgnoreFiles.Contains(filename) &&
  348. !_alwaysIgnoreExtensions.Contains(Path.GetExtension(path)) &&
  349. _alwaysIgnoreSubstrings.All(i => path.IndexOf(i, StringComparison.OrdinalIgnoreCase) == -1);
  350. // Ignore certain files
  351. var tempIgnorePaths = _tempIgnoredPaths.Keys.ToList();
  352. // If the parent of an ignored path has a change event, ignore that too
  353. if (tempIgnorePaths.Any(i =>
  354. {
  355. if (_fileSystem.AreEqual(i, path))
  356. {
  357. _logger.LogDebug("Ignoring change to {Path}", path);
  358. return true;
  359. }
  360. if (_fileSystem.ContainsSubPath(i, path))
  361. {
  362. _logger.LogDebug("Ignoring change to {Path}", path);
  363. return true;
  364. }
  365. // Go up a level
  366. var parent = Path.GetDirectoryName(i);
  367. if (!string.IsNullOrEmpty(parent) && _fileSystem.AreEqual(parent, path))
  368. {
  369. _logger.LogDebug("Ignoring change to {Path}", path);
  370. return true;
  371. }
  372. return false;
  373. }))
  374. {
  375. monitorPath = false;
  376. }
  377. if (monitorPath)
  378. {
  379. // Avoid implicitly captured closure
  380. CreateRefresher(path);
  381. }
  382. }
  383. private void CreateRefresher(string path)
  384. {
  385. var parentPath = Path.GetDirectoryName(path);
  386. lock (_activeRefreshers)
  387. {
  388. foreach (var refresher in _activeRefreshers)
  389. {
  390. // Path is already being refreshed
  391. if (_fileSystem.AreEqual(path, refresher.Path))
  392. {
  393. refresher.RestartTimer();
  394. return;
  395. }
  396. // Parent folder is already being refreshed
  397. if (_fileSystem.ContainsSubPath(refresher.Path, path))
  398. {
  399. refresher.AddPath(path);
  400. return;
  401. }
  402. // New path is a parent
  403. if (_fileSystem.ContainsSubPath(path, refresher.Path))
  404. {
  405. refresher.ResetPath(path, null);
  406. return;
  407. }
  408. // They are siblings. Rebase the refresher to the parent folder.
  409. if (string.Equals(parentPath, Path.GetDirectoryName(refresher.Path), StringComparison.Ordinal))
  410. {
  411. refresher.ResetPath(parentPath, path);
  412. return;
  413. }
  414. }
  415. var newRefresher = new FileRefresher(path, _configurationManager, _libraryManager, _logger);
  416. newRefresher.Completed += NewRefresher_Completed;
  417. _activeRefreshers.Add(newRefresher);
  418. }
  419. }
  420. private void NewRefresher_Completed(object sender, EventArgs e)
  421. {
  422. var refresher = (FileRefresher)sender;
  423. DisposeRefresher(refresher);
  424. }
  425. /// <summary>
  426. /// Stops this instance.
  427. /// </summary>
  428. public void Stop()
  429. {
  430. _libraryManager.ItemAdded -= OnLibraryManagerItemAdded;
  431. _libraryManager.ItemRemoved -= OnLibraryManagerItemRemoved;
  432. foreach (var watcher in _fileSystemWatchers.Values.ToList())
  433. {
  434. DisposeWatcher(watcher, false);
  435. }
  436. _fileSystemWatchers.Clear();
  437. DisposeRefreshers();
  438. }
  439. private void DisposeRefresher(FileRefresher refresher)
  440. {
  441. lock (_activeRefreshers)
  442. {
  443. refresher.Dispose();
  444. _activeRefreshers.Remove(refresher);
  445. }
  446. }
  447. private void DisposeRefreshers()
  448. {
  449. lock (_activeRefreshers)
  450. {
  451. foreach (var refresher in _activeRefreshers.ToList())
  452. {
  453. refresher.Dispose();
  454. }
  455. _activeRefreshers.Clear();
  456. }
  457. }
  458. private bool _disposed = false;
  459. /// <summary>
  460. /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
  461. /// </summary>
  462. public void Dispose()
  463. {
  464. Dispose(true);
  465. GC.SuppressFinalize(this);
  466. }
  467. /// <summary>
  468. /// Releases unmanaged and - optionally - managed resources.
  469. /// </summary>
  470. /// <param name="disposing"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
  471. protected virtual void Dispose(bool disposing)
  472. {
  473. if (_disposed)
  474. {
  475. return;
  476. }
  477. if (disposing)
  478. {
  479. Stop();
  480. }
  481. _disposed = true;
  482. }
  483. }
  484. public class LibraryMonitorStartup : IServerEntryPoint
  485. {
  486. private readonly ILibraryMonitor _monitor;
  487. public LibraryMonitorStartup(ILibraryMonitor monitor)
  488. {
  489. _monitor = monitor;
  490. }
  491. public Task RunAsync()
  492. {
  493. _monitor.Start();
  494. return Task.CompletedTask;
  495. }
  496. public void Dispose()
  497. {
  498. }
  499. }
  500. }