SubtitleEncoder.cs 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009
  1. #pragma warning disable CS1591
  2. using System;
  3. using System.Collections.Generic;
  4. using System.Diagnostics;
  5. using System.Diagnostics.CodeAnalysis;
  6. using System.Globalization;
  7. using System.IO;
  8. using System.Linq;
  9. using System.Net.Http;
  10. using System.Text;
  11. using System.Threading;
  12. using System.Threading.Tasks;
  13. using AsyncKeyedLock;
  14. using MediaBrowser.Common;
  15. using MediaBrowser.Common.Configuration;
  16. using MediaBrowser.Common.Extensions;
  17. using MediaBrowser.Common.Net;
  18. using MediaBrowser.Controller.Configuration;
  19. using MediaBrowser.Controller.Entities;
  20. using MediaBrowser.Controller.IO;
  21. using MediaBrowser.Controller.Library;
  22. using MediaBrowser.Controller.MediaEncoding;
  23. using MediaBrowser.Model.Dto;
  24. using MediaBrowser.Model.Entities;
  25. using MediaBrowser.Model.IO;
  26. using MediaBrowser.Model.MediaInfo;
  27. using Microsoft.Extensions.Logging;
  28. using UtfUnknown;
  29. namespace MediaBrowser.MediaEncoding.Subtitles
  30. {
  31. public sealed class SubtitleEncoder : ISubtitleEncoder, IDisposable
  32. {
  33. private readonly ILogger<SubtitleEncoder> _logger;
  34. private readonly IFileSystem _fileSystem;
  35. private readonly IMediaEncoder _mediaEncoder;
  36. private readonly IHttpClientFactory _httpClientFactory;
  37. private readonly IMediaSourceManager _mediaSourceManager;
  38. private readonly ISubtitleParser _subtitleParser;
  39. private readonly IPathManager _pathManager;
  40. private readonly IServerConfigurationManager _serverConfigurationManager;
  41. /// <summary>
  42. /// The _semaphoreLocks.
  43. /// </summary>
  44. private readonly AsyncKeyedLocker<string> _semaphoreLocks = new(o =>
  45. {
  46. o.PoolSize = 20;
  47. o.PoolInitialFill = 1;
  48. });
  49. public SubtitleEncoder(
  50. ILogger<SubtitleEncoder> logger,
  51. IFileSystem fileSystem,
  52. IMediaEncoder mediaEncoder,
  53. IHttpClientFactory httpClientFactory,
  54. IMediaSourceManager mediaSourceManager,
  55. ISubtitleParser subtitleParser,
  56. IPathManager pathManager,
  57. IServerConfigurationManager serverConfigurationManager)
  58. {
  59. _logger = logger;
  60. _fileSystem = fileSystem;
  61. _mediaEncoder = mediaEncoder;
  62. _httpClientFactory = httpClientFactory;
  63. _mediaSourceManager = mediaSourceManager;
  64. _subtitleParser = subtitleParser;
  65. _pathManager = pathManager;
  66. _serverConfigurationManager = serverConfigurationManager;
  67. }
  68. private MemoryStream ConvertSubtitles(
  69. Stream stream,
  70. string inputFormat,
  71. string outputFormat,
  72. long startTimeTicks,
  73. long endTimeTicks,
  74. bool preserveOriginalTimestamps,
  75. CancellationToken cancellationToken)
  76. {
  77. var ms = new MemoryStream();
  78. try
  79. {
  80. var trackInfo = _subtitleParser.Parse(stream, inputFormat);
  81. FilterEvents(trackInfo, startTimeTicks, endTimeTicks, preserveOriginalTimestamps);
  82. var writer = GetWriter(outputFormat);
  83. writer.Write(trackInfo, ms, cancellationToken);
  84. ms.Position = 0;
  85. }
  86. catch
  87. {
  88. ms.Dispose();
  89. throw;
  90. }
  91. return ms;
  92. }
  93. private void FilterEvents(SubtitleTrackInfo track, long startPositionTicks, long endTimeTicks, bool preserveTimestamps)
  94. {
  95. // Drop subs that are earlier than what we're looking for
  96. track.TrackEvents = track.TrackEvents
  97. .SkipWhile(i => (i.StartPositionTicks - startPositionTicks) < 0 || (i.EndPositionTicks - startPositionTicks) < 0)
  98. .ToArray();
  99. if (endTimeTicks > 0)
  100. {
  101. track.TrackEvents = track.TrackEvents
  102. .TakeWhile(i => i.StartPositionTicks <= endTimeTicks)
  103. .ToArray();
  104. }
  105. if (!preserveTimestamps)
  106. {
  107. foreach (var trackEvent in track.TrackEvents)
  108. {
  109. trackEvent.EndPositionTicks -= startPositionTicks;
  110. trackEvent.StartPositionTicks -= startPositionTicks;
  111. }
  112. }
  113. }
  114. async Task<Stream> ISubtitleEncoder.GetSubtitles(BaseItem item, string mediaSourceId, int subtitleStreamIndex, string outputFormat, long startTimeTicks, long endTimeTicks, bool preserveOriginalTimestamps, CancellationToken cancellationToken)
  115. {
  116. ArgumentNullException.ThrowIfNull(item);
  117. if (string.IsNullOrWhiteSpace(mediaSourceId))
  118. {
  119. throw new ArgumentNullException(nameof(mediaSourceId));
  120. }
  121. var mediaSources = await _mediaSourceManager.GetPlaybackMediaSources(item, null, true, false, cancellationToken).ConfigureAwait(false);
  122. var mediaSource = mediaSources
  123. .First(i => string.Equals(i.Id, mediaSourceId, StringComparison.OrdinalIgnoreCase));
  124. var subtitleStream = mediaSource.MediaStreams
  125. .First(i => i.Type == MediaStreamType.Subtitle && i.Index == subtitleStreamIndex);
  126. var (stream, inputFormat) = await GetSubtitleStream(mediaSource, subtitleStream, cancellationToken)
  127. .ConfigureAwait(false);
  128. // Return the original if the same format is being requested
  129. // Character encoding was already handled in GetSubtitleStream
  130. if (string.Equals(inputFormat, outputFormat, StringComparison.OrdinalIgnoreCase))
  131. {
  132. return stream;
  133. }
  134. using (stream)
  135. {
  136. return ConvertSubtitles(stream, inputFormat, outputFormat, startTimeTicks, endTimeTicks, preserveOriginalTimestamps, cancellationToken);
  137. }
  138. }
  139. private async Task<(Stream Stream, string Format)> GetSubtitleStream(
  140. MediaSourceInfo mediaSource,
  141. MediaStream subtitleStream,
  142. CancellationToken cancellationToken)
  143. {
  144. var fileInfo = await GetReadableFile(mediaSource, subtitleStream, cancellationToken).ConfigureAwait(false);
  145. var stream = await GetSubtitleStream(fileInfo, cancellationToken).ConfigureAwait(false);
  146. return (stream, fileInfo.Format);
  147. }
  148. private async Task<Stream> GetSubtitleStream(SubtitleInfo fileInfo, CancellationToken cancellationToken)
  149. {
  150. if (fileInfo.IsExternal)
  151. {
  152. var stream = await GetStream(fileInfo.Path, fileInfo.Protocol, cancellationToken).ConfigureAwait(false);
  153. await using (stream.ConfigureAwait(false))
  154. {
  155. var result = await CharsetDetector.DetectFromStreamAsync(stream, cancellationToken).ConfigureAwait(false);
  156. var detected = result.Detected;
  157. stream.Position = 0;
  158. if (detected is not null)
  159. {
  160. _logger.LogDebug("charset {CharSet} detected for {Path}", detected.EncodingName, fileInfo.Path);
  161. using var reader = new StreamReader(stream, detected.Encoding);
  162. var text = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false);
  163. return new MemoryStream(Encoding.UTF8.GetBytes(text));
  164. }
  165. }
  166. }
  167. return AsyncFile.OpenRead(fileInfo.Path);
  168. }
  169. internal async Task<SubtitleInfo> GetReadableFile(
  170. MediaSourceInfo mediaSource,
  171. MediaStream subtitleStream,
  172. CancellationToken cancellationToken)
  173. {
  174. if (!subtitleStream.IsExternal || subtitleStream.Path.EndsWith(".mks", StringComparison.OrdinalIgnoreCase))
  175. {
  176. await ExtractAllExtractableSubtitles(mediaSource, cancellationToken).ConfigureAwait(false);
  177. var outputFileExtension = GetExtractableSubtitleFileExtension(subtitleStream);
  178. var outputFormat = GetExtractableSubtitleFormat(subtitleStream);
  179. var outputPath = GetSubtitleCachePath(mediaSource, subtitleStream.Index, "." + outputFileExtension);
  180. return new SubtitleInfo()
  181. {
  182. Path = outputPath,
  183. Protocol = MediaProtocol.File,
  184. Format = outputFormat,
  185. IsExternal = false
  186. };
  187. }
  188. var currentFormat = (Path.GetExtension(subtitleStream.Path) ?? subtitleStream.Codec)
  189. .TrimStart('.');
  190. // Handle PGS subtitles as raw streams for the client to render
  191. if (MediaStream.IsPgsFormat(currentFormat))
  192. {
  193. return new SubtitleInfo()
  194. {
  195. Path = subtitleStream.Path,
  196. Protocol = _mediaSourceManager.GetPathProtocol(subtitleStream.Path),
  197. Format = "pgssub",
  198. IsExternal = true
  199. };
  200. }
  201. // Fallback to ffmpeg conversion
  202. if (!_subtitleParser.SupportsFileExtension(currentFormat))
  203. {
  204. // Convert
  205. var outputPath = GetSubtitleCachePath(mediaSource, subtitleStream.Index, ".srt");
  206. await ConvertTextSubtitleToSrt(subtitleStream, mediaSource, outputPath, cancellationToken).ConfigureAwait(false);
  207. return new SubtitleInfo()
  208. {
  209. Path = outputPath,
  210. Protocol = MediaProtocol.File,
  211. Format = "srt",
  212. IsExternal = true
  213. };
  214. }
  215. // It's possible that the subtitleStream and mediaSource don't share the same protocol (e.g. .STRM file with local subs)
  216. return new SubtitleInfo()
  217. {
  218. Path = subtitleStream.Path,
  219. Protocol = _mediaSourceManager.GetPathProtocol(subtitleStream.Path),
  220. Format = currentFormat,
  221. IsExternal = true
  222. };
  223. }
  224. private bool TryGetWriter(string format, [NotNullWhen(true)] out ISubtitleWriter? value)
  225. {
  226. ArgumentException.ThrowIfNullOrEmpty(format);
  227. if (string.Equals(format, SubtitleFormat.ASS, StringComparison.OrdinalIgnoreCase))
  228. {
  229. value = new AssWriter();
  230. return true;
  231. }
  232. if (string.Equals(format, "json", StringComparison.OrdinalIgnoreCase))
  233. {
  234. value = new JsonWriter();
  235. return true;
  236. }
  237. if (string.Equals(format, SubtitleFormat.SRT, StringComparison.OrdinalIgnoreCase) || string.Equals(format, SubtitleFormat.SUBRIP, StringComparison.OrdinalIgnoreCase))
  238. {
  239. value = new SrtWriter();
  240. return true;
  241. }
  242. if (string.Equals(format, SubtitleFormat.SSA, StringComparison.OrdinalIgnoreCase))
  243. {
  244. value = new SsaWriter();
  245. return true;
  246. }
  247. if (string.Equals(format, SubtitleFormat.VTT, StringComparison.OrdinalIgnoreCase) || string.Equals(format, SubtitleFormat.WEBVTT, StringComparison.OrdinalIgnoreCase))
  248. {
  249. value = new VttWriter();
  250. return true;
  251. }
  252. if (string.Equals(format, SubtitleFormat.TTML, StringComparison.OrdinalIgnoreCase))
  253. {
  254. value = new TtmlWriter();
  255. return true;
  256. }
  257. value = null;
  258. return false;
  259. }
  260. private ISubtitleWriter GetWriter(string format)
  261. {
  262. if (TryGetWriter(format, out var writer))
  263. {
  264. return writer;
  265. }
  266. throw new ArgumentException("Unsupported format: " + format);
  267. }
  268. /// <summary>
  269. /// Converts the text subtitle to SRT.
  270. /// </summary>
  271. /// <param name="subtitleStream">The subtitle stream.</param>
  272. /// <param name="mediaSource">The input mediaSource.</param>
  273. /// <param name="outputPath">The output path.</param>
  274. /// <param name="cancellationToken">The cancellation token.</param>
  275. /// <returns>Task.</returns>
  276. private async Task ConvertTextSubtitleToSrt(MediaStream subtitleStream, MediaSourceInfo mediaSource, string outputPath, CancellationToken cancellationToken)
  277. {
  278. using (await _semaphoreLocks.LockAsync(outputPath, cancellationToken).ConfigureAwait(false))
  279. {
  280. if (!File.Exists(outputPath))
  281. {
  282. await ConvertTextSubtitleToSrtInternal(subtitleStream, mediaSource, outputPath, cancellationToken).ConfigureAwait(false);
  283. }
  284. }
  285. }
  286. /// <summary>
  287. /// Converts the text subtitle to SRT internal.
  288. /// </summary>
  289. /// <param name="subtitleStream">The subtitle stream.</param>
  290. /// <param name="mediaSource">The input mediaSource.</param>
  291. /// <param name="outputPath">The output path.</param>
  292. /// <param name="cancellationToken">The cancellation token.</param>
  293. /// <returns>Task.</returns>
  294. /// <exception cref="ArgumentNullException">
  295. /// The <c>inputPath</c> or <c>outputPath</c> is <c>null</c>.
  296. /// </exception>
  297. private async Task ConvertTextSubtitleToSrtInternal(MediaStream subtitleStream, MediaSourceInfo mediaSource, string outputPath, CancellationToken cancellationToken)
  298. {
  299. var inputPath = subtitleStream.Path;
  300. ArgumentException.ThrowIfNullOrEmpty(inputPath);
  301. ArgumentException.ThrowIfNullOrEmpty(outputPath);
  302. Directory.CreateDirectory(Path.GetDirectoryName(outputPath) ?? throw new ArgumentException($"Provided path ({outputPath}) is not valid.", nameof(outputPath)));
  303. var encodingParam = await GetSubtitleFileCharacterSet(subtitleStream, subtitleStream.Language, mediaSource, cancellationToken).ConfigureAwait(false);
  304. // FFmpeg automatically convert character encoding when it is UTF-16
  305. // If we specify character encoding, it rejects with "do not specify a character encoding" and "Unable to recode subtitle event"
  306. if ((inputPath.EndsWith(".smi", StringComparison.Ordinal) || inputPath.EndsWith(".sami", StringComparison.Ordinal)) &&
  307. (encodingParam.Equals("UTF-16BE", StringComparison.OrdinalIgnoreCase) ||
  308. encodingParam.Equals("UTF-16LE", StringComparison.OrdinalIgnoreCase)))
  309. {
  310. encodingParam = string.Empty;
  311. }
  312. else if (!string.IsNullOrEmpty(encodingParam))
  313. {
  314. encodingParam = " -sub_charenc " + encodingParam;
  315. }
  316. int exitCode;
  317. using (var process = new Process
  318. {
  319. StartInfo = new ProcessStartInfo
  320. {
  321. CreateNoWindow = true,
  322. UseShellExecute = false,
  323. FileName = _mediaEncoder.EncoderPath,
  324. Arguments = string.Format(CultureInfo.InvariantCulture, "{0} -i \"{1}\" -c:s srt \"{2}\"", encodingParam, inputPath, outputPath),
  325. WindowStyle = ProcessWindowStyle.Hidden,
  326. ErrorDialog = false
  327. },
  328. EnableRaisingEvents = true
  329. })
  330. {
  331. _logger.LogInformation("{0} {1}", process.StartInfo.FileName, process.StartInfo.Arguments);
  332. try
  333. {
  334. process.Start();
  335. }
  336. catch (Exception ex)
  337. {
  338. _logger.LogError(ex, "Error starting ffmpeg");
  339. throw;
  340. }
  341. try
  342. {
  343. var timeoutMinutes = _serverConfigurationManager.GetEncodingOptions().SubtitleExtractionTimeoutMinutes;
  344. await process.WaitForExitAsync(TimeSpan.FromMinutes(timeoutMinutes)).ConfigureAwait(false);
  345. exitCode = process.ExitCode;
  346. }
  347. catch (OperationCanceledException)
  348. {
  349. process.Kill(true);
  350. exitCode = -1;
  351. }
  352. }
  353. var failed = false;
  354. if (exitCode == -1)
  355. {
  356. failed = true;
  357. if (File.Exists(outputPath))
  358. {
  359. try
  360. {
  361. _logger.LogInformation("Deleting converted subtitle due to failure: {Path}", outputPath);
  362. _fileSystem.DeleteFile(outputPath);
  363. }
  364. catch (IOException ex)
  365. {
  366. _logger.LogError(ex, "Error deleting converted subtitle {Path}", outputPath);
  367. }
  368. }
  369. }
  370. else if (!File.Exists(outputPath))
  371. {
  372. failed = true;
  373. }
  374. if (failed)
  375. {
  376. _logger.LogError("ffmpeg subtitle conversion failed for {Path}", inputPath);
  377. throw new FfmpegException(
  378. string.Format(CultureInfo.InvariantCulture, "ffmpeg subtitle conversion failed for {0}", inputPath));
  379. }
  380. await SetAssFont(outputPath, cancellationToken).ConfigureAwait(false);
  381. _logger.LogInformation("ffmpeg subtitle conversion succeeded for {Path}", inputPath);
  382. }
  383. private string GetExtractableSubtitleFormat(MediaStream subtitleStream)
  384. {
  385. if (string.Equals(subtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
  386. || string.Equals(subtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)
  387. || string.Equals(subtitleStream.Codec, "pgssub", StringComparison.OrdinalIgnoreCase))
  388. {
  389. return subtitleStream.Codec;
  390. }
  391. else
  392. {
  393. return "srt";
  394. }
  395. }
  396. private string GetExtractableSubtitleFileExtension(MediaStream subtitleStream)
  397. {
  398. // Using .pgssub as file extension is not allowed by ffmpeg. The file extension for pgs subtitles is .sup.
  399. if (string.Equals(subtitleStream.Codec, "pgssub", StringComparison.OrdinalIgnoreCase))
  400. {
  401. return "sup";
  402. }
  403. else
  404. {
  405. return GetExtractableSubtitleFormat(subtitleStream);
  406. }
  407. }
  408. private bool IsCodecCopyable(string codec)
  409. {
  410. return string.Equals(codec, "ass", StringComparison.OrdinalIgnoreCase)
  411. || string.Equals(codec, "ssa", StringComparison.OrdinalIgnoreCase)
  412. || string.Equals(codec, "srt", StringComparison.OrdinalIgnoreCase)
  413. || string.Equals(codec, "subrip", StringComparison.OrdinalIgnoreCase)
  414. || string.Equals(codec, "pgssub", StringComparison.OrdinalIgnoreCase);
  415. }
  416. /// <inheritdoc />
  417. public async Task ExtractAllExtractableSubtitles(MediaSourceInfo mediaSource, CancellationToken cancellationToken)
  418. {
  419. var locks = new List<IDisposable>();
  420. var extractableStreams = new List<MediaStream>();
  421. try
  422. {
  423. var subtitleStreams = mediaSource.MediaStreams
  424. .Where(stream => stream is { IsExtractableSubtitleStream: true, SupportsExternalStream: true });
  425. foreach (var subtitleStream in subtitleStreams)
  426. {
  427. if (subtitleStream.IsExternal && !subtitleStream.Path.EndsWith(".mks", StringComparison.OrdinalIgnoreCase))
  428. {
  429. continue;
  430. }
  431. var outputPath = GetSubtitleCachePath(mediaSource, subtitleStream.Index, "." + GetExtractableSubtitleFileExtension(subtitleStream));
  432. var releaser = await _semaphoreLocks.LockAsync(outputPath, cancellationToken).ConfigureAwait(false);
  433. if (File.Exists(outputPath))
  434. {
  435. releaser.Dispose();
  436. continue;
  437. }
  438. locks.Add(releaser);
  439. extractableStreams.Add(subtitleStream);
  440. }
  441. if (extractableStreams.Count > 0)
  442. {
  443. await ExtractAllExtractableSubtitlesInternal(mediaSource, extractableStreams, cancellationToken).ConfigureAwait(false);
  444. await ExtractAllExtractableSubtitlesMKS(mediaSource, extractableStreams, cancellationToken).ConfigureAwait(false);
  445. }
  446. }
  447. catch (Exception ex)
  448. {
  449. _logger.LogWarning(ex, "Unable to get streams for File:{File}", mediaSource.Path);
  450. }
  451. finally
  452. {
  453. locks.ForEach(x => x.Dispose());
  454. }
  455. }
  456. private async Task ExtractAllExtractableSubtitlesMKS(
  457. MediaSourceInfo mediaSource,
  458. List<MediaStream> subtitleStreams,
  459. CancellationToken cancellationToken)
  460. {
  461. var mksFiles = new List<string>();
  462. foreach (var subtitleStream in subtitleStreams)
  463. {
  464. if (string.IsNullOrEmpty(subtitleStream.Path) || !subtitleStream.Path.EndsWith(".mks", StringComparison.OrdinalIgnoreCase))
  465. {
  466. continue;
  467. }
  468. if (!mksFiles.Contains(subtitleStream.Path))
  469. {
  470. mksFiles.Add(subtitleStream.Path);
  471. }
  472. }
  473. if (mksFiles.Count == 0)
  474. {
  475. return;
  476. }
  477. foreach (string mksFile in mksFiles)
  478. {
  479. var inputPath = _mediaEncoder.GetInputArgument(mksFile, mediaSource);
  480. var outputPaths = new List<string>();
  481. var args = string.Format(
  482. CultureInfo.InvariantCulture,
  483. "-i {0} -copyts",
  484. inputPath);
  485. foreach (var subtitleStream in subtitleStreams)
  486. {
  487. if (!subtitleStream.Path.Equals(mksFile, StringComparison.OrdinalIgnoreCase))
  488. {
  489. continue;
  490. }
  491. var outputPath = GetSubtitleCachePath(mediaSource, subtitleStream.Index, "." + GetExtractableSubtitleFileExtension(subtitleStream));
  492. var outputCodec = IsCodecCopyable(subtitleStream.Codec) ? "copy" : "srt";
  493. var streamIndex = EncodingHelper.FindIndex(mediaSource.MediaStreams, subtitleStream);
  494. if (streamIndex == -1)
  495. {
  496. _logger.LogError("Cannot find subtitle stream index for {InputPath} ({Index}), skipping this stream", inputPath, subtitleStream.Index);
  497. continue;
  498. }
  499. Directory.CreateDirectory(Path.GetDirectoryName(outputPath) ?? throw new FileNotFoundException($"Calculated path ({outputPath}) is not valid."));
  500. outputPaths.Add(outputPath);
  501. args += string.Format(
  502. CultureInfo.InvariantCulture,
  503. " -map 0:{0} -an -vn -c:s {1} \"{2}\"",
  504. streamIndex,
  505. outputCodec,
  506. outputPath);
  507. }
  508. await ExtractSubtitlesForFile(inputPath, args, outputPaths, cancellationToken).ConfigureAwait(false);
  509. }
  510. }
  511. private async Task ExtractAllExtractableSubtitlesInternal(
  512. MediaSourceInfo mediaSource,
  513. List<MediaStream> subtitleStreams,
  514. CancellationToken cancellationToken)
  515. {
  516. var inputPath = _mediaEncoder.GetInputArgument(mediaSource.Path, mediaSource);
  517. var outputPaths = new List<string>();
  518. var args = string.Format(
  519. CultureInfo.InvariantCulture,
  520. "-i {0} -copyts",
  521. inputPath);
  522. foreach (var subtitleStream in subtitleStreams)
  523. {
  524. if (!string.IsNullOrEmpty(subtitleStream.Path) && subtitleStream.Path.EndsWith(".mks", StringComparison.OrdinalIgnoreCase))
  525. {
  526. _logger.LogDebug("Subtitle {Index} for file {InputPath} is part in an MKS file. Skipping", inputPath, subtitleStream.Index);
  527. continue;
  528. }
  529. var outputPath = GetSubtitleCachePath(mediaSource, subtitleStream.Index, "." + GetExtractableSubtitleFileExtension(subtitleStream));
  530. var outputCodec = IsCodecCopyable(subtitleStream.Codec) ? "copy" : "srt";
  531. var streamIndex = EncodingHelper.FindIndex(mediaSource.MediaStreams, subtitleStream);
  532. if (streamIndex == -1)
  533. {
  534. _logger.LogError("Cannot find subtitle stream index for {InputPath} ({Index}), skipping this stream", inputPath, subtitleStream.Index);
  535. continue;
  536. }
  537. Directory.CreateDirectory(Path.GetDirectoryName(outputPath) ?? throw new FileNotFoundException($"Calculated path ({outputPath}) is not valid."));
  538. outputPaths.Add(outputPath);
  539. args += string.Format(
  540. CultureInfo.InvariantCulture,
  541. " -map 0:{0} -an -vn -c:s {1} \"{2}\"",
  542. streamIndex,
  543. outputCodec,
  544. outputPath);
  545. }
  546. if (outputPaths.Count == 0)
  547. {
  548. return;
  549. }
  550. await ExtractSubtitlesForFile(inputPath, args, outputPaths, cancellationToken).ConfigureAwait(false);
  551. }
  552. private async Task ExtractSubtitlesForFile(
  553. string inputPath,
  554. string args,
  555. List<string> outputPaths,
  556. CancellationToken cancellationToken)
  557. {
  558. int exitCode;
  559. using (var process = new Process
  560. {
  561. StartInfo = new ProcessStartInfo
  562. {
  563. CreateNoWindow = true,
  564. UseShellExecute = false,
  565. FileName = _mediaEncoder.EncoderPath,
  566. Arguments = args,
  567. WindowStyle = ProcessWindowStyle.Hidden,
  568. ErrorDialog = false
  569. },
  570. EnableRaisingEvents = true
  571. })
  572. {
  573. _logger.LogInformation("{File} {Arguments}", process.StartInfo.FileName, process.StartInfo.Arguments);
  574. try
  575. {
  576. process.Start();
  577. }
  578. catch (Exception ex)
  579. {
  580. _logger.LogError(ex, "Error starting ffmpeg");
  581. throw;
  582. }
  583. try
  584. {
  585. var timeoutMinutes = _serverConfigurationManager.GetEncodingOptions().SubtitleExtractionTimeoutMinutes;
  586. await process.WaitForExitAsync(TimeSpan.FromMinutes(timeoutMinutes)).ConfigureAwait(false);
  587. exitCode = process.ExitCode;
  588. }
  589. catch (OperationCanceledException)
  590. {
  591. process.Kill(true);
  592. exitCode = -1;
  593. }
  594. }
  595. var failed = false;
  596. if (exitCode == -1)
  597. {
  598. failed = true;
  599. foreach (var outputPath in outputPaths)
  600. {
  601. try
  602. {
  603. _logger.LogWarning("Deleting extracted subtitle due to failure: {Path}", outputPath);
  604. _fileSystem.DeleteFile(outputPath);
  605. }
  606. catch (FileNotFoundException)
  607. {
  608. }
  609. catch (IOException ex)
  610. {
  611. _logger.LogError(ex, "Error deleting extracted subtitle {Path}", outputPath);
  612. }
  613. }
  614. }
  615. else
  616. {
  617. foreach (var outputPath in outputPaths)
  618. {
  619. if (!File.Exists(outputPath))
  620. {
  621. _logger.LogError("ffmpeg subtitle extraction failed for {InputPath} to {OutputPath}", inputPath, outputPath);
  622. failed = true;
  623. continue;
  624. }
  625. if (outputPath.EndsWith("ass", StringComparison.OrdinalIgnoreCase))
  626. {
  627. await SetAssFont(outputPath, cancellationToken).ConfigureAwait(false);
  628. }
  629. _logger.LogInformation("ffmpeg subtitle extraction completed for {InputPath} to {OutputPath}", inputPath, outputPath);
  630. }
  631. }
  632. if (failed)
  633. {
  634. throw new FfmpegException(
  635. string.Format(CultureInfo.InvariantCulture, "ffmpeg subtitle extraction failed for {0}", inputPath));
  636. }
  637. }
  638. /// <summary>
  639. /// Extracts the text subtitle.
  640. /// </summary>
  641. /// <param name="mediaSource">The mediaSource.</param>
  642. /// <param name="subtitleStream">The subtitle stream.</param>
  643. /// <param name="outputCodec">The output codec.</param>
  644. /// <param name="outputPath">The output path.</param>
  645. /// <param name="cancellationToken">The cancellation token.</param>
  646. /// <returns>Task.</returns>
  647. /// <exception cref="ArgumentException">Must use inputPath list overload.</exception>
  648. private async Task ExtractTextSubtitle(
  649. MediaSourceInfo mediaSource,
  650. MediaStream subtitleStream,
  651. string outputCodec,
  652. string outputPath,
  653. CancellationToken cancellationToken)
  654. {
  655. using (await _semaphoreLocks.LockAsync(outputPath, cancellationToken).ConfigureAwait(false))
  656. {
  657. if (!File.Exists(outputPath))
  658. {
  659. var subtitleStreamIndex = EncodingHelper.FindIndex(mediaSource.MediaStreams, subtitleStream);
  660. var args = _mediaEncoder.GetInputArgument(mediaSource.Path, mediaSource);
  661. if (subtitleStream.IsExternal)
  662. {
  663. args = _mediaEncoder.GetExternalSubtitleInputArgument(subtitleStream.Path);
  664. }
  665. await ExtractTextSubtitleInternal(
  666. args,
  667. subtitleStreamIndex,
  668. outputCodec,
  669. outputPath,
  670. cancellationToken).ConfigureAwait(false);
  671. }
  672. }
  673. }
  674. private async Task ExtractTextSubtitleInternal(
  675. string inputPath,
  676. int subtitleStreamIndex,
  677. string outputCodec,
  678. string outputPath,
  679. CancellationToken cancellationToken)
  680. {
  681. ArgumentException.ThrowIfNullOrEmpty(inputPath);
  682. ArgumentException.ThrowIfNullOrEmpty(outputPath);
  683. Directory.CreateDirectory(Path.GetDirectoryName(outputPath) ?? throw new ArgumentException($"Provided path ({outputPath}) is not valid.", nameof(outputPath)));
  684. var processArgs = string.Format(
  685. CultureInfo.InvariantCulture,
  686. "-i {0} -copyts -map 0:{1} -an -vn -c:s {2} \"{3}\"",
  687. inputPath,
  688. subtitleStreamIndex,
  689. outputCodec,
  690. outputPath);
  691. int exitCode;
  692. using (var process = new Process
  693. {
  694. StartInfo = new ProcessStartInfo
  695. {
  696. CreateNoWindow = true,
  697. UseShellExecute = false,
  698. FileName = _mediaEncoder.EncoderPath,
  699. Arguments = processArgs,
  700. WindowStyle = ProcessWindowStyle.Hidden,
  701. ErrorDialog = false
  702. },
  703. EnableRaisingEvents = true
  704. })
  705. {
  706. _logger.LogInformation("{File} {Arguments}", process.StartInfo.FileName, process.StartInfo.Arguments);
  707. try
  708. {
  709. process.Start();
  710. }
  711. catch (Exception ex)
  712. {
  713. _logger.LogError(ex, "Error starting ffmpeg");
  714. throw;
  715. }
  716. try
  717. {
  718. var timeoutMinutes = _serverConfigurationManager.GetEncodingOptions().SubtitleExtractionTimeoutMinutes;
  719. await process.WaitForExitAsync(TimeSpan.FromMinutes(timeoutMinutes)).ConfigureAwait(false);
  720. exitCode = process.ExitCode;
  721. }
  722. catch (OperationCanceledException)
  723. {
  724. process.Kill(true);
  725. exitCode = -1;
  726. }
  727. }
  728. var failed = false;
  729. if (exitCode == -1)
  730. {
  731. failed = true;
  732. try
  733. {
  734. _logger.LogWarning("Deleting extracted subtitle due to failure: {Path}", outputPath);
  735. _fileSystem.DeleteFile(outputPath);
  736. }
  737. catch (FileNotFoundException)
  738. {
  739. }
  740. catch (IOException ex)
  741. {
  742. _logger.LogError(ex, "Error deleting extracted subtitle {Path}", outputPath);
  743. }
  744. }
  745. else if (!File.Exists(outputPath))
  746. {
  747. failed = true;
  748. }
  749. if (failed)
  750. {
  751. _logger.LogError("ffmpeg subtitle extraction failed for {InputPath} to {OutputPath}", inputPath, outputPath);
  752. throw new FfmpegException(
  753. string.Format(CultureInfo.InvariantCulture, "ffmpeg subtitle extraction failed for {0} to {1}", inputPath, outputPath));
  754. }
  755. _logger.LogInformation("ffmpeg subtitle extraction completed for {InputPath} to {OutputPath}", inputPath, outputPath);
  756. if (string.Equals(outputCodec, "ass", StringComparison.OrdinalIgnoreCase))
  757. {
  758. await SetAssFont(outputPath, cancellationToken).ConfigureAwait(false);
  759. }
  760. }
  761. /// <summary>
  762. /// Sets the ass font.
  763. /// </summary>
  764. /// <param name="file">The file.</param>
  765. /// <param name="cancellationToken">The token to monitor for cancellation requests. The default value is <c>System.Threading.CancellationToken.None</c>.</param>
  766. /// <returns>Task.</returns>
  767. private async Task SetAssFont(string file, CancellationToken cancellationToken = default)
  768. {
  769. _logger.LogInformation("Setting ass font within {File}", file);
  770. string text;
  771. Encoding encoding;
  772. using (var fileStream = AsyncFile.OpenRead(file))
  773. using (var reader = new StreamReader(fileStream, true))
  774. {
  775. encoding = reader.CurrentEncoding;
  776. text = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false);
  777. }
  778. var newText = text.Replace(",Arial,", ",Arial Unicode MS,", StringComparison.Ordinal);
  779. if (!string.Equals(text, newText, StringComparison.Ordinal))
  780. {
  781. var fileStream = new FileStream(file, FileMode.Create, FileAccess.Write, FileShare.None, IODefaults.FileStreamBufferSize, FileOptions.Asynchronous);
  782. await using (fileStream.ConfigureAwait(false))
  783. {
  784. var writer = new StreamWriter(fileStream, encoding);
  785. await using (writer.ConfigureAwait(false))
  786. {
  787. await writer.WriteAsync(newText.AsMemory(), cancellationToken).ConfigureAwait(false);
  788. }
  789. }
  790. }
  791. }
  792. private string GetSubtitleCachePath(MediaSourceInfo mediaSource, int subtitleStreamIndex, string outputSubtitleExtension)
  793. {
  794. return _pathManager.GetSubtitlePath(mediaSource.Id, subtitleStreamIndex, outputSubtitleExtension);
  795. }
  796. /// <inheritdoc />
  797. public async Task<string> GetSubtitleFileCharacterSet(MediaStream subtitleStream, string language, MediaSourceInfo mediaSource, CancellationToken cancellationToken)
  798. {
  799. var subtitleCodec = subtitleStream.Codec;
  800. var path = subtitleStream.Path;
  801. if (path.EndsWith(".mks", StringComparison.OrdinalIgnoreCase))
  802. {
  803. path = GetSubtitleCachePath(mediaSource, subtitleStream.Index, "." + subtitleCodec);
  804. await ExtractTextSubtitle(mediaSource, subtitleStream, subtitleCodec, path, cancellationToken)
  805. .ConfigureAwait(false);
  806. }
  807. var stream = await GetStream(path, mediaSource.Protocol, cancellationToken).ConfigureAwait(false);
  808. await using (stream.ConfigureAwait(false))
  809. {
  810. var result = await CharsetDetector.DetectFromStreamAsync(stream, cancellationToken).ConfigureAwait(false);
  811. var charset = result.Detected?.EncodingName ?? string.Empty;
  812. // UTF16 is automatically converted to UTF8 by FFmpeg, do not specify a character encoding
  813. if ((path.EndsWith(".ass", StringComparison.Ordinal) || path.EndsWith(".ssa", StringComparison.Ordinal) || path.EndsWith(".srt", StringComparison.Ordinal))
  814. && (string.Equals(charset, "utf-16le", StringComparison.OrdinalIgnoreCase)
  815. || string.Equals(charset, "utf-16be", StringComparison.OrdinalIgnoreCase)))
  816. {
  817. charset = string.Empty;
  818. }
  819. _logger.LogDebug("charset {0} detected for {Path}", charset, path);
  820. return charset;
  821. }
  822. }
  823. private async Task<Stream> GetStream(string path, MediaProtocol protocol, CancellationToken cancellationToken)
  824. {
  825. switch (protocol)
  826. {
  827. case MediaProtocol.Http:
  828. {
  829. using var response = await _httpClientFactory.CreateClient(NamedClient.Default)
  830. .GetAsync(new Uri(path), cancellationToken)
  831. .ConfigureAwait(false);
  832. return await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
  833. }
  834. case MediaProtocol.File:
  835. return AsyncFile.OpenRead(path);
  836. default:
  837. throw new ArgumentOutOfRangeException(nameof(protocol));
  838. }
  839. }
  840. public async Task<string> GetSubtitleFilePath(MediaStream subtitleStream, MediaSourceInfo mediaSource, CancellationToken cancellationToken)
  841. {
  842. var info = await GetReadableFile(mediaSource, subtitleStream, cancellationToken)
  843. .ConfigureAwait(false);
  844. return info.Path;
  845. }
  846. /// <inheritdoc />
  847. public void Dispose()
  848. {
  849. _semaphoreLocks.Dispose();
  850. }
  851. #pragma warning disable CA1034 // Nested types should not be visible
  852. // Only public for the unit tests
  853. public readonly record struct SubtitleInfo
  854. {
  855. public string Path { get; init; }
  856. public MediaProtocol Protocol { get; init; }
  857. public string Format { get; init; }
  858. public bool IsExternal { get; init; }
  859. }
  860. }
  861. }