using System; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; using Emby.Server.Implementations.Serialization; using Jellyfin.Database.Implementations; using Jellyfin.Server.Implementations.SystemBackupService; using Jellyfin.Server.Migrations.Stages; using Jellyfin.Server.ServerSetupApp; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.SystemBackupService; using MediaBrowser.Model.Configuration; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.Extensions.Logging; namespace Jellyfin.Server.Migrations; /// /// Handles Migration of the Jellyfin data structure. /// internal class JellyfinMigrationService { private const string DbFilename = "library.db"; private readonly IDbContextFactory _dbContextFactory; private readonly ILoggerFactory _loggerFactory; private readonly IStartupLogger _startupLogger; private readonly IBackupService? _backupService; private readonly IJellyfinDatabaseProvider? _jellyfinDatabaseProvider; private readonly IApplicationPaths _applicationPaths; private (string? LibraryDb, string? JellyfinDb, BackupManifestDto? FullBackup) _backupKey; /// /// Initializes a new instance of the class. /// /// Provides access to the jellyfin database. /// The logger factory. /// The startup logger for Startup UI intigration. /// Application paths for library.db backup. /// The jellyfin backup service. /// The jellyfin database provider. public JellyfinMigrationService( IDbContextFactory dbContextFactory, ILoggerFactory loggerFactory, IStartupLogger startupLogger, IApplicationPaths applicationPaths, IBackupService? backupService = null, IJellyfinDatabaseProvider? jellyfinDatabaseProvider = null) { _dbContextFactory = dbContextFactory; _loggerFactory = loggerFactory; _startupLogger = startupLogger; _backupService = backupService; _jellyfinDatabaseProvider = jellyfinDatabaseProvider; _applicationPaths = applicationPaths; #pragma warning disable CS0618 // Type or member is obsolete Migrations = [.. typeof(IMigrationRoutine).Assembly.GetTypes().Where(e => typeof(IMigrationRoutine).IsAssignableFrom(e) || typeof(IAsyncMigrationRoutine).IsAssignableFrom(e)) .Select(e => (Type: e, Metadata: e.GetCustomAttribute(), Backup: e.GetCustomAttributes())) .Where(e => e.Metadata != null) .GroupBy(e => e.Metadata!.Stage) .Select(f => { var stage = new MigrationStage(f.Key); foreach (var item in f) { JellyfinMigrationBackupAttribute? backupMetadata = null; if (item.Backup?.Any() == true) { backupMetadata = item.Backup.Aggregate(MergeBackupAttributes); } stage.Add(new(item.Type, item.Metadata!, backupMetadata)); } return stage; })]; #pragma warning restore CS0618 // Type or member is obsolete } private interface IInternalMigration { Task PerformAsync(IStartupLogger logger); } private HashSet Migrations { get; set; } public async Task CheckFirstTimeRunOrMigration(IApplicationPaths appPaths) { var logger = _startupLogger.With(_loggerFactory.CreateLogger()).BeginGroup($"Migration Startup"); logger.LogInformation("Initialise Migration service."); var xmlSerializer = new MyXmlSerializer(); var serverConfig = File.Exists(appPaths.SystemConfigurationFilePath) ? (ServerConfiguration)xmlSerializer.DeserializeFromFile(typeof(ServerConfiguration), appPaths.SystemConfigurationFilePath)! : new ServerConfiguration(); if (!serverConfig.IsStartupWizardCompleted) { logger.LogInformation("System initialisation detected. Seed data."); var flatApplyMigrations = Migrations.SelectMany(e => e.Where(f => !f.Metadata.RunMigrationOnSetup)).ToArray(); var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); await using (dbContext.ConfigureAwait(false)) { var historyRepository = dbContext.GetService(); await historyRepository.CreateIfNotExistsAsync().ConfigureAwait(false); var appliedMigrations = await dbContext.Database.GetAppliedMigrationsAsync().ConfigureAwait(false); var startupScripts = flatApplyMigrations .Where(e => !appliedMigrations.Any(f => f != e.BuildCodeMigrationId())) .Select(e => (Migration: e.Metadata, Script: historyRepository.GetInsertScript(new HistoryRow(e.BuildCodeMigrationId(), GetJellyfinVersion())))) .ToArray(); foreach (var item in startupScripts) { logger.LogInformation("Seed migration {Key}-{Name}.", item.Migration.Key, item.Migration.Name); await dbContext.Database.ExecuteSqlRawAsync(item.Script).ConfigureAwait(false); } } logger.LogInformation("Migration system initialisation completed."); } else { // migrate any existing migration.xml files var migrationConfigPath = Path.Join(appPaths.ConfigurationDirectoryPath, "migrations.xml"); var migrationOptions = File.Exists(migrationConfigPath) ? (MigrationOptions)xmlSerializer.DeserializeFromFile(typeof(MigrationOptions), migrationConfigPath)! : null; if (migrationOptions != null && migrationOptions.Applied.Count > 0) { logger.LogInformation("Old migration style migration.xml detected. Migrate now."); try { var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); await using (dbContext.ConfigureAwait(false)) { var historyRepository = dbContext.GetService(); var appliedMigrations = await dbContext.Database.GetAppliedMigrationsAsync().ConfigureAwait(false); var lastOldAppliedMigration = Migrations .SelectMany(e => e.Where(e => e.Metadata.Key is not null)) // only consider migrations that have the key set as its the reference marker for legacy migrations. .Where(e => migrationOptions.Applied.Any(f => f.Id.Equals(e.Metadata.Key!.Value))) .Where(e => !appliedMigrations.Contains(e.BuildCodeMigrationId())) .OrderBy(e => e.BuildCodeMigrationId()) .Last(); // this is the latest migration applied in the old migration.xml IReadOnlyList oldMigrations = [ .. Migrations .SelectMany(e => e) .OrderBy(e => e.BuildCodeMigrationId()) .TakeWhile(e => e.BuildCodeMigrationId() != lastOldAppliedMigration.BuildCodeMigrationId()), lastOldAppliedMigration ]; // those are all migrations that had to run in the old migration system, even if not noted in the migration.xml file. var startupScripts = oldMigrations.Select(e => (Migration: e.Metadata, Script: historyRepository.GetInsertScript(new HistoryRow(e.BuildCodeMigrationId(), GetJellyfinVersion())))); foreach (var item in startupScripts) { logger.LogInformation("Migrate migration {Key}-{Name}.", item.Migration.Key, item.Migration.Name); await dbContext.Database.ExecuteSqlRawAsync(item.Script).ConfigureAwait(false); } logger.LogInformation("Rename old migration.xml to migration.xml.backup"); File.Move(migrationConfigPath, Path.ChangeExtension(migrationConfigPath, ".xml.backup"), true); } } catch (Exception ex) { logger.LogCritical(ex, "Failed to apply migrations"); throw; } } } } public async Task MigrateStepAsync(JellyfinMigrationStageTypes stage, IServiceProvider? serviceProvider) { var logger = _startupLogger.With(_loggerFactory.CreateLogger()).BeginGroup($"Migrate stage {stage}."); ICollection migrationStage = (Migrations.FirstOrDefault(e => e.Stage == stage) as ICollection) ?? []; var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); await using (dbContext.ConfigureAwait(false)) { var historyRepository = dbContext.GetService(); var migrationsAssembly = dbContext.GetService(); var appliedMigrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); var pendingCodeMigrations = migrationStage .Where(e => appliedMigrations.All(f => f.MigrationId != e.BuildCodeMigrationId())) .Select(e => (Key: e.BuildCodeMigrationId(), Migration: new InternalCodeMigration(e, serviceProvider, dbContext))) .ToArray(); (string Key, InternalDatabaseMigration Migration)[] pendingDatabaseMigrations = []; if (stage is JellyfinMigrationStageTypes.CoreInitialisation) { pendingDatabaseMigrations = migrationsAssembly.Migrations.Where(f => appliedMigrations.All(e => e.MigrationId != f.Key)) .Select(e => (Key: e.Key, Migration: new InternalDatabaseMigration(e, dbContext))) .ToArray(); } (string Key, IInternalMigration Migration)[] pendingMigrations = [.. pendingCodeMigrations, .. pendingDatabaseMigrations]; logger.LogInformation("There are {Pending} migrations for stage {Stage}.", pendingCodeMigrations.Length, stage); var migrations = pendingMigrations.OrderBy(e => e.Key).ToArray(); foreach (var item in migrations) { var migrationLogger = logger.With(_loggerFactory.CreateLogger(item.Migration.GetType().Name)).BeginGroup($"{item.Key}"); try { migrationLogger.LogInformation("Perform migration {Name}", item.Key); await item.Migration.PerformAsync(migrationLogger).ConfigureAwait(false); migrationLogger.LogInformation("Migration {Name} was successfully applied", item.Key); } catch (Exception ex) { migrationLogger.LogCritical("Error: {Error}", ex.Message); migrationLogger.LogError(ex, "Migration {Name} failed", item.Key); if (_backupKey != default && _backupService is not null && _jellyfinDatabaseProvider is not null) { if (_backupKey.LibraryDb is not null) { migrationLogger.LogInformation("Attempt to rollback librarydb."); try { var libraryDbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); File.Move(_backupKey.LibraryDb, libraryDbPath, true); } catch (Exception inner) { migrationLogger.LogCritical(inner, "Could not rollback {LibraryPath}. Manual intervention might be required to restore a operational state.", _backupKey.LibraryDb); } } if (_backupKey.JellyfinDb is not null) { migrationLogger.LogInformation("Attempt to rollback JellyfinDb."); try { await _jellyfinDatabaseProvider.RestoreBackupFast(_backupKey.JellyfinDb, CancellationToken.None).ConfigureAwait(false); } catch (Exception inner) { migrationLogger.LogCritical(inner, "Could not rollback {LibraryPath}. Manual intervention might be required to restore a operational state.", _backupKey.JellyfinDb); } } if (_backupKey.FullBackup is not null) { migrationLogger.LogInformation("Attempt to rollback from backup."); try { await _backupService.RestoreBackupAsync(_backupKey.FullBackup.Path).ConfigureAwait(false); } catch (Exception inner) { migrationLogger.LogCritical(inner, "Could not rollback from backup {Backup}. Manual intervention might be required to restore a operational state.", _backupKey.FullBackup.Path); } } } throw; } } } } private static string GetJellyfinVersion() { return Assembly.GetEntryAssembly()!.GetName().Version!.ToString(); } public async Task CleanupSystemAfterMigration(ILogger logger) { if (_backupKey != default) { if (_backupKey.LibraryDb is not null) { logger.LogInformation("Attempt to cleanup librarydb backup."); try { File.Delete(_backupKey.LibraryDb); } catch (Exception inner) { logger.LogCritical(inner, "Could not cleanup {LibraryPath}.", _backupKey.LibraryDb); } } if (_backupKey.JellyfinDb is not null && _jellyfinDatabaseProvider is not null) { logger.LogInformation("Attempt to cleanup JellyfinDb backup."); try { await _jellyfinDatabaseProvider.DeleteBackup(_backupKey.JellyfinDb).ConfigureAwait(false); } catch (Exception inner) { logger.LogCritical(inner, "Could not cleanup {LibraryPath}.", _backupKey.JellyfinDb); } } if (_backupKey.FullBackup is not null) { logger.LogInformation("Attempt to cleanup from migration backup."); try { File.Delete(_backupKey.FullBackup.Path); } catch (Exception inner) { logger.LogCritical(inner, "Could not cleanup backup {Backup}.", _backupKey.FullBackup.Path); } } } } public async Task PrepareSystemForMigration(ILogger logger) { logger.LogInformation("Prepare system for possible migrations"); JellyfinMigrationBackupAttribute backupInstruction; IReadOnlyList appliedMigrations; var dbContext = await _dbContextFactory.CreateDbContextAsync().ConfigureAwait(false); await using (dbContext.ConfigureAwait(false)) { var historyRepository = dbContext.GetService(); var migrationsAssembly = dbContext.GetService(); appliedMigrations = await historyRepository.GetAppliedMigrationsAsync().ConfigureAwait(false); backupInstruction = new JellyfinMigrationBackupAttribute() { JellyfinDb = migrationsAssembly.Migrations.Any(f => appliedMigrations.All(e => e.MigrationId != f.Key)) }; } backupInstruction = Migrations.SelectMany(e => e) .Where(e => appliedMigrations.All(f => f.MigrationId != e.BuildCodeMigrationId())) .Select(e => e.BackupRequirements) .Where(e => e is not null) .Aggregate(backupInstruction, MergeBackupAttributes!); if (backupInstruction.LegacyLibraryDb) { logger.LogInformation("A migration will attempt to modify the library.db, will attempt to backup the file now."); // for legacy migrations that still operates on the library.db var libraryDbPath = Path.Combine(_applicationPaths.DataPath, DbFilename); if (File.Exists(libraryDbPath)) { for (int i = 1; ; i++) { var bakPath = string.Format(CultureInfo.InvariantCulture, "{0}.bak{1}", libraryDbPath, i); if (!File.Exists(bakPath)) { try { logger.LogInformation("Backing up {Library} to {BackupPath}", DbFilename, bakPath); File.Copy(libraryDbPath, bakPath); _backupKey = (bakPath, _backupKey.JellyfinDb, _backupKey.FullBackup); logger.LogInformation("{Library} backed up to {BackupPath}", DbFilename, bakPath); break; } catch (Exception ex) { logger.LogError(ex, "Cannot make a backup of {Library} at path {BackupPath}", DbFilename, bakPath); throw; } } } logger.LogInformation("{Library} has been backed up as {BackupPath}", DbFilename, _backupKey.LibraryDb); } else { logger.LogError("Cannot make a backup of {Library} at path {BackupPath} because file could not be found at {LibraryPath}", DbFilename, libraryDbPath, _applicationPaths.DataPath); } } if (backupInstruction.JellyfinDb && _jellyfinDatabaseProvider != null) { logger.LogInformation("A migration will attempt to modify the jellyfin.db, will attempt to backup the file now."); _backupKey = (_backupKey.LibraryDb, await _jellyfinDatabaseProvider.MigrationBackupFast(CancellationToken.None).ConfigureAwait(false), _backupKey.FullBackup); logger.LogInformation("Jellyfin database has been backed up as {BackupPath}", _backupKey.JellyfinDb); } if (_backupService is not null && (backupInstruction.Metadata || backupInstruction.Subtitles || backupInstruction.Trickplay)) { logger.LogInformation("A migration will attempt to modify system resources. Will attempt to create backup now."); _backupKey = (_backupKey.LibraryDb, _backupKey.JellyfinDb, await _backupService.CreateBackupAsync(new BackupOptionsDto() { Metadata = backupInstruction.Metadata, Subtitles = backupInstruction.Subtitles, Trickplay = backupInstruction.Trickplay, Database = false // database backups are explicitly handled by the provider itself as the backup service requires parity with the current model }).ConfigureAwait(false)); logger.LogInformation("Pre-Migration backup successfully created as {BackupKey}", _backupKey.FullBackup.Path); } } private static JellyfinMigrationBackupAttribute MergeBackupAttributes(JellyfinMigrationBackupAttribute left, JellyfinMigrationBackupAttribute right) { return new JellyfinMigrationBackupAttribute() { JellyfinDb = left!.JellyfinDb || right!.JellyfinDb, LegacyLibraryDb = left.LegacyLibraryDb || right!.LegacyLibraryDb, Metadata = left.Metadata || right!.Metadata, Subtitles = left.Subtitles || right!.Subtitles, Trickplay = left.Trickplay || right!.Trickplay }; } private class InternalCodeMigration : IInternalMigration { private readonly CodeMigration _codeMigration; private readonly IServiceProvider? _serviceProvider; private JellyfinDbContext _dbContext; public InternalCodeMigration(CodeMigration codeMigration, IServiceProvider? serviceProvider, JellyfinDbContext dbContext) { _codeMigration = codeMigration; _serviceProvider = serviceProvider; _dbContext = dbContext; } public async Task PerformAsync(IStartupLogger logger) { await _codeMigration.Perform(_serviceProvider, logger, CancellationToken.None).ConfigureAwait(false); var historyRepository = _dbContext.GetService(); var createScript = historyRepository.GetInsertScript(new HistoryRow(_codeMigration.BuildCodeMigrationId(), GetJellyfinVersion())); await _dbContext.Database.ExecuteSqlRawAsync(createScript).ConfigureAwait(false); } } private class InternalDatabaseMigration : IInternalMigration { private readonly JellyfinDbContext _jellyfinDbContext; private KeyValuePair _databaseMigrationInfo; public InternalDatabaseMigration(KeyValuePair databaseMigrationInfo, JellyfinDbContext jellyfinDbContext) { _databaseMigrationInfo = databaseMigrationInfo; _jellyfinDbContext = jellyfinDbContext; } public async Task PerformAsync(IStartupLogger logger) { var migrator = _jellyfinDbContext.GetService(); await migrator.MigrateAsync(_databaseMigrationInfo.Key).ConfigureAwait(false); } } }