/** * Comprehensive Board Migration System * * This migration handles all database structure changes from previous Wekan versions * to the current per-swimlane lists structure. It ensures: * * 1. All cards are visible with proper swimlaneId and listId * 2. Lists are per-swimlane (no shared lists across swimlanes) * 3. No empty lists are created * 4. Handles various database structure versions from git history * * Supported versions and their database structures: * - v7.94 and earlier: Shared lists across all swimlanes * - v8.00-v8.02: Transition period with mixed structures * - v8.03+: Per-swimlane lists structure */ import { Meteor } from 'meteor/meteor'; import { check } from 'meteor/check'; import { ReactiveCache } from '/imports/reactiveCache'; import Boards from '/models/boards'; import Lists from '/models/lists'; import Cards from '/models/cards'; import Swimlanes from '/models/swimlanes'; import Attachments from '/models/attachments'; import { generateUniversalAttachmentUrl, isUniversalFileUrl } from '/models/lib/universalUrlGenerator'; class ComprehensiveBoardMigration { constructor() { this.name = 'comprehensive-board-migration'; this.version = 1; this.migrationSteps = [ 'analyze_board_structure', 'fix_orphaned_cards', 'convert_shared_lists', 'ensure_per_swimlane_lists', 'cleanup_empty_lists', 'validate_migration' ]; } /** * Check if migration is needed for a board */ needsMigration(boardId) { try { const board = ReactiveCache.getBoard(boardId); if (!board) return false; // Check if board has already been processed if (board.comprehensiveMigrationCompleted) { return false; } // Check for various issues that need migration const issues = this.detectMigrationIssues(boardId); return issues.length > 0; } catch (error) { console.error('Error checking if migration is needed:', error); return false; } } /** * Detect all migration issues in a board */ detectMigrationIssues(boardId) { const issues = []; try { const cards = ReactiveCache.getCards({ boardId }); const lists = ReactiveCache.getLists({ boardId }); const swimlanes = ReactiveCache.getSwimlanes({ boardId }); // Issue 1: Cards with missing swimlaneId const cardsWithoutSwimlane = cards.filter(card => !card.swimlaneId); if (cardsWithoutSwimlane.length > 0) { issues.push({ type: 'cards_without_swimlane', count: cardsWithoutSwimlane.length, description: `${cardsWithoutSwimlane.length} cards missing swimlaneId` }); } // Issue 2: Cards with missing listId const cardsWithoutList = cards.filter(card => !card.listId); if (cardsWithoutList.length > 0) { issues.push({ type: 'cards_without_list', count: cardsWithoutList.length, description: `${cardsWithoutList.length} cards missing listId` }); } // Issue 3: Lists without swimlaneId (shared lists) const sharedLists = lists.filter(list => !list.swimlaneId || list.swimlaneId === ''); if (sharedLists.length > 0) { issues.push({ type: 'shared_lists', count: sharedLists.length, description: `${sharedLists.length} lists without swimlaneId (shared lists)` }); } // Issue 4: Cards with mismatched listId/swimlaneId const listSwimlaneMap = new Map(); lists.forEach(list => { listSwimlaneMap.set(list._id, list.swimlaneId || ''); }); const mismatchedCards = cards.filter(card => { if (!card.listId || !card.swimlaneId) return false; const listSwimlaneId = listSwimlaneMap.get(card.listId); return listSwimlaneId && listSwimlaneId !== card.swimlaneId; }); if (mismatchedCards.length > 0) { issues.push({ type: 'mismatched_cards', count: mismatchedCards.length, description: `${mismatchedCards.length} cards with mismatched listId/swimlaneId` }); } // Issue 5: Empty lists (lists with no cards) const emptyLists = lists.filter(list => { const listCards = cards.filter(card => card.listId === list._id); return listCards.length === 0; }); if (emptyLists.length > 0) { issues.push({ type: 'empty_lists', count: emptyLists.length, description: `${emptyLists.length} empty lists (no cards)` }); } } catch (error) { console.error('Error detecting migration issues:', error); issues.push({ type: 'detection_error', count: 1, description: `Error detecting issues: ${error.message}` }); } return issues; } /** * Execute the comprehensive migration for a board */ async executeMigration(boardId, progressCallback = null) { try { if (process.env.DEBUG === 'true') { console.log(`Starting comprehensive board migration for board ${boardId}`); } const board = ReactiveCache.getBoard(boardId); if (!board) { throw new Error(`Board ${boardId} not found`); } const results = { boardId, steps: {}, totalCardsProcessed: 0, totalListsProcessed: 0, totalListsCreated: 0, totalListsRemoved: 0, errors: [] }; const totalSteps = this.migrationSteps.length; let currentStep = 0; // Helper function to update progress const updateProgress = (stepName, stepProgress, stepStatus, stepDetails = null) => { currentStep++; const overallProgress = Math.round((currentStep / totalSteps) * 100); const progressData = { overallProgress, currentStep: currentStep, totalSteps, stepName, stepProgress, stepStatus, stepDetails, boardId }; if (progressCallback) { progressCallback(progressData); } if (process.env.DEBUG === 'true') { console.log(`Migration Progress: ${stepName} - ${stepStatus} (${stepProgress}%)`); } }; // Step 1: Analyze board structure updateProgress('analyze_board_structure', 0, 'Starting analysis...'); results.steps.analyze = await this.analyzeBoardStructure(boardId); updateProgress('analyze_board_structure', 100, 'Analysis complete', { issuesFound: results.steps.analyze.issueCount, needsMigration: results.steps.analyze.needsMigration }); // Step 2: Fix orphaned cards updateProgress('fix_orphaned_cards', 0, 'Fixing orphaned cards...'); results.steps.fixOrphanedCards = await this.fixOrphanedCards(boardId, (progress, status) => { updateProgress('fix_orphaned_cards', progress, status); }); results.totalCardsProcessed += results.steps.fixOrphanedCards.cardsFixed || 0; updateProgress('fix_orphaned_cards', 100, 'Orphaned cards fixed', { cardsFixed: results.steps.fixOrphanedCards.cardsFixed }); // Step 3: Convert shared lists to per-swimlane lists updateProgress('convert_shared_lists', 0, 'Converting shared lists...'); results.steps.convertSharedLists = await this.convertSharedListsToPerSwimlane(boardId, (progress, status) => { updateProgress('convert_shared_lists', progress, status); }); results.totalListsProcessed += results.steps.convertSharedLists.listsProcessed || 0; results.totalListsCreated += results.steps.convertSharedLists.listsCreated || 0; updateProgress('convert_shared_lists', 100, 'Shared lists converted', { listsProcessed: results.steps.convertSharedLists.listsProcessed, listsCreated: results.steps.convertSharedLists.listsCreated }); // Step 4: Ensure all lists are per-swimlane updateProgress('ensure_per_swimlane_lists', 0, 'Ensuring per-swimlane structure...'); results.steps.ensurePerSwimlane = await this.ensurePerSwimlaneLists(boardId); results.totalListsProcessed += results.steps.ensurePerSwimlane.listsProcessed || 0; updateProgress('ensure_per_swimlane_lists', 100, 'Per-swimlane structure ensured', { listsProcessed: results.steps.ensurePerSwimlane.listsProcessed }); // Step 5: Cleanup empty lists updateProgress('cleanup_empty_lists', 0, 'Cleaning up empty lists...'); results.steps.cleanupEmpty = await this.cleanupEmptyLists(boardId); results.totalListsRemoved += results.steps.cleanupEmpty.listsRemoved || 0; updateProgress('cleanup_empty_lists', 100, 'Empty lists cleaned up', { listsRemoved: results.steps.cleanupEmpty.listsRemoved }); // Step 6: Validate migration updateProgress('validate_migration', 0, 'Validating migration...'); results.steps.validate = await this.validateMigration(boardId); updateProgress('validate_migration', 100, 'Migration validated', { migrationSuccessful: results.steps.validate.migrationSuccessful, totalCards: results.steps.validate.totalCards, totalLists: results.steps.validate.totalLists }); // Step 7: Fix avatar URLs updateProgress('fix_avatar_urls', 0, 'Fixing avatar URLs...'); results.steps.fixAvatarUrls = await this.fixAvatarUrls(boardId); updateProgress('fix_avatar_urls', 100, 'Avatar URLs fixed', { avatarsFixed: results.steps.fixAvatarUrls.avatarsFixed }); // Step 8: Fix attachment URLs updateProgress('fix_attachment_urls', 0, 'Fixing attachment URLs...'); results.steps.fixAttachmentUrls = await this.fixAttachmentUrls(boardId); updateProgress('fix_attachment_urls', 100, 'Attachment URLs fixed', { attachmentsFixed: results.steps.fixAttachmentUrls.attachmentsFixed }); // Mark board as processed Boards.update(boardId, { $set: { comprehensiveMigrationCompleted: true, comprehensiveMigrationCompletedAt: new Date(), comprehensiveMigrationResults: results } }); if (process.env.DEBUG === 'true') { console.log(`Comprehensive board migration completed for board ${boardId}:`, results); } return { success: true, results }; } catch (error) { console.error(`Error executing comprehensive migration for board ${boardId}:`, error); throw error; } } /** * Step 1: Analyze board structure */ async analyzeBoardStructure(boardId) { const issues = this.detectMigrationIssues(boardId); return { issues, issueCount: issues.length, needsMigration: issues.length > 0 }; } /** * Step 2: Fix orphaned cards (cards with missing swimlaneId or listId) */ async fixOrphanedCards(boardId, progressCallback = null) { const cards = ReactiveCache.getCards({ boardId }); const swimlanes = ReactiveCache.getSwimlanes({ boardId }); const lists = ReactiveCache.getLists({ boardId }); let cardsFixed = 0; const defaultSwimlane = swimlanes.find(s => s.title === 'Default') || swimlanes[0]; const totalCards = cards.length; for (let i = 0; i < cards.length; i++) { const card = cards[i]; let needsUpdate = false; const updates = {}; // Fix missing swimlaneId if (!card.swimlaneId) { updates.swimlaneId = defaultSwimlane._id; needsUpdate = true; } // Fix missing listId if (!card.listId) { // Find or create a default list for this swimlane const swimlaneId = updates.swimlaneId || card.swimlaneId; let defaultList = lists.find(list => list.swimlaneId === swimlaneId && list.title === 'Default' ); if (!defaultList) { // Create a default list for this swimlane const newListId = Lists.insert({ title: 'Default', boardId: boardId, swimlaneId: swimlaneId, sort: 0, archived: false, createdAt: new Date(), modifiedAt: new Date(), type: 'list' }); defaultList = { _id: newListId }; } updates.listId = defaultList._id; needsUpdate = true; } if (needsUpdate) { Cards.update(card._id, { $set: { ...updates, modifiedAt: new Date() } }); cardsFixed++; } // Update progress if (progressCallback && (i % 10 === 0 || i === totalCards - 1)) { const progress = Math.round(((i + 1) / totalCards) * 100); progressCallback(progress, `Processing card ${i + 1} of ${totalCards}...`); } } return { cardsFixed }; } /** * Step 3: Convert shared lists to per-swimlane lists */ async convertSharedListsToPerSwimlane(boardId, progressCallback = null) { const cards = ReactiveCache.getCards({ boardId }); const lists = ReactiveCache.getLists({ boardId }); const swimlanes = ReactiveCache.getSwimlanes({ boardId }); let listsProcessed = 0; let listsCreated = 0; // Group cards by swimlaneId const cardsBySwimlane = new Map(); cards.forEach(card => { if (!cardsBySwimlane.has(card.swimlaneId)) { cardsBySwimlane.set(card.swimlaneId, []); } cardsBySwimlane.get(card.swimlaneId).push(card); }); const swimlaneEntries = Array.from(cardsBySwimlane.entries()); const totalSwimlanes = swimlaneEntries.length; // Process each swimlane for (let i = 0; i < swimlaneEntries.length; i++) { const [swimlaneId, swimlaneCards] = swimlaneEntries[i]; if (!swimlaneId) continue; if (progressCallback) { const progress = Math.round(((i + 1) / totalSwimlanes) * 100); progressCallback(progress, `Processing swimlane ${i + 1} of ${totalSwimlanes}...`); } // Get existing lists for this swimlane const existingLists = lists.filter(list => list.swimlaneId === swimlaneId); const existingListTitles = new Set(existingLists.map(list => list.title)); // Group cards by their current listId const cardsByListId = new Map(); swimlaneCards.forEach(card => { if (!cardsByListId.has(card.listId)) { cardsByListId.set(card.listId, []); } cardsByListId.get(card.listId).push(card); }); // For each listId used by cards in this swimlane for (const [listId, cardsInList] of cardsByListId) { const originalList = lists.find(l => l._id === listId); if (!originalList) continue; // Check if this list's swimlaneId matches the card's swimlaneId if (originalList.swimlaneId === swimlaneId) { // List is already correctly assigned to this swimlane listsProcessed++; continue; } // Check if we already have a list with the same title in this swimlane let targetList = existingLists.find(list => list.title === originalList.title); if (!targetList) { // Create a new list for this swimlane const newListData = { title: originalList.title, boardId: boardId, swimlaneId: swimlaneId, sort: originalList.sort || 0, archived: originalList.archived || false, createdAt: new Date(), modifiedAt: new Date(), type: originalList.type || 'list' }; // Copy other properties if they exist if (originalList.color) newListData.color = originalList.color; if (originalList.wipLimit) newListData.wipLimit = originalList.wipLimit; if (originalList.wipLimitEnabled) newListData.wipLimitEnabled = originalList.wipLimitEnabled; if (originalList.wipLimitSoft) newListData.wipLimitSoft = originalList.wipLimitSoft; if (originalList.starred) newListData.starred = originalList.starred; if (originalList.collapsed) newListData.collapsed = originalList.collapsed; // Insert the new list const newListId = Lists.insert(newListData); targetList = { _id: newListId, ...newListData }; listsCreated++; } // Update all cards in this group to use the correct listId for (const card of cardsInList) { Cards.update(card._id, { $set: { listId: targetList._id, modifiedAt: new Date() } }); } listsProcessed++; } } return { listsProcessed, listsCreated }; } /** * Step 4: Ensure all lists are per-swimlane */ async ensurePerSwimlaneLists(boardId) { const lists = ReactiveCache.getLists({ boardId }); const swimlanes = ReactiveCache.getSwimlanes({ boardId }); const defaultSwimlane = swimlanes.find(s => s.title === 'Default') || swimlanes[0]; let listsProcessed = 0; for (const list of lists) { if (!list.swimlaneId || list.swimlaneId === '') { // Assign to default swimlane Lists.update(list._id, { $set: { swimlaneId: defaultSwimlane._id, modifiedAt: new Date() } }); listsProcessed++; } } return { listsProcessed }; } /** * Step 5: Cleanup empty lists (lists with no cards) */ async cleanupEmptyLists(boardId) { const lists = ReactiveCache.getLists({ boardId }); const cards = ReactiveCache.getCards({ boardId }); let listsRemoved = 0; for (const list of lists) { const listCards = cards.filter(card => card.listId === list._id); if (listCards.length === 0) { // Remove empty list Lists.remove(list._id); listsRemoved++; if (process.env.DEBUG === 'true') { console.log(`Removed empty list: ${list.title} (${list._id})`); } } } return { listsRemoved }; } /** * Step 6: Validate migration */ async validateMigration(boardId) { const issues = this.detectMigrationIssues(boardId); const cards = ReactiveCache.getCards({ boardId }); const lists = ReactiveCache.getLists({ boardId }); // Check that all cards have valid swimlaneId and listId const validCards = cards.filter(card => card.swimlaneId && card.listId); const invalidCards = cards.length - validCards.length; // Check that all lists have swimlaneId const validLists = lists.filter(list => list.swimlaneId && list.swimlaneId !== ''); const invalidLists = lists.length - validLists.length; return { issuesRemaining: issues.length, totalCards: cards.length, validCards, invalidCards, totalLists: lists.length, validLists, invalidLists, migrationSuccessful: issues.length === 0 && invalidCards === 0 && invalidLists === 0 }; } /** * Step 7: Fix avatar URLs (remove problematic auth parameters and fix URL formats) */ async fixAvatarUrls(boardId) { const users = ReactiveCache.getUsers({}); let avatarsFixed = 0; for (const user of users) { if (user.profile && user.profile.avatarUrl) { const avatarUrl = user.profile.avatarUrl; let needsUpdate = false; let cleanUrl = avatarUrl; // Check if URL has problematic parameters if (avatarUrl.includes('auth=false') || avatarUrl.includes('brokenIsFine=true')) { // Remove problematic parameters cleanUrl = cleanUrl.replace(/[?&]auth=false/g, ''); cleanUrl = cleanUrl.replace(/[?&]brokenIsFine=true/g, ''); cleanUrl = cleanUrl.replace(/\?&/g, '?'); cleanUrl = cleanUrl.replace(/\?$/g, ''); needsUpdate = true; } // Check if URL is using old CollectionFS format if (avatarUrl.includes('/cfs/files/avatars/')) { cleanUrl = cleanUrl.replace('/cfs/files/avatars/', '/cdn/storage/avatars/'); needsUpdate = true; } // Check if URL is missing the /cdn/storage/avatars/ prefix if (avatarUrl.includes('avatars/') && !avatarUrl.includes('/cdn/storage/avatars/') && !avatarUrl.includes('/cfs/files/avatars/')) { // This might be a relative URL, make it absolute if (!avatarUrl.startsWith('http') && !avatarUrl.startsWith('/')) { cleanUrl = `/cdn/storage/avatars/${avatarUrl}`; needsUpdate = true; } } if (needsUpdate) { // Update user's avatar URL Users.update(user._id, { $set: { 'profile.avatarUrl': cleanUrl, modifiedAt: new Date() } }); avatarsFixed++; } } } return { avatarsFixed }; } /** * Step 8: Fix attachment URLs (remove problematic auth parameters and fix URL formats) */ async fixAttachmentUrls(boardId) { const attachments = ReactiveCache.getAttachments({}); let attachmentsFixed = 0; for (const attachment of attachments) { // Check if attachment has URL field that needs fixing if (attachment.url) { const attachmentUrl = attachment.url; let needsUpdate = false; let cleanUrl = attachmentUrl; // Check if URL has problematic parameters if (attachmentUrl.includes('auth=false') || attachmentUrl.includes('brokenIsFine=true')) { // Remove problematic parameters cleanUrl = cleanUrl.replace(/[?&]auth=false/g, ''); cleanUrl = cleanUrl.replace(/[?&]brokenIsFine=true/g, ''); cleanUrl = cleanUrl.replace(/\?&/g, '?'); cleanUrl = cleanUrl.replace(/\?$/g, ''); needsUpdate = true; } // Check if URL is using old CollectionFS format if (attachmentUrl.includes('/cfs/files/attachments/')) { cleanUrl = cleanUrl.replace('/cfs/files/attachments/', '/cdn/storage/attachments/'); needsUpdate = true; } // Check if URL has /original/ path that should be removed if (attachmentUrl.includes('/original/')) { cleanUrl = cleanUrl.replace(/\/original\/[^\/\?#]+/, ''); needsUpdate = true; } // If we have a file ID, generate a universal URL const fileId = attachment._id; if (fileId && !isUniversalFileUrl(cleanUrl, 'attachment')) { cleanUrl = generateUniversalAttachmentUrl(fileId); needsUpdate = true; } if (needsUpdate) { // Update attachment URL Attachments.update(attachment._id, { $set: { url: cleanUrl, modifiedAt: new Date() } }); attachmentsFixed++; } } } return { attachmentsFixed }; } /** * Get migration status for a board */ getMigrationStatus(boardId) { try { const board = ReactiveCache.getBoard(boardId); if (!board) { return { status: 'board_not_found' }; } if (board.comprehensiveMigrationCompleted) { return { status: 'completed', completedAt: board.comprehensiveMigrationCompletedAt, results: board.comprehensiveMigrationResults }; } const needsMigration = this.needsMigration(boardId); const issues = this.detectMigrationIssues(boardId); return { status: needsMigration ? 'needed' : 'not_needed', issues, issueCount: issues.length }; } catch (error) { console.error('Error getting migration status:', error); return { status: 'error', error: error.message }; } } } // Export singleton instance export const comprehensiveBoardMigration = new ComprehensiveBoardMigration(); // Meteor methods Meteor.methods({ 'comprehensiveBoardMigration.check'(boardId) { check(boardId, String); if (!this.userId) { throw new Meteor.Error('not-authorized'); } return comprehensiveBoardMigration.getMigrationStatus(boardId); }, 'comprehensiveBoardMigration.execute'(boardId) { check(boardId, String); if (!this.userId) { throw new Meteor.Error('not-authorized'); } return comprehensiveBoardMigration.executeMigration(boardId); }, 'comprehensiveBoardMigration.needsMigration'(boardId) { check(boardId, String); if (!this.userId) { throw new Meteor.Error('not-authorized'); } return comprehensiveBoardMigration.needsMigration(boardId); }, 'comprehensiveBoardMigration.detectIssues'(boardId) { check(boardId, String); if (!this.userId) { throw new Meteor.Error('not-authorized'); } return comprehensiveBoardMigration.detectMigrationIssues(boardId); }, 'comprehensiveBoardMigration.fixAvatarUrls'(boardId) { check(boardId, String); if (!this.userId) { throw new Meteor.Error('not-authorized'); } return comprehensiveBoardMigration.fixAvatarUrls(boardId); } });