payload-wordpress-migrator 0.0.22

package/dist/utils/migration/orchestrator.js

@@ -0,0 +1,756 @@
+import { createSafeLexicalRoot, cleanEmptyTextNodes } from '../lexical/nodeFactories.js';
+import { checkMigrationFields } from '../media/validation.js';
+import { uploadMediaToPayload } from '../media/upload.js';
+import { MIGRATION_COLLECTION, isMediaUploadResult } from '../types.js';
+import { WordPressClient } from '../wordpress/client.js';
+import { invalidateMigrationCache } from '../helpers/cache.js';
+import { getContentTypeEndpoint } from '../helpers/wpHelpers.js';
+import { createConcurrencyLimiter } from '../helpers/concurrency.js';
+import { transformWordPressContent } from '../content/transformer.js';
+import { applyFieldMapping } from '../content/fieldMapping.js';
+
+const logBuffer = [];
+const addJobLog = (level, message)=>{
+    logBuffer.push({
+        level,
+        message,
+        timestamp: new Date()
+    });
+};
+const flushJobLogs = async (payload, jobId)=>{
+    if (logBuffer.length === 0) return;
+    try {
+        const currentJob = await payload.findByID({
+            id: jobId,
+            collection: MIGRATION_COLLECTION
+        });
+        await payload.update({
+            id: jobId,
+            collection: MIGRATION_COLLECTION,
+            data: {
+                logs: [
+                    ...currentJob?.logs || [],
+                    ...logBuffer
+                ]
+            }
+        });
+        logBuffer.length = 0;
+    } catch (error) {
+        payload.logger.error(`Failed to flush ${logBuffer.length} logs for job ${jobId}: ${error}`);
+    }
+};
+const processItem = async (item, job, payload, pluginOptions, client, action)=>{
+    try {
+        const hasMigrationFields = await checkMigrationFields(payload, job.targetCollection);
+        // Dedup check
+        if (hasMigrationFields) {
+            try {
+                const existingItem = await payload.find({
+                    collection: job.targetCollection,
+                    limit: 1,
+                    where: {
+                        'migratedFromWordPress.wpPostId': {
+                            equals: item.id
+                        }
+                    }
+                });
+                if (existingItem.docs.length > 0) {
+                    const existingDoc = existingItem.docs[0];
+                    // Update mode: fill only empty fields on existing items, preserving any manual
+                    // edits the user has made post-migration. Only meta (title, description, image),
+                    // categories, and tags are selectively updated — content is never overwritten.
+                    if (action === 'update') {
+                        const payloadData = await transformWordPressContent(item, job.contentType, job.configuration, payload, pluginOptions, {
+                            apiUrl: client.wpApiUrl,
+                            headers: client.wpHeaders
+                        });
+                        const mappedData = await applyFieldMapping(payloadData, item, job.configuration?.fieldMapping, payload);
+                        const updateData = {};
+                        let hasUpdates = false;
+                        if (!existingDoc.meta?.title && mappedData.meta?.title) {
+                            updateData.meta = updateData.meta || {};
+                            updateData.meta.title = mappedData.meta.title;
+                            hasUpdates = true;
+                        }
+                        if (!existingDoc.meta?.description && mappedData.meta?.description) {
+                            updateData.meta = updateData.meta || {};
+                            updateData.meta.description = mappedData.meta.description;
+                            hasUpdates = true;
+                        }
+                        if (!existingDoc.meta?.image) {
+                            const metaImageValue = existingDoc.heroImage || mappedData.meta?.image;
+                            if (metaImageValue && typeof metaImageValue !== 'string') {
+                                updateData.meta = updateData.meta || {};
+                                updateData.meta.image = metaImageValue;
+                                hasUpdates = true;
+                            }
+                        }
+                        if (!existingDoc.categories?.length && mappedData.categories?.length) {
+                            updateData.categories = mappedData.categories;
+                            hasUpdates = true;
+                        }
+                        if (!existingDoc.tags?.length && mappedData.tags?.length) {
+                            updateData.tags = mappedData.tags;
+                            hasUpdates = true;
+                        }
+                        if (hasUpdates) {
+                            try {
+                                await payload.update({
+                                    id: existingDoc.id,
+                                    collection: job.targetCollection,
+                                    data: updateData
+                                });
+                            } catch (updateError) {
+                                payload.logger.error(`Failed to update existing post ${item.id}: ${updateError}`);
+                                return {
+                                    incrementProcessed: true,
+                                    status: 'failed',
+                                    wpId: item.id,
+                                    error: `Update failed: ${updateError}`,
+                                    failedRecord: {
+                                        error: `Update failed: ${updateError}`,
+                                        errorType: 'UpdateError',
+                                        stage: 'update',
+                                        timestamp: new Date(),
+                                        wpId: item.id
+                                    }
+                                };
+                            }
+                        }
+                        return {
+                            incrementProcessed: true,
+                            status: 'success',
+                            wpId: item.id
+                        };
+                    } else {
+                        // Already exists, skip
+                        return {
+                            incrementProcessed: true,
+                            status: 'skipped',
+                            wpId: item.id
+                        };
+                    }
+                }
+            } catch (queryError) {
+                payload.logger.warn(`Error checking for existing items in collection ${job.targetCollection}: ${queryError}`);
+                addJobLog('warning', `Error checking for existing items: ${queryError instanceof Error ? queryError.message : String(queryError)}`);
+            }
+        }
+        // Transform
+        const payloadData = await transformWordPressContent(item, job.contentType, job.configuration, payload, pluginOptions, {
+            apiUrl: client.wpApiUrl,
+            headers: client.wpHeaders
+        });
+        // Media upload path
+        if (isMediaUploadResult(payloadData)) {
+            if (job.dryRun) {
+                addJobLog('info', `[DRY RUN] Would upload media: "${payloadData.mediaMetadata?.title || `Item ${item.id}`}" (wpId: ${item.id})`);
+                return {
+                    incrementProcessed: true,
+                    status: 'success',
+                    wpId: item.id
+                };
+            }
+            try {
+                const hasMigFields = await checkMigrationFields(payload, job.targetCollection);
+                await uploadMediaToPayload(payload, job.targetCollection, payloadData.fileData, payloadData.mediaMetadata, hasMigFields);
+                return {
+                    incrementProcessed: true,
+                    status: 'success',
+                    wpId: item.id
+                };
+            } catch (uploadError) {
+                const errorMessage = uploadError instanceof Error ? uploadError.message : 'Unknown error';
+                payload.logger.error(`Failed to upload media file for item ${item.id}: ${uploadError}`);
+                return {
+                    error: `Media upload failed: ${errorMessage}`,
+                    failedRecord: {
+                        error: `Media upload failed: ${errorMessage}`,
+                        errorType: uploadError?.constructor?.name || 'Error',
+                        stage: 'media_upload',
+                        timestamp: new Date(),
+                        wpId: item.id
+                    },
+                    incrementProcessed: true,
+                    status: 'failed',
+                    wpId: item.id
+                };
+            }
+        } else if (job.contentType === 'media' && payloadData.migrationNote) {
+            return {
+                incrementProcessed: true,
+                status: 'skipped_media_note',
+                wpId: item.id
+            };
+        }
+        // Regular content creation path
+        const hasMigFields = await checkMigrationFields(payload, job.targetCollection);
+        const mappedData = await applyFieldMapping(payloadData, item, job.configuration?.fieldMapping, payload);
+        const createData = {
+            ...mappedData
+        };
+        if (hasMigFields) {
+            createData.migratedFromWordPress = {
+                migrationDate: new Date(),
+                wpPostId: item.id,
+                wpPostType: job.contentType
+            };
+        }
+        try {
+            // Content validation and cleanup
+            if (createData.content !== undefined) {
+                if (typeof createData.content === 'string') {
+                    if (createData.content.trim().length === 0) {
+                        payload.logger.warn(`Pre-validation: Empty content string for item ${item.id}, creating safe Lexical structure`);
+                        createData.content = createSafeLexicalRoot();
+                    } else {
+                        payload.logger.warn(`Pre-validation: Plain text content for item ${item.id}, converting to Lexical`);
+                        createData.content = {
+                            type: 'root',
+                            children: [
+                                {
+                                    type: 'paragraph',
+                                    children: [
+                                        {
+                                            type: 'text',
+                                            detail: 0,
+                                            format: 0,
+                                            mode: 'normal',
+                                            style: '',
+                                            text: createData.content,
+                                            version: 1
+                                        }
+                                    ],
+                                    direction: 'ltr',
+                                    format: '',
+                                    indent: 0,
+                                    version: 1
+                                }
+                            ],
+                            direction: 'ltr',
+                            format: '',
+                            indent: 0,
+                            version: 1
+                        };
+                    }
+                } else if (typeof createData.content === 'object') {
+                    const contentStr = JSON.stringify(createData.content);
+                    if (contentStr.includes('"text":""')) {
+                        payload.logger.warn(`Pre-validation: Cleaning empty text nodes for item ${item.id}`);
+                        if (createData.content.root) {
+                            createData.content = cleanEmptyTextNodes(createData.content);
+                        } else {
+                            const wrapped = {
+                                root: createData.content
+                            };
+                            const cleaned = cleanEmptyTextNodes(wrapped);
+                            createData.content = cleaned.root || cleaned;
+                        }
+                    }
+                    if (!createData.content.root || !createData.content.root.type || createData.content.root.type !== 'root' || !createData.content.root.children) {
+                        payload.logger.warn(`Pre-validation: Invalid content structure for item ${item.id}, using safe fallback`);
+                        createData.content = createSafeLexicalRoot();
+                    } else {
+                        let hasNonWhitespaceContent = false;
+                        const cleanedChildren = createData.content.root.children.filter((child)=>{
+                            if (child.type === 'paragraph' && child.children) {
+                                const hasText = child.children.some((textNode)=>{
+                                    return textNode.type === 'text' && textNode.text && textNode.text.trim().length > 0;
+                                });
+                                if (hasText) {
+                                    hasNonWhitespaceContent = true;
+                                    return true;
+                                } else {
+                                    return false;
+                                }
+                            } else {
+                                hasNonWhitespaceContent = true;
+                                return true;
+                            }
+                        });
+                        createData.content.root.children = cleanedChildren;
+                        if (!hasNonWhitespaceContent || cleanedChildren.length === 0) {
+                            payload.logger.warn(`Pre-validation: Content only contains whitespace for item ${item.id}, adding placeholder`);
+                            createData.content = createSafeLexicalRoot();
+                        }
+                    }
+                }
+            } else {
+                payload.logger.warn(`Pre-validation: No content field for item ${item.id}, creating safe default`);
+                createData.content = createSafeLexicalRoot();
+            }
+            if (createData.content && createData.content.root && createData.content.root.type === 'root' && createData.content.root.children) {
+                createData.content.root.children = createData.content.root.children.map((child)=>{
+                    if (child.type === 'paragraph' && child.children) {
+                        const validNodes = child.children.map((node)=>{
+                            if (node.type === 'text') {
+                                return node.text && node.text.trim().length > 0 ? node : null;
+                            }
+                            if (node.type === 'link') {
+                                if (!node.url || node.url.trim().length === 0) {
+                                    payload.logger.warn(`Converting link with empty URL to text for item ${item.id}`);
+                                    return {
+                                        type: 'text',
+                                        detail: 0,
+                                        format: 0,
+                                        mode: 'normal',
+                                        style: '',
+                                        text: node.children && node.children[0] && node.children[0].text ? node.children[0].text : 'link',
+                                        version: 1
+                                    };
+                                }
+                                return {
+                                    ...node,
+                                    rel: node.rel || '',
+                                    target: node.target || '',
+                                    title: node.title || '',
+                                    url: node.url.trim()
+                                };
+                            }
+                            return node;
+                        }).filter(Boolean);
+                        if (validNodes.length === 0) {
+                            validNodes.push({
+                                type: 'text',
+                                detail: 0,
+                                format: 0,
+                                mode: 'normal',
+                                style: '',
+                                text: '(Empty paragraph)',
+                                version: 1
+                            });
+                        }
+                        return {
+                            ...child,
+                            children: validNodes
+                        };
+                    }
+                    return child;
+                });
+            }
+            if (job.dryRun) {
+                const title = createData.title || createData.name || `Item ${item.id}`;
+                addJobLog('info', `[DRY RUN] Would create ${job.contentType}: "${title}" (wpId: ${item.id}, fields: ${Object.keys(createData).length})`);
+                return {
+                    incrementProcessed: true,
+                    status: 'success',
+                    wpId: item.id
+                };
+            }
+            await payload.create({
+                collection: job.targetCollection,
+                data: createData
+            });
+            return {
+                incrementProcessed: true,
+                status: 'success',
+                wpId: item.id
+            };
+        } catch (createError) {
+            const errorMessage = createError instanceof Error ? createError.message : String(createError);
+            if (errorMessage.includes('Content') || errorMessage.includes('ValidationError')) {
+                payload.logger.error(`Content validation error for item ${item.id}: ${errorMessage} | title="${item.title?.rendered || 'N/A'}" contentLength=${item.content?.rendered?.length || 0}`);
+            }
+            throw createError;
+        }
+    } catch (itemError) {
+        let errorMessage = itemError instanceof Error ? itemError.message : String(itemError);
+        if (itemError instanceof Error && 'data' in itemError) {
+            const errorData = itemError.data;
+            payload.logger.error(`Detailed error data for item ${item.id}: ${JSON.stringify(errorData, null, 2)}`);
+            if (errorData && errorData.errors) {
+                payload.logger.error(`Field validation errors: ${JSON.stringify(errorData.errors, null, 2)}`);
+                if (Array.isArray(errorData.errors)) {
+                    const fieldErrors = errorData.errors.map((e)=>`${e.field || e.path || 'unknown'}: ${e.message}`).join(', ');
+                    errorMessage += ` [Field errors: ${fieldErrors}]`;
+                }
+            }
+        }
+        const itemTitle = item.title?.rendered || item.slug || `Item ${item.id}`;
+        const itemType = job.contentType || 'content';
+        const shortError = errorMessage.length > 100 ? errorMessage.substring(0, 100) + '...' : errorMessage;
+        addJobLog('error', `Failed to migrate ${itemType} "${itemTitle}" (ID: ${item.id}) - ${shortError}`);
+        payload.logger.error(`MIGRATION FAILURE: ${itemType} "${itemTitle}" (ID: ${item.id}) - ${shortError}`);
+        payload.logger.error(`Failed to migrate item ${item.id}: ${itemError}`);
+        return {
+            error: errorMessage,
+            failedRecord: {
+                error: errorMessage,
+                errorType: itemError?.constructor?.name || 'Error',
+                stage: 'create',
+                timestamp: new Date(),
+                wpId: item.id
+            },
+            incrementProcessed: true,
+            status: 'failed',
+            wpId: item.id
+        };
+    }
+};
+const processBatch = async (batch, job, payload, pluginOptions, client, counters, action)=>{
+    const concurrency = job.configuration?.concurrency || pluginOptions.migrationConcurrency || 1;
+    const limit = createConcurrencyLimiter(concurrency);
+    const batchResults = await Promise.allSettled(batch.map((item)=>limit(()=>processItem(item, job, payload, pluginOptions, client, action))));
+    for (const result of batchResults){
+        if (result.status === 'fulfilled') {
+            const r = result.value;
+            if (r.incrementProcessed) {
+                counters.processedItems++;
+            }
+            if (r.status === 'success' || r.status === 'skipped') {
+                counters.successfulItems++;
+            } else if (r.status === 'failed') {
+                counters.failedItems++;
+                if (r.failedRecord) {
+                    counters.failedItemIds.push(r.failedRecord);
+                }
+            }
+        // skipped_media_note: only increments processedItems (already done above)
+        } else {
+            // Should not happen since processItem catches internally
+            counters.processedItems++;
+            counters.failedItems++;
+        }
+    }
+};
+// ---------------------------------------------------------------------------
+// Main orchestrator
+// ---------------------------------------------------------------------------
+const processMigrationJob = async (jobId, job, payload, pluginOptions, siteConfig, action)=>{
+    try {
+        const client = new WordPressClient({
+            wpPassword: siteConfig.wpPassword,
+            wpSiteUrl: siteConfig.wpSiteUrl,
+            wpUsername: siteConfig.wpUsername
+        });
+        const batchSize = job.configuration?.batchSize || pluginOptions.migrationBatchSize || 10;
+        const batchDelay = job.configuration?.batchDelay ?? 200;
+        const requestDelay = job.configuration?.requestDelay || pluginOptions.wpRequestDelay || 0;
+        addJobLog('info', `Starting migration for ${job.contentType} from ${siteConfig.siteName || 'WordPress Site'}`);
+        invalidateMigrationCache();
+        const endpoint = getContentTypeEndpoint(client.wpApiUrl, job.contentType);
+        const rawIncludeIds = job.includeIds || job.configuration?.includeIds;
+        let includeIds;
+        if (rawIncludeIds) {
+            if (typeof rawIncludeIds === 'string') {
+                includeIds = rawIncludeIds.split(',').map((id)=>parseInt(id.trim(), 10)).filter((id)=>!isNaN(id) && id > 0);
+            } else if (Array.isArray(rawIncludeIds)) {
+                includeIds = rawIncludeIds.map((id)=>typeof id === 'number' ? id : parseInt(String(id), 10)).filter((id)=>!isNaN(id) && id > 0);
+            }
+        }
+        addJobLog('info', `Connecting to WordPress API at ${client.wpApiUrl}...`);
+        await flushJobLogs(payload, jobId);
+        // Retry and resume need the full item array for filtering; fresh start / restart / update can stream
+        const needsFullFetch = action === 'retry' || action === 'resume';
+        const counters = {
+            failedItemIds: [],
+            failedItems: 0,
+            processedItems: 0,
+            successfulItems: 0
+        };
+        let totalItems = 0;
+        if (needsFullFetch) {
+            // -----------------------------------------------------------------------
+            // RETRY / RESUME path — fetch all items, then filter and batch-process
+            // -----------------------------------------------------------------------
+            const wpContent = await client.fetchAllPages(endpoint, async (page, totalPages, itemCount)=>{
+                addJobLog('info', `Fetched page ${page} of ${totalPages} (${itemCount} items)`);
+            }, job.contentType, includeIds);
+            await flushJobLogs(payload, jobId);
+            totalItems = wpContent.length;
+            await payload.update({
+                id: jobId,
+                collection: MIGRATION_COLLECTION,
+                data: {
+                    progress: {
+                        failedItems: 0,
+                        processedItems: 0,
+                        successfulItems: 0,
+                        totalItems
+                    }
+                }
+            });
+            invalidateMigrationCache();
+            const currentJob = await payload.findByID({
+                id: jobId,
+                collection: MIGRATION_COLLECTION
+            });
+            counters.processedItems = currentJob.progress?.processedItems || 0;
+            counters.successfulItems = currentJob.progress?.successfulItems || 0;
+            counters.failedItems = currentJob.progress?.failedItems || 0;
+            counters.failedItemIds = currentJob.progress?.failedItemIds || [];
+            let itemsToProcess = wpContent;
+            if (action === 'retry' && currentJob.progress?.failedItemIds && currentJob.progress.failedItemIds.length > 0) {
+                const failedIds = currentJob.progress.failedItemIds.map((item)=>item.wpId);
+                itemsToProcess = wpContent.filter((item)=>failedIds.includes(item.id));
+                addJobLog('info', `Retrying ${itemsToProcess.length} previously failed items`);
+            // Resume deduplication: build a Set of WordPress post IDs that already exist in the
+            // target collection by paginating through all migrated items. We then filter the
+            // fetched WordPress content to skip items that were already successfully imported,
+            // so the job continues from where it left off without creating duplicates.
+            } else if (action === 'resume' && counters.processedItems > 0 && counters.processedItems < totalItems) {
+                const processedIds = new Set();
+                try {
+                    const hasMigrationFields = await checkMigrationFields(payload, currentJob.targetCollection);
+                    if (hasMigrationFields) {
+                        let page = 1;
+                        let hasMore = true;
+                        while(hasMore){
+                            const existingItems = await payload.find({
+                                collection: currentJob.targetCollection,
+                                limit: 100,
+                                page,
+                                where: {
+                                    'migratedFromWordPress.wpPostType': {
+                                        equals: currentJob.contentType
+                                    }
+                                }
+                            });
+                            existingItems.docs.forEach((doc)=>{
+                                if (doc.migratedFromWordPress?.wpPostId) {
+                                    processedIds.add(doc.migratedFromWordPress.wpPostId);
+                                }
+                            });
+                            hasMore = page < existingItems.totalPages;
+                            page++;
+                        }
+                    }
+                } catch (error) {
+                    payload.logger.warn('Could not determine processed items for resume, continuing from beginning');
+                    addJobLog('warning', 'Could not determine processed items for resume, continuing from beginning');
+                }
+                itemsToProcess = wpContent.filter((item)=>!processedIds.has(item.id));
+                addJobLog('info', `Resuming migration from ${counters.processedItems}/${totalItems} items (${itemsToProcess.length} remaining)`);
+                await flushJobLogs(payload, jobId);
+            }
+            // Batch-process the filtered items
+            for(let i = 0; i < itemsToProcess.length; i += batchSize){
+                // Pause check
+                const jobState = await payload.findByID({
+                    id: jobId,
+                    collection: MIGRATION_COLLECTION
+                });
+                if (jobState.status === 'paused') {
+                    addJobLog('info', `Migration paused at ${counters.processedItems}/${totalItems} items processed`);
+                    await flushJobLogs(payload, jobId);
+                    await payload.update({
+                        id: jobId,
+                        collection: MIGRATION_COLLECTION,
+                        data: {
+                            progress: {
+                                failedItemIds: counters.failedItemIds,
+                                failedItems: counters.failedItems,
+                                processedItems: counters.processedItems,
+                                successfulItems: counters.successfulItems,
+                                totalItems
+                            }
+                        }
+                    });
+                    invalidateMigrationCache();
+                    return;
+                }
+                const batch = itemsToProcess.slice(i, i + batchSize);
+                await processBatch(batch, job, payload, pluginOptions, client, counters, action);
+                // Progress update after each batch
+                await payload.update({
+                    id: jobId,
+                    collection: MIGRATION_COLLECTION,
+                    data: {
+                        progress: {
+                            failedItemIds: counters.failedItemIds,
+                            failedItems: counters.failedItems,
+                            processedItems: counters.processedItems,
+                            successfulItems: counters.successfulItems,
+                            totalItems
+                        }
+                    }
+                });
+                await flushJobLogs(payload, jobId);
+                if (batchDelay > 0) {
+                    await new Promise((resolve)=>setTimeout(resolve, batchDelay));
+                }
+            }
+        } else {
+            // -----------------------------------------------------------------------
+            // FRESH START / RESTART / UPDATE path — stream pages, process in batches
+            // -----------------------------------------------------------------------
+            if (action === 'restart') {
+                counters.processedItems = 0;
+                counters.successfulItems = 0;
+                counters.failedItems = 0;
+            }
+            let totalItemsSet = false;
+            let batchBuffer = [];
+            for await (const { items, totalItems: headerTotal } of client.fetchPages(endpoint, {
+                contentType: job.contentType,
+                includeIds,
+                onProgress: (page, totalPages, count)=>{
+                    addJobLog('info', `Fetched page ${page} of ${totalPages} (${count} items)`);
+                },
+                requestDelay
+            })){
+                // Set totalItems from X-WP-Total header on first page
+                if (!totalItemsSet) {
+                    totalItems = headerTotal || items.length;
+                    totalItemsSet = true;
+                    await payload.update({
+                        id: jobId,
+                        collection: MIGRATION_COLLECTION,
+                        data: {
+                            progress: {
+                                failedItems: 0,
+                                processedItems: 0,
+                                successfulItems: 0,
+                                totalItems
+                            }
+                        }
+                    });
+                    invalidateMigrationCache();
+                    await flushJobLogs(payload, jobId);
+                }
+                // Accumulate items into batch buffer
+                for (const item of items){
+                    batchBuffer.push(item);
+                    if (batchBuffer.length >= batchSize) {
+                        // Pause check before each batch
+                        const jobState = await payload.findByID({
+                            id: jobId,
+                            collection: MIGRATION_COLLECTION
+                        });
+                        if (jobState.status === 'paused') {
+                            addJobLog('info', `Migration paused at ${counters.processedItems}/${totalItems} items processed`);
+                            await flushJobLogs(payload, jobId);
+                            await payload.update({
+                                id: jobId,
+                                collection: MIGRATION_COLLECTION,
+                                data: {
+                                    progress: {
+                                        failedItemIds: counters.failedItemIds,
+                                        failedItems: counters.failedItems,
+                                        processedItems: counters.processedItems,
+                                        successfulItems: counters.successfulItems,
+                                        totalItems
+                                    }
+                                }
+                            });
+                            invalidateMigrationCache();
+                            return;
+                        }
+                        await processBatch(batchBuffer, job, payload, pluginOptions, client, counters, action);
+                        // Progress update after each batch
+                        await payload.update({
+                            id: jobId,
+                            collection: MIGRATION_COLLECTION,
+                            data: {
+                                progress: {
+                                    failedItemIds: counters.failedItemIds,
+                                    failedItems: counters.failedItems,
+                                    processedItems: counters.processedItems,
+                                    successfulItems: counters.successfulItems,
+                                    totalItems
+                                }
+                            }
+                        });
+                        await flushJobLogs(payload, jobId);
+                        if (batchDelay > 0) {
+                            await new Promise((resolve)=>setTimeout(resolve, batchDelay));
+                        }
+                        batchBuffer = [];
+                    }
+                }
+            }
+            // Process remaining items in buffer
+            if (batchBuffer.length > 0) {
+                const jobState = await payload.findByID({
+                    id: jobId,
+                    collection: MIGRATION_COLLECTION
+                });
+                if (jobState.status === 'paused') {
+                    addJobLog('info', `Migration paused at ${counters.processedItems}/${totalItems} items processed`);
+                    await flushJobLogs(payload, jobId);
+                    await payload.update({
+                        id: jobId,
+                        collection: MIGRATION_COLLECTION,
+                        data: {
+                            progress: {
+                                failedItemIds: counters.failedItemIds,
+                                failedItems: counters.failedItems,
+                                processedItems: counters.processedItems,
+                                successfulItems: counters.successfulItems,
+                                totalItems
+                            }
+                        }
+                    });
+                    invalidateMigrationCache();
+                    return;
+                }
+                await processBatch(batchBuffer, job, payload, pluginOptions, client, counters, action);
+                await payload.update({
+                    id: jobId,
+                    collection: MIGRATION_COLLECTION,
+                    data: {
+                        progress: {
+                            failedItemIds: counters.failedItemIds,
+                            failedItems: counters.failedItems,
+                            processedItems: counters.processedItems,
+                            successfulItems: counters.successfulItems,
+                            totalItems
+                        }
+                    }
+                });
+                await flushJobLogs(payload, jobId);
+            }
+            // If no items were fetched at all, ensure totalItems is set
+            if (!totalItemsSet) {
+                totalItems = 0;
+                await payload.update({
+                    id: jobId,
+                    collection: MIGRATION_COLLECTION,
+                    data: {
+                        progress: {
+                            failedItems: 0,
+                            processedItems: 0,
+                            successfulItems: 0,
+                            totalItems: 0
+                        }
+                    }
+                });
+                invalidateMigrationCache();
+            }
+        }
+        addJobLog('info', `Migration completed. ${counters.successfulItems} items successfully migrated, ${counters.failedItems} failed.`);
+        await flushJobLogs(payload, jobId);
+        await payload.update({
+            id: jobId,
+            collection: MIGRATION_COLLECTION,
+            data: {
+                endTime: new Date(),
+                progress: {
+                    failedItemIds: counters.failedItemIds,
+                    failedItems: counters.failedItems,
+                    processedItems: counters.processedItems,
+                    successfulItems: counters.successfulItems,
+                    totalItems
+                },
+                status: 'completed'
+            }
+        });
+        invalidateMigrationCache();
+    } catch (error) {
+        addJobLog('error', `Migration failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        await flushJobLogs(payload, jobId);
+        await payload.update({
+            id: jobId,
+            collection: MIGRATION_COLLECTION,
+            data: {
+                endTime: new Date(),
+                status: 'failed'
+            }
+        });
+        invalidateMigrationCache();
+        throw error;
+    }
+};
+
+export { processMigrationJob };
+//# sourceMappingURL=orchestrator.js.map