pdf-oxide-fips 0.3.47

package/lib/managers/batch-manager.d.ts

@@ -0,0 +1,185 @@
+/**
+ * Batch Processing Manager for Parallel Document Operations
+ *
+ * Enables efficient parallel processing of multiple PDF documents with:
+ * - Configurable concurrency control
+ * - Real-time progress tracking with ETA calculation
+ * - Memory-aware backpressure handling
+ * - Per-document timeout support
+ * - Error resilience and detailed reporting
+ *
+ * @example
+ * ```typescript
+ * import { BatchManager } from 'pdf-oxide';
+ * import { PdfDocument } from 'pdf-oxide';
+ *
+ * const batch = new BatchManager([
+ *   { path: 'doc1.pdf' },
+ *   { path: 'doc2.pdf' },
+ *   { path: 'doc3.pdf' }
+ * ]);
+ *
+ * // Extract text from multiple documents in parallel
+ * const results = await batch.extractTextBatch({
+ *   maxParallel: 4,
+ *   timeout: 30000,
+ *   onProgress: (progress) => {
+ *     console.log('Progress: ' + Math.round(progress.progress * 100) + '%');
+ *     console.log('ETA: ' + progress.eta + 'ms');
+ *   }
+ * });
+ *
+ * results.forEach(result => {
+ *   if (result.success) {
+ *     console.log(result.document.path + ': ' + result.data.length + ' chars');
+ *   } else {
+ *     console.error(result.document.path + ': ' + result.error.message);
+ *   }
+ * });
+ * ```
+ */
+/**
+ * Represents a document to be processed in a batch
+ */
+export interface BatchDocument {
+    /** File path to the PDF document */
+    path: string;
+    /** Optional unique identifier */
+    id?: string;
+    /** Priority (1-10, default 5) */
+    priority?: number;
+    /** Optional metadata */
+    metadata?: Record<string, any>;
+}
+/**
+ * Progress information for batch operations
+ */
+export interface BatchProgress {
+    /** Total documents in batch */
+    total: number;
+    /** Number of successfully completed documents */
+    completed: number;
+    /** Number of failed documents */
+    failed: number;
+    /** Current document being processed (index) */
+    current: number;
+    /** Progress percentage (0.0-1.0) */
+    progress: number;
+    /** Estimated time remaining in milliseconds */
+    eta: number;
+    /** Number of currently active operations */
+    activeOperations: number;
+    /** Batch start time (milliseconds since epoch) */
+    startTime: number;
+    /** Elapsed time since start (milliseconds) */
+    elapsedTime: number;
+}
+/**
+ * Result of processing a single document in a batch
+ */
+export interface BatchResult<T = any> {
+    /** The document that was processed */
+    document: BatchDocument;
+    /** Whether the operation succeeded */
+    success: boolean;
+    /** The result data if successful */
+    data?: T;
+    /** Error if operation failed */
+    error?: Error;
+    /** Time to process this document (milliseconds) */
+    duration: number;
+}
+/**
+ * Options for batch processing
+ */
+export interface BatchOptions {
+    /** Maximum number of parallel operations (default: CPU count) */
+    maxParallel?: number;
+    /** Timeout per document in milliseconds (default: 30000) */
+    timeout?: number;
+    /** Progress callback invoked on each document completion */
+    onProgress?: (progress: BatchProgress) => void;
+    /** Backpressure configuration for memory management */
+    backpressure?: {
+        /** Maximum memory usage in MB (default: 500) */
+        maxMemoryMB?: number;
+        /** Interval to check memory in ms (default: 1000) */
+        checkInterval?: number;
+    };
+}
+/**
+ * Statistics for completed batch operations
+ */
+export interface BatchStatistics {
+    /** Total documents processed */
+    total: number;
+    /** Successfully completed documents */
+    completed: number;
+    /** Failed documents */
+    failed: number;
+    /** Total time elapsed (milliseconds) */
+    totalTime: number;
+    /** Average time per document (milliseconds) */
+    averageTime: number;
+    /** Documents per second throughput */
+    throughput: number;
+    /** Peak memory usage (MB) */
+    peakMemory: number;
+}
+/**
+ * Batch processor for parallel document operations
+ */
+export declare class BatchManager {
+    private documents;
+    private stats;
+    /**
+     * Creates a new BatchManager
+     * @param documents - Array of documents to process
+     * @throws Error if documents array is empty or invalid
+     */
+    constructor(documents: BatchDocument[]);
+    /**
+     * Get current statistics
+     */
+    getStatistics(): BatchStatistics;
+    /**
+     * Process documents in a queue with concurrency control
+     * @private
+     */
+    private processQueue;
+    /**
+     * Extract text from multiple documents in parallel
+     * @param options - Batch processing options
+     * @returns Array of extraction results
+     */
+    extractTextBatch(options?: BatchOptions): Promise<BatchResult<string>[]>;
+    /**
+     * Extract markdown from multiple documents in parallel
+     * @param options - Batch processing options
+     * @returns Array of extraction results
+     */
+    extractMarkdownBatch(options?: BatchOptions): Promise<BatchResult<string>[]>;
+    /**
+     * Extract HTML from multiple documents in parallel
+     * @param options - Batch processing options
+     * @returns Array of extraction results
+     */
+    extractHtmlBatch(options?: BatchOptions): Promise<BatchResult<string>[]>;
+    /**
+     * Search for a term in multiple documents in parallel
+     * @param searchTerm - Term to search for
+     * @param options - Batch processing options
+     * @returns Array of search results
+     */
+    searchBatch(searchTerm: string, options?: BatchOptions): Promise<BatchResult<Array<{
+        page: number;
+        count: number;
+    }>>[]>;
+    /**
+     * Generic batch processor for custom operations
+     * @param processor - Function to process each document
+     * @param options - Batch processing options
+     * @returns Array of results
+     */
+    processBatch<T>(processor: (doc: BatchDocument, pdfDoc: any) => Promise<T>, options?: BatchOptions): Promise<BatchResult<T>[]>;
+}

package/lib/managers/batch-manager.js

@@ -0,0 +1,385 @@
+/**
+ * Batch Processing Manager for Parallel Document Operations
+ *
+ * Enables efficient parallel processing of multiple PDF documents with:
+ * - Configurable concurrency control
+ * - Real-time progress tracking with ETA calculation
+ * - Memory-aware backpressure handling
+ * - Per-document timeout support
+ * - Error resilience and detailed reporting
+ *
+ * @example
+ * ```typescript
+ * import { BatchManager } from 'pdf-oxide';
+ * import { PdfDocument } from 'pdf-oxide';
+ *
+ * const batch = new BatchManager([
+ *   { path: 'doc1.pdf' },
+ *   { path: 'doc2.pdf' },
+ *   { path: 'doc3.pdf' }
+ * ]);
+ *
+ * // Extract text from multiple documents in parallel
+ * const results = await batch.extractTextBatch({
+ *   maxParallel: 4,
+ *   timeout: 30000,
+ *   onProgress: (progress) => {
+ *     console.log('Progress: ' + Math.round(progress.progress * 100) + '%');
+ *     console.log('ETA: ' + progress.eta + 'ms');
+ *   }
+ * });
+ *
+ * results.forEach(result => {
+ *   if (result.success) {
+ *     console.log(result.document.path + ': ' + result.data.length + ' chars');
+ *   } else {
+ *     console.error(result.document.path + ': ' + result.error.message);
+ *   }
+ * });
+ * ```
+ */
+import os from 'os';
+/**
+ * Batch processor for parallel document operations
+ */
+export class BatchManager {
+    /**
+     * Creates a new BatchManager
+     * @param documents - Array of documents to process
+     * @throws Error if documents array is empty or invalid
+     */
+    constructor(documents) {
+        this.stats = {
+            total: 0,
+            completed: 0,
+            failed: 0,
+            totalTime: 0,
+            averageTime: 0,
+            throughput: 0,
+            peakMemory: 0,
+        };
+        if (!Array.isArray(documents) || documents.length === 0) {
+            throw new Error('Documents array must not be empty');
+        }
+        for (const doc of documents) {
+            if (!doc.path || typeof doc.path !== 'string') {
+                throw new Error('Each document must have a valid path property');
+            }
+        }
+        this.documents = documents;
+        this.stats.total = documents.length;
+    }
+    /**
+     * Get current statistics
+     */
+    getStatistics() {
+        return { ...this.stats };
+    }
+    /**
+     * Process documents in a queue with concurrency control
+     * @private
+     */
+    async processQueue(processor, options = {}) {
+        const maxParallel = options.maxParallel || os.cpus().length;
+        const timeout = options.timeout || 30000;
+        const backpressure = options.backpressure || {
+            maxMemoryMB: 500,
+            checkInterval: 1000,
+        };
+        const results = [];
+        const startTime = Date.now();
+        let completed = 0;
+        let failed = 0;
+        let active = 0;
+        // Progress tracking helper
+        const reportProgress = () => {
+            const elapsedTime = Date.now() - startTime;
+            const completedDocs = completed + failed;
+            const avgTimePerDoc = completedDocs > 0 ? elapsedTime / completedDocs : 0;
+            const eta = completedDocs > 0 ? (this.documents.length - completedDocs) * avgTimePerDoc : 0;
+            if (options.onProgress) {
+                options.onProgress({
+                    total: this.documents.length,
+                    completed,
+                    failed,
+                    current: completedDocs,
+                    progress: this.documents.length > 0 ? completedDocs / this.documents.length : 0,
+                    eta: Math.max(0, eta),
+                    activeOperations: active,
+                    startTime,
+                    elapsedTime,
+                });
+            }
+        };
+        // Memory monitoring helper
+        const checkMemory = async () => {
+            const memUsageMB = process.memoryUsage().heapUsed / 1024 / 1024;
+            this.stats.peakMemory = Math.max(this.stats.peakMemory, memUsageMB);
+            if (memUsageMB > (backpressure.maxMemoryMB || 500)) {
+                // Wait a bit for garbage collection
+                await new Promise((resolve) => setTimeout(resolve, 100));
+            }
+        };
+        // Process all documents with concurrency control
+        let index = 0;
+        const queue = [];
+        while (index < this.documents.length || queue.length > 0) {
+            // Check memory before starting new operations
+            await checkMemory();
+            // Start new operations while under concurrency limit
+            while (active < maxParallel && index < this.documents.length) {
+                const docIndex = index++;
+                const doc = this.documents[docIndex];
+                active++;
+                const promise = (async () => {
+                    const docStartTime = Date.now();
+                    try {
+                        const result = await Promise.race([
+                            processor(doc, docIndex),
+                            new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout after ' + timeout + 'ms')), timeout)),
+                        ]);
+                        result.duration = Date.now() - docStartTime;
+                        results[docIndex] = result;
+                        if (result.success) {
+                            completed++;
+                        }
+                        else {
+                            failed++;
+                        }
+                    }
+                    catch (error) {
+                        const duration = Date.now() - docStartTime;
+                        results[docIndex] = {
+                            document: doc,
+                            success: false,
+                            error: error instanceof Error ? error : new Error(String(error)),
+                            duration,
+                        };
+                        failed++;
+                    }
+                    finally {
+                        active--;
+                        reportProgress();
+                    }
+                })();
+                queue.push(promise);
+            }
+            // Wait for at least one operation to complete
+            if (queue.length > 0) {
+                await Promise.race(queue);
+                const idx = queue.findIndex((p) => p !== undefined);
+                if (idx >= 0) {
+                    queue.splice(idx, 1);
+                }
+            }
+        }
+        // Update final statistics
+        const totalTime = Date.now() - startTime;
+        this.stats.totalTime = totalTime;
+        this.stats.completed = completed;
+        this.stats.failed = failed;
+        this.stats.averageTime = completed > 0 ? totalTime / completed : 0;
+        this.stats.throughput = totalTime > 0 ? (completed / totalTime) * 1000 : 0;
+        return results.filter((r) => r !== undefined);
+    }
+    /**
+     * Extract text from multiple documents in parallel
+     * @param options - Batch processing options
+     * @returns Array of extraction results
+     */
+    async extractTextBatch(options = {}) {
+        return this.processQueue(async (doc, _index) => {
+            try {
+                // Dynamic import to avoid circular dependencies
+                const { PdfDocument } = await import('../index.js');
+                const pdfDoc = PdfDocument.open(doc.path);
+                const extractionMgr = pdfDoc.createExtractionManager?.();
+                if (!extractionMgr) {
+                    throw new Error('Failed to create extraction manager');
+                }
+                const text = extractionMgr.extractAllText();
+                if (typeof pdfDoc.close === 'function') {
+                    pdfDoc.close();
+                }
+                return {
+                    document: doc,
+                    success: true,
+                    data: text,
+                    duration: 0,
+                };
+            }
+            catch (error) {
+                return {
+                    document: doc,
+                    success: false,
+                    error: error instanceof Error ? error : new Error(String(error)),
+                    duration: 0,
+                };
+            }
+        }, options);
+    }
+    /**
+     * Extract markdown from multiple documents in parallel
+     * @param options - Batch processing options
+     * @returns Array of extraction results
+     */
+    async extractMarkdownBatch(options = {}) {
+        return this.processQueue(async (doc, _index) => {
+            try {
+                const { PdfDocument } = await import('../index.js');
+                const pdfDoc = PdfDocument.open(doc.path);
+                const extractionMgr = pdfDoc.createExtractionManager?.();
+                if (!extractionMgr) {
+                    throw new Error('Failed to create extraction manager');
+                }
+                // Extract markdown from all pages
+                let markdown = '';
+                const pageCount = pdfDoc.pageCount || 0;
+                for (let i = 0; i < pageCount; i++) {
+                    markdown += extractionMgr.extractMarkdown(i) || '';
+                    if (i < pageCount - 1) {
+                        markdown += '\n\n---\n\n';
+                    }
+                }
+                if (typeof pdfDoc.close === 'function') {
+                    pdfDoc.close();
+                }
+                return {
+                    document: doc,
+                    success: true,
+                    data: markdown,
+                    duration: 0,
+                };
+            }
+            catch (error) {
+                return {
+                    document: doc,
+                    success: false,
+                    error: error instanceof Error ? error : new Error(String(error)),
+                    duration: 0,
+                };
+            }
+        }, options);
+    }
+    /**
+     * Extract HTML from multiple documents in parallel
+     * @param options - Batch processing options
+     * @returns Array of extraction results
+     */
+    async extractHtmlBatch(options = {}) {
+        return this.processQueue(async (doc, _index) => {
+            try {
+                const { PdfDocument } = await import('../index.js');
+                const pdfDoc = PdfDocument.open(doc.path);
+                const extractionMgr = pdfDoc.createExtractionManager?.();
+                if (!extractionMgr) {
+                    throw new Error('Failed to create extraction manager');
+                }
+                // Extract HTML from all pages
+                let html = '<html><body>';
+                const pageCount = pdfDoc.pageCount || 0;
+                for (let i = 0; i < pageCount; i++) {
+                    html += '<div class="page page-' + (i + 1) + '">';
+                    html += extractionMgr.extractHtml(i) || '';
+                    html += '</div>';
+                }
+                html += '</body></html>';
+                if (typeof pdfDoc.close === 'function') {
+                    pdfDoc.close();
+                }
+                return {
+                    document: doc,
+                    success: true,
+                    data: html,
+                    duration: 0,
+                };
+            }
+            catch (error) {
+                return {
+                    document: doc,
+                    success: false,
+                    error: error instanceof Error ? error : new Error(String(error)),
+                    duration: 0,
+                };
+            }
+        }, options);
+    }
+    /**
+     * Search for a term in multiple documents in parallel
+     * @param searchTerm - Term to search for
+     * @param options - Batch processing options
+     * @returns Array of search results
+     */
+    async searchBatch(searchTerm, options = {}) {
+        if (!searchTerm || typeof searchTerm !== 'string') {
+            throw new Error('Search term must be a non-empty string');
+        }
+        return this.processQueue(async (doc, _index) => {
+            try {
+                const { PdfDocument } = await import('../index.js');
+                const pdfDoc = PdfDocument.open(doc.path);
+                const searchMgr = pdfDoc.createSearchManager?.();
+                if (!searchMgr) {
+                    throw new Error('Failed to create search manager');
+                }
+                const results = [];
+                const pageCount = pdfDoc.pageCount || 0;
+                for (let i = 0; i < pageCount; i++) {
+                    const matches = searchMgr.search(searchTerm, i) || [];
+                    if (matches.length > 0) {
+                        results.push({ page: i, count: matches.length });
+                    }
+                }
+                if (typeof pdfDoc.close === 'function') {
+                    pdfDoc.close();
+                }
+                return {
+                    document: doc,
+                    success: true,
+                    data: results,
+                    duration: 0,
+                };
+            }
+            catch (error) {
+                return {
+                    document: doc,
+                    success: false,
+                    error: error instanceof Error ? error : new Error(String(error)),
+                    duration: 0,
+                };
+            }
+        }, options);
+    }
+    /**
+     * Generic batch processor for custom operations
+     * @param processor - Function to process each document
+     * @param options - Batch processing options
+     * @returns Array of results
+     */
+    async processBatch(processor, options = {}) {
+        return this.processQueue(async (doc, _index) => {
+            try {
+                const { PdfDocument } = await import('../index.js');
+                const pdfDoc = PdfDocument.open(doc.path);
+                const data = await processor(doc, pdfDoc);
+                if (typeof pdfDoc.close === 'function') {
+                    pdfDoc.close();
+                }
+                return {
+                    document: doc,
+                    success: true,
+                    data,
+                    duration: 0,
+                };
+            }
+            catch (error) {
+                return {
+                    document: doc,
+                    success: false,
+                    error: error instanceof Error ? error : new Error(String(error)),
+                    duration: 0,
+                };
+            }
+        }, options);
+    }
+}