rag-lite-ts 1.0.2 → 2.0.1

package/dist/core/content-manager.d.ts

@@ -0,0 +1,343 @@
+/**
+ * Content Manager - Handles content ingestion routing for unified content system
+ * Routes filesystem content to reference storage and memory content to content directory
+ * Implements deduplication and content ID generation
+ */
+import { DatabaseConnection } from './db.js';
+/**
+ * Memory content metadata for ingestion
+ */
+export interface MemoryContentMetadata {
+    displayName: string;
+    contentType?: string;
+    originalPath?: string;
+}
+/**
+ * Content ingestion result
+ */
+export interface ContentIngestionResult {
+    contentId: string;
+    wasDeduped: boolean;
+    storageType: 'filesystem' | 'content_dir';
+    contentPath: string;
+}
+/**
+ * Configuration for ContentManager (input interface)
+ */
+export interface ContentManagerConfig {
+    contentDir: string;
+    maxFileSize: number | string;
+    maxContentDirSize: number | string;
+    enableDeduplication: boolean;
+    enableStorageTracking: boolean;
+    storageWarningThreshold: number;
+    storageErrorThreshold: number;
+}
+/**
+ * ContentManager class for handling content ingestion routing
+ * Implements the unified content system's ingestion logic
+ */
+export declare class ContentManager {
+    private db;
+    private config;
+    private streamingOps;
+    private performanceOptimizer;
+    constructor(db: DatabaseConnection, config?: Partial<ContentManagerConfig>);
+    /**
+     * Ingests content from filesystem by creating references without copying files
+     * @param filePath - Path to the file to ingest
+     * @returns Promise that resolves to content ingestion result
+     */
+    ingestFromFilesystem(filePath: string): Promise<ContentIngestionResult>;
+    /**
+     * Ingests content from memory by storing it in content directory with hash-based filenames
+     * @param content - Buffer containing the content
+     * @param metadata - Memory content metadata
+     * @returns Promise that resolves to content ingestion result
+     */
+    ingestFromMemory(content: Buffer, metadata: MemoryContentMetadata): Promise<ContentIngestionResult>;
+    /**
+     * Generates a stable content ID using SHA-256 hash of content
+     * @param content - Buffer containing the content
+     * @returns Content ID string
+     */
+    generateContentId(content: Buffer): string;
+    /**
+     * Generates a unique content ID from an existing hash
+     * @param hash - Content hash
+     * @returns Content ID string
+     */
+    generateContentIdFromHash(hash: string): string;
+    /**
+     * Gets performance statistics for monitoring and optimization
+     * @returns Performance statistics
+     */
+    getPerformanceStats(): {
+        hashCache: {
+            size: number;
+            maxSize: number;
+            hitRate: number;
+        };
+        operations: {
+            totalOperations: number;
+            averageDuration: number;
+            totalBytesProcessed: number;
+            averageSpeed: number;
+            cacheHitRate: number;
+            errorRate: number;
+        };
+    };
+    /**
+     * Clears performance caches and resets metrics
+     */
+    clearPerformanceCaches(): void;
+    /**
+     * Checks if content with given ID already exists (deduplication check)
+     * @param contentId - Content ID to check
+     * @returns Promise that resolves to true if content exists, false otherwise
+     */
+    deduplicateContent(contentId: string): Promise<boolean>;
+    /**
+     * Enforces storage limits before accepting new content
+     * @param contentSize - Size of content to add in bytes
+     * @returns Promise that resolves if content can be added, throws error otherwise
+     */
+    enforceStorageLimits(contentSize: number): Promise<void>;
+    /**
+     * Gets storage limit status and recommendations
+     * @returns Promise that resolves to storage limit status
+     */
+    getStorageLimitStatus(): Promise<{
+        currentUsagePercent: number;
+        isNearWarningThreshold: boolean;
+        isNearErrorThreshold: boolean;
+        canAcceptContent: boolean;
+        recommendations: string[];
+        limits: {
+            warningThreshold: number;
+            errorThreshold: number;
+            maxSizeMB: number;
+            currentSizeMB: number;
+            remainingSizeMB: number;
+        };
+    }>;
+    /**
+     * Parses size string or number to bytes
+     * @param size - Size as number (bytes) or string like "50MB", "2GB"
+     * @returns Size in bytes
+     */
+    private parseSizeToBytes;
+    /**
+     * Generates SHA-256 hash of content
+     * @param content - Buffer containing the content
+     * @returns SHA-256 hash string
+     */
+    private generateContentHash;
+    /**
+     * Detects content type from file path and optionally content using enhanced magic number detection
+     * @param filePath - Path to the file
+     * @param content - File content buffer (optional)
+     * @returns MIME type string
+     */
+    private detectContentType;
+    /**
+     * Detects content type from buffer and optional filename for memory-based ingestion
+     * @param content - Content buffer
+     * @param filename - Optional filename for extension-based detection
+     * @returns MIME type string
+     */
+    private detectContentTypeFromBuffer;
+    /**
+     * Enhanced magic number detection for comprehensive content type identification
+     * @param content - Content buffer to analyze
+     * @returns MIME type string based on magic numbers, or 'application/octet-stream' if unknown
+     */
+    private detectContentTypeByMagicNumbers;
+    /**
+     * Extension-based content type detection with comprehensive mapping
+     * @param extension - File extension (with or without dot)
+     * @returns MIME type string based on extension, or 'application/octet-stream' if unknown
+     */
+    private detectContentTypeByExtension;
+    /**
+     * Validates if a content type is supported for processing
+     * @param contentType - MIME type to validate
+     * @returns Object with validation result and error message if unsupported
+     */
+    private validateContentType;
+    /**
+     * Gets file extension from content type with enhanced mapping
+     * @param contentType - MIME type
+     * @returns File extension with dot, or null if unknown
+     */
+    private getExtensionFromContentType;
+    /**
+     * Enhanced text content detection with better UTF-8 and encoding support
+     * @param content - Content buffer
+     * @returns True if content appears to be text
+     */
+    private isTextContent;
+    /**
+     * Checks if a byte at given position is part of a valid UTF-8 sequence
+     * @param buffer - Buffer to check
+     * @param index - Index of the byte to check
+     * @returns True if the byte is part of valid UTF-8
+     */
+    private isValidUTF8Byte;
+    /**
+     * Ensures content directory exists
+     * @returns Promise that resolves when directory is created
+     */
+    private ensureContentDirectory;
+    /**
+     * Writes content to file atomically to prevent partial writes
+     * @param filePath - Path to write to
+     * @param content - Content to write
+     * @returns Promise that resolves when write is complete
+     * @deprecated Use writeFileAtomic from resource-cleanup.ts for better resource management
+     */
+    private writeContentAtomic;
+    /**
+     * Gets comprehensive storage statistics for monitoring and reporting
+     * @returns Promise that resolves to detailed storage statistics
+     */
+    getStorageStats(): Promise<{
+        contentDirectory: {
+            totalFiles: number;
+            totalSize: number;
+            totalSizeMB: number;
+            averageFileSize: number;
+        };
+        filesystemReferences: {
+            totalRefs: number;
+            totalSize: number;
+            totalSizeMB: number;
+        };
+        overall: {
+            totalContentItems: number;
+            totalStorageUsed: number;
+            totalStorageUsedMB: number;
+            storageEfficiency: number;
+        };
+        limits: {
+            maxContentDirSize: number;
+            maxContentDirSizeMB: number;
+            currentUsagePercent: number;
+            remainingSpace: number;
+            remainingSpaceMB: number;
+        };
+        lastUpdated: Date;
+        lastCleanup: Date | null;
+    }>;
+    /**
+     * Gets current storage statistics for the content directory (legacy method)
+     * @returns Promise that resolves to storage statistics
+     * @deprecated Use getStorageStats() for more comprehensive statistics
+     */
+    getContentDirectoryStats(): Promise<{
+        totalFiles: number;
+        totalSize: number;
+        filesystemRefs: number;
+        lastCleanup: Date | null;
+    }>;
+    /**
+     * Generates a simple, human-readable storage usage report
+     * @returns Promise that resolves to formatted storage report
+     */
+    generateStorageReport(): Promise<string>;
+    /**
+     * Gets storage statistics in a format suitable for monitoring systems
+     * @returns Promise that resolves to monitoring-friendly statistics
+     */
+    getStorageMetrics(): Promise<{
+        contentDirFiles: number;
+        contentDirSizeBytes: number;
+        contentDirSizeMB: number;
+        filesystemRefs: number;
+        filesystemSizeBytes: number;
+        filesystemSizeMB: number;
+        totalContentItems: number;
+        totalStorageBytes: number;
+        totalStorageMB: number;
+        usagePercent: number;
+        remainingBytes: number;
+        remainingMB: number;
+        lastCleanupTimestamp: number | null;
+        lastUpdatedTimestamp: number;
+    }>;
+    /**
+     * Updates storage statistics by scanning the content directory
+     * @returns Promise that resolves when stats are updated
+     */
+    updateStorageStats(): Promise<void>;
+    /**
+     * Checks if adding new content would exceed storage limits (legacy method)
+     * @param contentSize - Size of content to add
+     * @returns Promise that resolves to true if within limits, false otherwise
+     * @deprecated Use enforceStorageLimits() for better error handling and guidance
+     */
+    checkStorageLimits(contentSize: number): Promise<boolean>;
+    /**
+     * Removes orphaned files that exist in content directory but have no metadata references
+     * @returns Promise that resolves to cleanup results
+     */
+    removeOrphanedFiles(): Promise<{
+        removedFiles: string[];
+        errors: string[];
+        freedSpace: number;
+    }>;
+    /**
+     * Removes duplicate content files based on content hash, keeping the first occurrence
+     * @returns Promise that resolves to deduplication results
+     */
+    removeDuplicateContent(): Promise<{
+        removedFiles: string[];
+        errors: string[];
+        freedSpace: number;
+    }>;
+    /**
+     * Cleans up orphaned files in the content directory
+     * Removes files that exist in the directory but have no corresponding metadata
+     * @returns Promise that resolves to cleanup results
+     */
+    cleanupOrphanedFiles(): Promise<{
+        removedFiles: string[];
+        errors: string[];
+        freedSpace: number;
+    }>;
+    /**
+     * Removes duplicate content files based on content hash
+     * Keeps the first occurrence and removes duplicates
+     * @returns Promise that resolves to deduplication results
+     */
+    deduplicateContentFiles(): Promise<{
+        removedFiles: string[];
+        errors: string[];
+        freedSpace: number;
+    }>;
+    /**
+     * Ensures content directory has proper permissions
+     * @returns Promise that resolves when permissions are set
+     */
+    ensureContentDirectoryPermissions(): Promise<void>;
+    /**
+     * Validates content directory structure and repairs if needed
+     * @returns Promise that resolves to validation results
+     */
+    validateAndRepairContentDirectory(): Promise<{
+        isValid: boolean;
+        issues: string[];
+        repaired: string[];
+    }>;
+    /**
+     * Gets actual directory statistics by scanning the filesystem
+     * @returns Promise that resolves to actual directory stats
+     */
+    private getActualDirectoryStats;
+    /**
+     * Cleanup resources to prevent memory leaks and hanging processes
+     * Should be called when ContentManager is no longer needed
+     */
+    cleanup(): void;
+}
+//# sourceMappingURL=content-manager.d.ts.map