monoco-toolkit 0.3.10__py3-none-any.whl → 0.3.12__py3-none-any.whl

monoco/core/ingestion/__init__.py

@@ -0,0 +1,20 @@
+"""
+Monoco Mailroom - Automated Ingestion System
+
+Provides environment discovery, file watching, and automated conversion
+for document ingestion into the Monoco Artifact System.
+"""
+
+from .discovery import EnvironmentDiscovery, ConversionTool
+from .worker import ConversionWorker, ConversionTask, ConversionResult
+from .watcher import DropzoneWatcher, IngestionEvent
+
+__all__ = [
+    "EnvironmentDiscovery",
+    "ConversionTool",
+    "ConversionWorker",
+    "ConversionTask",
+    "ConversionResult",
+    "DropzoneWatcher",
+    "IngestionEvent",
+]

monoco/core/ingestion/discovery.py

@@ -0,0 +1,248 @@
+"""
+Environment Discovery Module for Monoco Mailroom.
+
+Automatically detects available document conversion tools in the system,
+including LibreOffice (soffice), Pandoc, and PDF processing engines.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from dataclasses import dataclass, field
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+
+class ToolType(str, Enum):
+    """Types of conversion tools supported."""
+    LIBREOFFICE = "libreoffice"
+    PANDOC = "pandoc"
+    PDF2TEXT = "pdf2text"
+    PDFTOHTML = "pdftohtml"
+    CUSTOM = "custom"
+
+
+class ToolCapability(str, Enum):
+    """Capabilities of conversion tools."""
+    DOCX_TO_TEXT = "docx_to_text"
+    DOCX_TO_MD = "docx_to_md"
+    PDF_TO_TEXT = "pdf_to_text"
+    PDF_TO_HTML = "pdf_to_html"
+    ODT_TO_TEXT = "odt_to_text"
+    XLSX_TO_CSV = "xlsx_to_csv"
+    PPTX_TO_TEXT = "pptx_to_text"
+
+
+@dataclass
+class ConversionTool:
+    """Represents a discovered conversion tool."""
+    name: str
+    tool_type: ToolType
+    executable_path: Path
+    version: str = "unknown"
+    capabilities: list[ToolCapability] = field(default_factory=list)
+    priority: int = 0  # Higher = preferred
+
+    def is_available(self) -> bool:
+        """Check if the tool executable exists and is runnable."""
+        return self.executable_path.exists() and os.access(self.executable_path, os.X_OK)
+
+
+class EnvironmentDiscovery:
+    """
+    Discovers and manages document conversion tools in the system.
+    
+    Automatically detects:
+    - LibreOffice (soffice) for Office document conversion
+    - Pandoc for markdown/text conversion
+    - PDF utilities (pdftotext, pdftohtml)
+    """
+
+    # Known executable names to search for
+    LIBREOFFICE_BINARIES = ["soffice", "libreoffice", "soffice.bin"]
+    PANDOC_BINARIES = ["pandoc"]
+    PDF_TOOLS = ["pdftotext", "pdftohtml", "pdf2txt.py"]
+
+    def __init__(self):
+        self._tools: dict[ToolType, list[ConversionTool]] = {}
+        self._discovered = False
+
+    def discover(self, force: bool = False) -> dict[ToolType, list[ConversionTool]]:
+        """
+        Discover all available conversion tools.
+        
+        Args:
+            force: Force re-discovery even if already done
+            
+        Returns:
+            Dictionary mapping tool types to lists of discovered tools
+        """
+        if self._discovered and not force:
+            return self._tools
+
+        self._tools = {
+            ToolType.LIBREOFFICE: self._discover_libreoffice(),
+            ToolType.PANDOC: self._discover_pandoc(),
+            ToolType.PDF2TEXT: self._discover_pdf_tools(),
+        }
+        
+        self._discovered = True
+        return self._tools
+
+    def _find_executable(self, names: list[str]) -> Optional[Path]:
+        """Find the first available executable from a list of names."""
+        for name in names:
+            path = shutil.which(name)
+            if path:
+                return Path(path).resolve()
+        return None
+
+    def _get_version(self, executable: Path, version_arg: str = "--version") -> str:
+        """Get version string from an executable."""
+        try:
+            result = subprocess.run(
+                [str(executable), version_arg],
+                capture_output=True,
+                text=True,
+                timeout=5,
+                check=False,
+            )
+            # Extract version from first line of output
+            output = result.stdout or result.stderr
+            if output:
+                first_line = output.strip().split("\n")[0]
+                return first_line
+        except (subprocess.TimeoutExpired, OSError, ValueError):
+            pass
+        return "unknown"
+
+    def _discover_libreoffice(self) -> list[ConversionTool]:
+        """Discover LibreOffice installation."""
+        tools = []
+        executable = self._find_executable(self.LIBREOFFICE_BINARIES)
+        
+        if executable:
+            version = self._get_version(executable)
+            tools.append(ConversionTool(
+                name="LibreOffice",
+                tool_type=ToolType.LIBREOFFICE,
+                executable_path=executable,
+                version=version,
+                capabilities=[
+                    ToolCapability.DOCX_TO_TEXT,
+                    ToolCapability.DOCX_TO_MD,
+                    ToolCapability.ODT_TO_TEXT,
+                    ToolCapability.XLSX_TO_CSV,
+                    ToolCapability.PPTX_TO_TEXT,
+                ],
+                priority=100,  # High priority for Office docs
+            ))
+        
+        return tools
+
+    def _discover_pandoc(self) -> list[ConversionTool]:
+        """Discover Pandoc installation."""
+        tools = []
+        executable = self._find_executable(self.PANDOC_BINARIES)
+        
+        if executable:
+            version = self._get_version(executable)
+            tools.append(ConversionTool(
+                name="Pandoc",
+                tool_type=ToolType.PANDOC,
+                executable_path=executable,
+                version=version,
+                capabilities=[
+                    ToolCapability.DOCX_TO_MD,
+                    ToolCapability.DOCX_TO_TEXT,
+                    ToolCapability.ODT_TO_TEXT,
+                ],
+                priority=90,
+            ))
+        
+        return tools
+
+    def _discover_pdf_tools(self) -> list[ConversionTool]:
+        """Discover PDF conversion tools."""
+        tools = []
+        
+        # pdftotext (from poppler-utils)
+        pdftotext = self._find_executable(["pdftotext"])
+        if pdftotext:
+            version = self._get_version(pdftotext, "-v")
+            tools.append(ConversionTool(
+                name="pdftotext",
+                tool_type=ToolType.PDF2TEXT,
+                executable_path=pdftotext,
+                version=version,
+                capabilities=[ToolCapability.PDF_TO_TEXT],
+                priority=100,
+            ))
+        
+        # pdftohtml
+        pdftohtml = self._find_executable(["pdftohtml"])
+        if pdftohtml:
+            version = self._get_version(pdftohtml, "-v")
+            tools.append(ConversionTool(
+                name="pdftohtml",
+                tool_type=ToolType.PDFTOHTML,
+                executable_path=pdftohtml,
+                version=version,
+                capabilities=[ToolCapability.PDF_TO_HTML],
+                priority=80,
+            ))
+        
+        return tools
+
+    def get_best_tool(self, capability: ToolCapability) -> Optional[ConversionTool]:
+        """
+        Get the best available tool for a specific capability.
+        
+        Args:
+            capability: The required conversion capability
+            
+        Returns:
+            Best matching ConversionTool or None
+        """
+        if not self._discovered:
+            self.discover()
+
+        candidates = []
+        for tool_list in self._tools.values():
+            for tool in tool_list:
+                if capability in tool.capabilities:
+                    candidates.append(tool)
+
+        if not candidates:
+            return None
+
+        # Sort by priority (highest first)
+        candidates.sort(key=lambda t: t.priority, reverse=True)
+        return candidates[0]
+
+    def get_all_tools(self) -> list[ConversionTool]:
+        """Get all discovered tools."""
+        if not self._discovered:
+            self.discover()
+        
+        all_tools = []
+        for tool_list in self._tools.values():
+            all_tools.extend(tool_list)
+        return all_tools
+
+    def has_capability(self, capability: ToolCapability) -> bool:
+        """Check if any tool supports the given capability."""
+        return self.get_best_tool(capability) is not None
+
+    def get_capabilities_summary(self) -> dict[str, bool]:
+        """Get a summary of available capabilities."""
+        return {
+            cap.value: self.has_capability(cap)
+            for cap in ToolCapability
+        }
+
+
+# Import os here to avoid issues with dataclass
+import os

monoco/core/ingestion/watcher.py

@@ -0,0 +1,343 @@
+"""
+Dropzone Watcher for Monoco Mailroom.
+
+Monitors dropzone directories for new files and triggers
+automated ingestion workflows.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from pathlib import Path
+from typing import Optional, Callable, Any, Set
+
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler, FileCreatedEvent, FileMovedEvent
+
+from .worker import ConversionWorker, ConversionTask, ConversionResult, ConversionStatus
+from ..artifacts.manager import ArtifactManager
+from ..artifacts.models import ArtifactSourceType
+
+logger = logging.getLogger(__name__)
+
+
+class IngestionEventType(str, Enum):
+    """Types of ingestion events."""
+    FILE_DETECTED = "file_detected"
+    CONVERSION_STARTED = "conversion_started"
+    CONVERSION_COMPLETED = "conversion_completed"
+    CONVERSION_FAILED = "conversion_failed"
+    ARTIFACT_REGISTERED = "artifact_registered"
+
+
+@dataclass
+class IngestionEvent:
+    """Event emitted during the ingestion process."""
+    event_type: IngestionEventType
+    file_path: Path
+    task_id: Optional[str] = None
+    artifact_id: Optional[str] = None
+    error_message: Optional[str] = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+
+
+class DropzoneHandler(FileSystemEventHandler):
+    """File system event handler for dropzone monitoring."""
+
+    def __init__(
+        self,
+        dropzone_path: Path,
+        on_file_detected: Callable[[Path], None],
+        supported_extensions: Optional[Set[str]] = None,
+    ):
+        self.dropzone_path = Path(dropzone_path)
+        self.on_file_detected = on_file_detected
+        self.supported_extensions = supported_extensions or {
+            ".docx", ".doc", ".pdf", ".odt",
+            ".xlsx", ".xls", ".pptx", ".ppt",
+        }
+        self._processed_files: Set[Path] = set()
+
+    def on_created(self, event):
+        """Handle file creation events."""
+        if event.is_directory:
+            return
+        
+        file_path = Path(event.src_path)
+        if self._should_process(file_path):
+            self._processed_files.add(file_path.resolve())
+            self.on_file_detected(file_path)
+
+    def on_moved(self, event):
+        """Handle file move events (e.g., atomic writes)."""
+        if event.is_directory:
+            return
+        
+        file_path = Path(event.dest_path)
+        if self._should_process(file_path):
+            self._processed_files.add(file_path.resolve())
+            self.on_file_detected(file_path)
+
+    def _should_process(self, file_path: Path) -> bool:
+        """Check if a file should be processed."""
+        # Skip hidden files
+        if file_path.name.startswith("."):
+            return False
+        
+        # Skip temporary files
+        if file_path.suffix in (".tmp", ".temp", ".part"):
+            return False
+        
+        # Check extension
+        if file_path.suffix.lower() not in self.supported_extensions:
+            return False
+        
+        # Skip already processed
+        if file_path.resolve() in self._processed_files:
+            return False
+        
+        return True
+
+
+class DropzoneWatcher:
+    """
+    Watches dropzone directories and orchestrates automated ingestion.
+    
+    Features:
+    - Real-time file system monitoring
+    - Automatic conversion using ConversionWorker
+    - Artifact registration with ArtifactManager
+    - Event callbacks for integration
+    """
+
+    def __init__(
+        self,
+        dropzone_path: Path,
+        artifact_manager: ArtifactManager,
+        conversion_worker: Optional[ConversionWorker] = None,
+        output_dir: Optional[Path] = None,
+        process_existing: bool = False,
+    ):
+        """
+        Initialize the dropzone watcher.
+        
+        Args:
+            dropzone_path: Directory to monitor for new files
+            artifact_manager: ArtifactManager for registering converted files
+            conversion_worker: ConversionWorker for document conversion
+            output_dir: Directory for converted files (default: dropzone/converted)
+            process_existing: Whether to process files already in dropzone
+        """
+        self.dropzone_path = Path(dropzone_path)
+        self.artifact_manager = artifact_manager
+        self.conversion_worker = conversion_worker or ConversionWorker()
+        self.output_dir = output_dir or (self.dropzone_path / "converted")
+        self.process_existing = process_existing
+        
+        # Event callbacks
+        self._on_event: Optional[Callable[[IngestionEvent], None]] = None
+        
+        # State
+        self._observer: Optional[Observer] = None
+        self._running = False
+        self._pending_tasks: dict[str, asyncio.Task] = {}
+
+    def set_event_callback(self, callback: Callable[[IngestionEvent], None]) -> None:
+        """Set callback for ingestion events."""
+        self._on_event = callback
+
+    def _emit_event(self, event: IngestionEvent) -> None:
+        """Emit an ingestion event."""
+        if self._on_event:
+            try:
+                self._on_event(event)
+            except Exception:
+                pass
+
+    def start(self) -> None:
+        """Start watching the dropzone directory."""
+        if self._running:
+            return
+
+        # Ensure directories exist
+        self.dropzone_path.mkdir(parents=True, exist_ok=True)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Set up file system observer
+        self._handler = DropzoneHandler(
+            self.dropzone_path,
+            self._on_file_detected,
+            set(self.conversion_worker.get_supported_extensions()),
+        )
+        
+        self._observer = Observer()
+        self._observer.schedule(self._handler, str(self.dropzone_path), recursive=False)
+        self._observer.start()
+        
+        self._running = True
+        logger.info(f"Started watching dropzone: {self.dropzone_path}")
+
+        # Process existing files if requested
+        if self.process_existing:
+            self._scan_existing_files()
+
+    def stop(self) -> None:
+        """Stop watching the dropzone directory."""
+        if not self._running:
+            return
+
+        self._running = False
+
+        # Cancel pending tasks
+        for task in self._pending_tasks.values():
+            task.cancel()
+        self._pending_tasks.clear()
+
+        # Stop observer
+        if self._observer:
+            self._observer.stop()
+            self._observer.join()
+            self._observer = None
+
+        logger.info(f"Stopped watching dropzone: {self.dropzone_path}")
+
+    def _scan_existing_files(self) -> None:
+        """Scan and process existing files in dropzone."""
+        for file_path in self.dropzone_path.iterdir():
+            if file_path.is_file() and self._handler._should_process(file_path):
+                self._on_file_detected(file_path)
+
+    def _on_file_detected(self, file_path: Path) -> None:
+        """Handle newly detected file."""
+        logger.info(f"File detected: {file_path}")
+        
+        self._emit_event(IngestionEvent(
+            event_type=IngestionEventType.FILE_DETECTED,
+            file_path=file_path,
+        ))
+
+        # Create async task for processing
+        task_id = str(uuid.uuid4())
+        asyncio.create_task(self._process_file(file_path, task_id))
+
+    async def _process_file(self, file_path: Path, task_id: str) -> None:
+        """Process a detected file through the ingestion pipeline."""
+        try:
+            # Step 1: Check if conversion is needed/possible
+            if not self.conversion_worker.can_convert(file_path):
+                logger.warning(f"Cannot convert file: {file_path}")
+                self._emit_event(IngestionEvent(
+                    event_type=IngestionEventType.CONVERSION_FAILED,
+                    file_path=file_path,
+                    task_id=task_id,
+                    error_message="No conversion tool available for this file type",
+                ))
+                return
+
+            # Step 2: Create conversion task
+            conversion_task = ConversionTask(
+                task_id=task_id,
+                source_path=file_path,
+                target_format="txt",
+                output_dir=self.output_dir,
+            )
+
+            self._emit_event(IngestionEvent(
+                event_type=IngestionEventType.CONVERSION_STARTED,
+                file_path=file_path,
+                task_id=task_id,
+            ))
+
+            # Step 3: Perform conversion
+            result = await self.conversion_worker.submit(conversion_task)
+
+            if result.status != ConversionStatus.SUCCESS:
+                logger.error(f"Conversion failed for {file_path}: {result.error_message}")
+                self._emit_event(IngestionEvent(
+                    event_type=IngestionEventType.CONVERSION_FAILED,
+                    file_path=file_path,
+                    task_id=task_id,
+                    error_message=result.error_message,
+                ))
+                return
+
+            self._emit_event(IngestionEvent(
+                event_type=IngestionEventType.CONVERSION_COMPLETED,
+                file_path=file_path,
+                task_id=task_id,
+                metadata={
+                    "output_path": str(result.output_path),
+                    "processing_time_ms": result.processing_time_ms,
+                },
+            ))
+
+            # Step 4: Register as artifact
+            if result.output_path and result.output_path.exists():
+                artifact_meta = self._register_artifact(
+                    result.output_path,
+                    source_file=file_path,
+                    conversion_metadata=result.metadata,
+                )
+                
+                self._emit_event(IngestionEvent(
+                    event_type=IngestionEventType.ARTIFACT_REGISTERED,
+                    file_path=file_path,
+                    task_id=task_id,
+                    artifact_id=artifact_meta.artifact_id,
+                    metadata={
+                        "content_hash": artifact_meta.content_hash,
+                        "content_type": artifact_meta.content_type,
+                    },
+                ))
+
+                logger.info(f"Successfully ingested {file_path} as artifact {artifact_meta.artifact_id}")
+
+        except Exception as e:
+            logger.exception(f"Error processing file {file_path}")
+            self._emit_event(IngestionEvent(
+                event_type=IngestionEventType.CONVERSION_FAILED,
+                file_path=file_path,
+                task_id=task_id,
+                error_message=str(e),
+            ))
+
+    def _register_artifact(
+        self,
+        file_path: Path,
+        source_file: Path,
+        conversion_metadata: dict[str, Any],
+    ) -> Any:
+        """Register converted file as an artifact."""
+        metadata = {
+            "source_file": str(source_file),
+            "original_filename": source_file.name,
+            **conversion_metadata,
+        }
+        
+        return self.artifact_manager.store_file(
+            file_path=file_path,
+            source_type=ArtifactSourceType.IMPORTED,
+            content_type="text/plain",
+            tags=["mailroom", "converted", source_file.suffix.lower().lstrip(".")],
+            metadata=metadata,
+        )
+
+    def is_running(self) -> bool:
+        """Check if the watcher is currently running."""
+        return self._running
+
+    def get_stats(self) -> dict[str, Any]:
+        """Get watcher statistics."""
+        return {
+            "running": self._running,
+            "dropzone_path": str(self.dropzone_path),
+            "output_dir": str(self.output_dir),
+            "pending_tasks": len(self._pending_tasks),
+            "supported_extensions": list(self.conversion_worker.get_supported_extensions()),
+        }