monoco-toolkit 0.3.9__py3-none-any.whl → 0.3.11__py3-none-any.whl

monoco/core/ingestion/worker.py

@@ -0,0 +1,436 @@
+"""
+Conversion Worker for Monoco Mailroom.
+
+Handles document conversion tasks using discovered tools.
+Supports concurrent processing with asyncio.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import shutil
+import subprocess
+import tempfile
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from pathlib import Path
+from typing import Optional, Callable, Any
+
+from .discovery import EnvironmentDiscovery, ToolCapability, ConversionTool
+
+logger = logging.getLogger(__name__)
+
+
+class ConversionStatus(str, Enum):
+    """Status of a conversion task."""
+    PENDING = "pending"
+    PROCESSING = "processing"
+    SUCCESS = "success"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+
+@dataclass
+class ConversionTask:
+    """Represents a document conversion task."""
+    task_id: str
+    source_path: Path
+    target_format: str
+    output_dir: Path
+    options: dict[str, Any] = field(default_factory=dict)
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    
+    @property
+    def source_extension(self) -> str:
+        """Get the source file extension."""
+        return self.source_path.suffix.lower()
+
+
+@dataclass
+class ConversionResult:
+    """Result of a conversion operation."""
+    task_id: str
+    status: ConversionStatus
+    output_path: Optional[Path] = None
+    error_message: Optional[str] = None
+    processing_time_ms: float = 0.0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class ConversionWorker:
+    """
+    Worker for processing document conversion tasks.
+    
+    Features:
+    - Async processing with semaphore-controlled concurrency
+    - Tool selection based on file type and capability
+    - Automatic cleanup of temporary files
+    - Progress callbacks
+    """
+
+    # File extension to required capability mapping
+    EXTENSION_CAPABILITIES = {
+        ".docx": ToolCapability.DOCX_TO_MD,
+        ".doc": ToolCapability.DOCX_TO_TEXT,
+        ".odt": ToolCapability.ODT_TO_TEXT,
+        ".pdf": ToolCapability.PDF_TO_TEXT,
+        ".xlsx": ToolCapability.XLSX_TO_CSV,
+        ".xls": ToolCapability.XLSX_TO_CSV,
+        ".pptx": ToolCapability.PPTX_TO_TEXT,
+        ".ppt": ToolCapability.PPTX_TO_TEXT,
+    }
+
+    def __init__(
+        self,
+        discovery: Optional[EnvironmentDiscovery] = None,
+        max_concurrent: int = 4,
+        timeout_seconds: float = 120.0,
+    ):
+        """
+        Initialize the conversion worker.
+        
+        Args:
+            discovery: EnvironmentDiscovery instance (creates new if None)
+            max_concurrent: Maximum concurrent conversion tasks
+            timeout_seconds: Timeout for individual conversions
+        """
+        self.discovery = discovery or EnvironmentDiscovery()
+        self.discovery.discover()
+        
+        self.max_concurrent = max_concurrent
+        self.timeout_seconds = timeout_seconds
+        self._semaphore = asyncio.Semaphore(max_concurrent)
+        self._active_tasks: dict[str, asyncio.Task] = {}
+        
+        # Callbacks
+        self._on_progress: Optional[Callable[[str, ConversionStatus, float], None]] = None
+        self._on_complete: Optional[Callable[[ConversionResult], None]] = None
+
+    def set_callbacks(
+        self,
+        on_progress: Optional[Callable[[str, ConversionStatus, float], None]] = None,
+        on_complete: Optional[Callable[[ConversionResult], None]] = None,
+    ) -> None:
+        """Set progress and completion callbacks."""
+        self._on_progress = on_progress
+        self._on_complete = on_complete
+
+    def _notify_progress(self, task_id: str, status: ConversionStatus, progress: float) -> None:
+        """Notify progress callback."""
+        if self._on_progress:
+            try:
+                self._on_progress(task_id, status, progress)
+            except Exception:
+                pass
+
+    def _notify_complete(self, result: ConversionResult) -> None:
+        """Notify completion callback."""
+        if self._on_complete:
+            try:
+                self._on_complete(result)
+            except Exception:
+                pass
+
+    def can_convert(self, file_path: Path) -> bool:
+        """
+        Check if a file can be converted.
+        
+        Args:
+            file_path: Path to the file to check
+            
+        Returns:
+            True if conversion is possible
+        """
+        ext = file_path.suffix.lower()
+        if ext not in self.EXTENSION_CAPABILITIES:
+            return False
+        
+        capability = self.EXTENSION_CAPABILITIES[ext]
+        return self.discovery.has_capability(capability)
+
+    def get_supported_extensions(self) -> list[str]:
+        """Get list of supported file extensions."""
+        supported = []
+        for ext, capability in self.EXTENSION_CAPABILITIES.items():
+            if self.discovery.has_capability(capability):
+                supported.append(ext)
+        return supported
+
+    async def submit(self, task: ConversionTask) -> ConversionResult:
+        """
+        Submit a conversion task for processing.
+        
+        Args:
+            task: The conversion task to process
+            
+        Returns:
+            ConversionResult with status and output details
+        """
+        async with self._semaphore:
+            return await self._process_task(task)
+
+    async def submit_batch(
+        self,
+        tasks: list[ConversionTask],
+    ) -> list[ConversionResult]:
+        """
+        Submit multiple tasks for concurrent processing.
+        
+        Args:
+            tasks: List of conversion tasks
+            
+        Returns:
+            List of ConversionResults (order may vary)
+        """
+        coroutines = [self.submit(task) for task in tasks]
+        return await asyncio.gather(*coroutines, return_exceptions=True)
+
+    async def _process_task(self, task: ConversionTask) -> ConversionResult:
+        """Process a single conversion task."""
+        import time
+        start_time = time.time()
+        
+        self._notify_progress(task.task_id, ConversionStatus.PROCESSING, 0.0)
+        
+        try:
+            # Validate source file
+            if not task.source_path.exists():
+                return self._create_error_result(
+                    task, "Source file does not exist"
+                )
+
+            # Get required capability
+            ext = task.source_extension
+            if ext not in self.EXTENSION_CAPABILITIES:
+                return self._create_error_result(
+                    task, f"Unsupported file extension: {ext}"
+                )
+
+            capability = self.EXTENSION_CAPABILITIES[ext]
+            tool = self.discovery.get_best_tool(capability)
+            
+            if not tool:
+                return self._create_error_result(
+                    task, f"No tool available for {capability.value}"
+                )
+
+            # Perform conversion
+            result = await self._convert_with_tool(task, tool, capability)
+            
+            processing_time = (time.time() - start_time) * 1000
+            result.processing_time_ms = processing_time
+            
+            self._notify_complete(result)
+            return result
+
+        except asyncio.TimeoutError:
+            result = self._create_error_result(task, "Conversion timeout")
+            self._notify_complete(result)
+            return result
+        except Exception as e:
+            logger.exception(f"Conversion failed for task {task.task_id}")
+            result = self._create_error_result(task, str(e))
+            self._notify_complete(result)
+            return result
+
+    def _create_error_result(self, task: ConversionTask, message: str) -> ConversionResult:
+        """Create a failed conversion result."""
+        return ConversionResult(
+            task_id=task.task_id,
+            status=ConversionStatus.FAILED,
+            error_message=message,
+        )
+
+    async def _convert_with_tool(
+        self,
+        task: ConversionTask,
+        tool: ConversionTool,
+        capability: ToolCapability,
+    ) -> ConversionResult:
+        """Convert using the specified tool."""
+        
+        if tool.tool_type.value == "libreoffice":
+            return await self._convert_with_libreoffice(task, tool)
+        elif tool.tool_type.value == "pandoc":
+            return await self._convert_with_pandoc(task, tool)
+        elif tool.tool_type.value in ("pdf2text", "pdftohtml"):
+            return await self._convert_with_pdf_tool(task, tool)
+        else:
+            return self._create_error_result(task, f"Unknown tool type: {tool.tool_type}")
+
+    async def _convert_with_libreoffice(
+        self,
+        task: ConversionTask,
+        tool: ConversionTool,
+    ) -> ConversionResult:
+        """Convert using LibreOffice."""
+        # Create temp directory for conversion
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmp_path = Path(tmpdir)
+            
+            # Build LibreOffice command
+            cmd = [
+                str(tool.executable_path),
+                "--headless",
+                "--convert-to", "txt:Text",
+                "--outdir", str(tmp_path),
+                str(task.source_path),
+            ]
+            
+            # Run conversion
+            try:
+                process = await asyncio.create_subprocess_exec(
+                    *cmd,
+                    stdout=asyncio.subprocess.PIPE,
+                    stderr=asyncio.subprocess.PIPE,
+                )
+                stdout, stderr = await asyncio.wait_for(
+                    process.communicate(),
+                    timeout=self.timeout_seconds,
+                )
+                
+                if process.returncode != 0:
+                    error_msg = stderr.decode() if stderr else "Unknown error"
+                    return self._create_error_result(task, f"LibreOffice error: {error_msg}")
+                
+                # Find output file
+                base_name = task.source_path.stem
+                output_file = tmp_path / f"{base_name}.txt"
+                
+                if not output_file.exists():
+                    return self._create_error_result(task, "Output file not created")
+                
+                # Move to final destination
+                task.output_dir.mkdir(parents=True, exist_ok=True)
+                final_output = task.output_dir / f"{base_name}.txt"
+                shutil.move(str(output_file), str(final_output))
+                
+                return ConversionResult(
+                    task_id=task.task_id,
+                    status=ConversionStatus.SUCCESS,
+                    output_path=final_output,
+                    metadata={
+                        "tool": tool.name,
+                        "tool_version": tool.version,
+                    },
+                )
+                
+            except asyncio.TimeoutError:
+                return self._create_error_result(task, "LibreOffice conversion timeout")
+
+    async def _convert_with_pandoc(
+        self,
+        task: ConversionTask,
+        tool: ConversionTool,
+    ) -> ConversionResult:
+        """Convert using Pandoc."""
+        task.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        base_name = task.source_path.stem
+        output_file = task.output_dir / f"{base_name}.md"
+        
+        cmd = [
+            str(tool.executable_path),
+            str(task.source_path),
+            "-o", str(output_file),
+            "-t", "markdown",
+        ]
+        
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            stdout, stderr = await asyncio.wait_for(
+                process.communicate(),
+                timeout=self.timeout_seconds,
+            )
+            
+            if process.returncode != 0:
+                error_msg = stderr.decode() if stderr else "Unknown error"
+                return self._create_error_result(task, f"Pandoc error: {error_msg}")
+            
+            return ConversionResult(
+                task_id=task.task_id,
+                status=ConversionStatus.SUCCESS,
+                output_path=output_file,
+                metadata={
+                    "tool": tool.name,
+                    "tool_version": tool.version,
+                },
+            )
+            
+        except asyncio.TimeoutError:
+            return self._create_error_result(task, "Pandoc conversion timeout")
+
+    async def _convert_with_pdf_tool(
+        self,
+        task: ConversionTask,
+        tool: ConversionTool,
+    ) -> ConversionResult:
+        """Convert PDF using pdftotext or similar."""
+        task.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        base_name = task.source_path.stem
+        output_file = task.output_dir / f"{base_name}.txt"
+        
+        cmd = [
+            str(tool.executable_path),
+            str(task.source_path),
+            str(output_file),
+        ]
+        
+        # Add layout preservation for pdftotext
+        if "pdftotext" in tool.name:
+            cmd.insert(1, "-layout")
+        
+        try:
+            process = await asyncio.create_subprocess_exec(
+                *cmd,
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            stdout, stderr = await asyncio.wait_for(
+                process.communicate(),
+                timeout=self.timeout_seconds,
+            )
+            
+            if process.returncode != 0:
+                error_msg = stderr.decode() if stderr else "Unknown error"
+                return self._create_error_result(task, f"PDF tool error: {error_msg}")
+            
+            return ConversionResult(
+                task_id=task.task_id,
+                status=ConversionStatus.SUCCESS,
+                output_path=output_file,
+                metadata={
+                    "tool": tool.name,
+                    "tool_version": tool.version,
+                },
+            )
+            
+        except asyncio.TimeoutError:
+            return self._create_error_result(task, "PDF conversion timeout")
+
+    async def cancel_task(self, task_id: str) -> bool:
+        """
+        Cancel an active task.
+        
+        Args:
+            task_id: ID of the task to cancel
+            
+        Returns:
+            True if cancelled, False if not found
+        """
+        if task_id in self._active_tasks:
+            task = self._active_tasks[task_id]
+            task.cancel()
+            return True
+        return False
+
+    def get_active_count(self) -> int:
+        """Get number of currently active tasks."""
+        return len(self._active_tasks)

monoco/core/injection.py

@@ -10,6 +10,8 @@ class PromptInjector:
     """
 
     MANAGED_HEADER = "## Monoco Toolkit"
+    MANAGED_START = "<!-- MONOCO_GENERATED_START -->"
+    MANAGED_END = "<!-- MONOCO_GENERATED_END -->"
 
     def __init__(self, target_file: Path):
         self.target_file = target_file
@@ -52,19 +54,40 @@ class PromptInjector:
             # Sanitize content: remove leading header if it matches the title
             clean_content = content.strip()
             # Regex to match optional leading hash header matching the title (case insensitive)
-            # e.g. "### Issue Management" or "# Issue Management"
             pattern = r"^(#+\s*)" + re.escape(title) + r"\s*\n"
             match = re.match(pattern, clean_content, re.IGNORECASE)
 
             if match:
                 clean_content = clean_content[match.end() :].strip()
-
-            managed_block.append(clean_content)
+            
+            # Demote headers in content to be below ### (so start at ####)
+            # We assume the content headers start at # or ##. 
+            # We map # -> ####, ## -> #####, etc. (+3 offset)
+            demoted_content = []
+            for line in clean_content.splitlines():
+                if line.lstrip().startswith("#"):
+                    demoted_content.append("###" + line)
+                else:
+                    demoted_content.append(line)
+            
+            managed_block.append("\n".join(demoted_content))
             managed_block.append("")  # Blank line after section
 
         managed_block_str = "\n".join(managed_block).strip() + "\n"
+        managed_block_str = f"{self.MANAGED_START}\n{managed_block_str}\n{self.MANAGED_END}\n"
 
         # 2. Find and replace/append in the original content
+        # Check for delimiters first
+        if self.MANAGED_START in original and self.MANAGED_END in original:
+            try:
+                pre = original.split(self.MANAGED_START)[0]
+                post = original.split(self.MANAGED_END)[1]
+                # Reconstruct
+                return pre + managed_block_str.strip() + post
+            except IndexError:
+                # Fallback to header detection if delimiters malformed
+                pass
+
         lines = original.splitlines()
         start_idx = -1
         end_idx = -1
@@ -74,31 +97,29 @@ class PromptInjector:
             if line.strip() == self.MANAGED_HEADER:
                 start_idx = i
                 break
+        
+        if start_idx == -1:
+             # Check if we have delimiters even if header is missing/changed?
+             # Handled above.
+             pass
 
         if start_idx == -1:
             # Block not found, append to end
             if original and not original.endswith("\n"):
-                return original + "\n\n" + managed_block_str
+                return original + "\n\n" + managed_block_str.strip()
             elif original:
-                return original + "\n" + managed_block_str
+                return original + "\n" + managed_block_str.strip()
             else:
-                return managed_block_str
-
-        # Find end: Look for next header of level 1 (assuming Managed Header is H1)
-        # Or EOF
-        # Note: If MANAGED_HEADER is "# ...", we look for next "# ..."
-        # But allow "## ..." as children.
+                return managed_block_str.strip() + "\n"
 
+        # Find end: Look for next header of level 1 or 2 (siblings or parents)
         header_level_match = re.match(r"^(#+)\s", self.MANAGED_HEADER)
-        header_level_prefix = header_level_match.group(1) if header_level_match else "#"
+        header_level_prefix = header_level_match.group(1) if header_level_match else "##"
 
         for i in range(start_idx + 1, len(lines)):
             line = lines[i]
             # Check if this line is a header of the same level or higher (fewer #s)
-            # e.g. if Managed is "###", then "#" and "##" are higher/parents, "###" is sibling.
-            # We treat siblings as end of block too.
             if line.startswith("#"):
-                # Match regex to get level
                 match = re.match(r"^(#+)\s", line)
                 if match:
                     level = match.group(1)
@@ -146,26 +167,39 @@ class PromptInjector:
 
         # Find start
         for i, line in enumerate(lines):
-            if line.strip() == self.MANAGED_HEADER:
+            if self.MANAGED_START in line:
                 start_idx = i
+                # Look for end from here
+                for j in range(i, len(lines)):
+                    if self.MANAGED_END in lines[j]:
+                        end_idx = j + 1 # Include the end line
+                        break
                 break
+        
+        if start_idx == -1:
+            # Fallback to header logic
+             for i, line in enumerate(lines):
+                if line.strip() == self.MANAGED_HEADER:
+                    start_idx = i
+                    break
 
         if start_idx == -1:
             return False
 
-        # Find end: exact logic as in _merge_content
-        header_level_match = re.match(r"^(#+)\s", self.MANAGED_HEADER)
-        header_level_prefix = header_level_match.group(1) if header_level_match else "#"
-
-        for i in range(start_idx + 1, len(lines)):
-            line = lines[i]
-            if line.startswith("#"):
-                match = re.match(r"^(#+)\s", line)
-                if match:
-                    level = match.group(1)
-                    if len(level) <= len(header_level_prefix):
-                        end_idx = i
-                        break
+        if end_idx == -1:
+            # Find end: exact logic as in _merge_content
+            header_level_match = re.match(r"^(#+)\s", self.MANAGED_HEADER)
+            header_level_prefix = header_level_match.group(1) if header_level_match else "##"
+
+            for i in range(start_idx + 1, len(lines)):
+                line = lines[i]
+                if line.startswith("#"):
+                    match = re.match(r"^(#+)\s", line)
+                    if match:
+                        level = match.group(1)
+                        if len(level) <= len(header_level_prefix):
+                            end_idx = i
+                            break
 
         if end_idx == -1:
             end_idx = len(lines)

monoco/core/integrations.py

@@ -129,8 +129,8 @@ DEFAULT_INTEGRATIONS: Dict[str, AgentIntegration] = {
     "kimi": AgentIntegration(
         key="kimi",
         name="Kimi CLI",
-        system_prompt_file="KIMI.md",
-        skill_root_dir=".kimi/skills/",
+        system_prompt_file="AGENTS.md",
+        skill_root_dir=".agent/skills/",
         bin_name="kimi",
         version_cmd="--version",
     ),