mdify-cli 2.11.6__tar.gz → 2.11.8__tar.gz

{mdify_cli-2.11.6/mdify_cli.egg-info → mdify_cli-2.11.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mdify-cli
-Version: 2.11.6
+Version: 2.11.8
 Summary: Convert PDFs and document images into structured Markdown for LLM workflows
 Author: tiroq
 License-Expression: MIT

{mdify_cli-2.11.6 → mdify_cli-2.11.8}/mdify/__init__.py

@@ -1,3 +1,3 @@
 """mdify - Convert documents to Markdown via Docling container."""
 
-__version__ = "2.11.6"
+__version__ = "2.11.8"

{mdify_cli-2.11.6 → mdify_cli-2.11.8}/mdify/cli.py

@@ -41,6 +41,108 @@ OTHER_RUNTIMES_PRIORITY = ("docker", "podman")
 # Debug mode
 DEBUG = os.environ.get("MDIFY_DEBUG", "").lower() in ("1", "true", "yes")
 
+# Resource profiles for container execution
+RESOURCE_PROFILES = {
+    "minimal": {"cpus": 4, "memory": "8g", "description": "Small PDFs, text-only documents"},
+    "default": {"cpus": 6, "memory": "12g", "description": "Large PDFs, OCR, tables (recommended)"},
+    "heavy": {"cpus": 8, "memory": "16g", "description": "Batch processing, very large files"},
+}
+
+
+def get_available_memory_gb() -> float:
+    """Get available system memory in GB.
+    
+    Returns:
+        Available memory in GB, or -1 if unable to determine
+    """
+    system = platform.system()
+    try:
+        if system == "Darwin":  # macOS
+            # Get page size
+            result = subprocess.run(["pagesize"], capture_output=True, text=True, check=True)
+            page_size = int(result.stdout.strip())
+            
+            # Get memory stats
+            result = subprocess.run(["vm_stat"], capture_output=True, text=True, check=True)
+            free_pages = 0
+            inactive_pages = 0
+            speculative_pages = 0
+            
+            for line in result.stdout.split("\n"):
+                if "Pages free" in line:
+                    free_pages = int(line.split(":")[1].strip().rstrip("."))
+                elif "Pages inactive" in line:
+                    inactive_pages = int(line.split(":")[1].strip().rstrip("."))
+                elif "Pages speculative" in line:
+                    speculative_pages = int(line.split(":")[1].strip().rstrip("."))
+            
+            # Available memory = free + inactive + speculative
+            available_pages = free_pages + inactive_pages + speculative_pages
+            available_bytes = available_pages * page_size
+            return available_bytes / (1024**3)  # Convert to GB
+        elif system == "Linux":
+            # Read from /proc/meminfo
+            with open("/proc/meminfo") as f:
+                for line in f:
+                    if line.startswith("MemAvailable:"):
+                        kb = int(line.split()[1])
+                        return kb / (1024**2)  # Convert to GB
+    except Exception:
+        pass
+    
+    return -1
+
+
+def parse_memory_string(mem_str: str) -> float:
+    """Parse memory string (e.g., '12g', '8192m') to GB.
+    
+    Args:
+        mem_str: Memory string with unit (g, m, gb, mb)
+        
+    Returns:
+        Memory in GB
+    """
+    mem_str = mem_str.lower().strip()
+    
+    if mem_str.endswith("gb"):
+        return float(mem_str[:-2])
+    elif mem_str.endswith("g"):
+        return float(mem_str[:-1])
+    elif mem_str.endswith("mb"):
+        return float(mem_str[:-2]) / 1024
+    elif mem_str.endswith("m"):
+        return float(mem_str[:-1]) / 1024
+    else:
+        raise ValueError(f"Invalid memory format: {mem_str}")
+
+
+def validate_memory_availability(required_gb: float) -> tuple[bool, str]:
+    """Check if system has sufficient available memory.
+    
+    Args:
+        required_gb: Required memory in GB
+        
+    Returns:
+        Tuple of (is_sufficient, error_message)
+    """
+    available_gb = get_available_memory_gb()
+    
+    if available_gb < 0:
+        # Unable to determine, allow startup with warning
+        return True, ""
+    
+    if available_gb < required_gb:
+        error = (
+            f"Insufficient memory available for container startup.\n"
+            f"  Required: {required_gb:.1f} GB\n"
+            f"  Available: {available_gb:.1f} GB\n"
+            f"  Short by: {required_gb - available_gb:.1f} GB\n\n"
+            f"Please close other applications or use a smaller profile (--profile minimal)"
+        )
+        return False, error
+    
+    return True, ""
+
 
 # =============================================================================
 # Update checking functions
@@ -752,6 +854,34 @@ Examples:
         help="Conversion timeout in seconds (default: 1200, can be set via MDIFY_TIMEOUT env var)",
     )
 
+    parser.add_argument(
+        "--memory",
+        type=str,
+        default=None,
+        help="Container memory limit (e.g., 2g, 512m, 4096m). Overrides --profile setting",
+    )
+
+    parser.add_argument(
+        "--cpus",
+        type=int,
+        default=None,
+        help="Number of CPUs to allocate to container. Overrides --profile setting",
+    )
+
+    parser.add_argument(
+        "--profile",
+        type=str,
+        choices=["minimal", "default", "heavy"],
+        default="default",
+        help="Resource profile for container: minimal (4 CPU, 8GB), default (6 CPU, 12GB), heavy (8 CPU, 16GB)",
+    )
+
+    parser.add_argument(
+        "--skip-memory-check",
+        action="store_true",
+        help="Skip memory availability validation (not recommended)",
+    )
+
     # Utility options
     parser.add_argument(
         "--check-update",
@@ -952,7 +1082,25 @@ def main() -> int:
 
     try:
         if not args.quiet:
-            print(f"Starting docling-serve container...")
+            print(f"Starting docling-serve container...\\n")
+
+        # Apply resource profile
+        profile = RESOURCE_PROFILES[args.profile]
+        cpus = args.cpus if args.cpus is not None else profile["cpus"]
+        memory = args.memory if args.memory is not None else profile["memory"]
+        
+        # Validate memory availability unless skipped
+        if not args.skip_memory_check:
+            required_gb = parse_memory_string(memory)
+            is_sufficient, error_msg = validate_memory_availability(required_gb)
+            if not is_sufficient:
+                print(f"Error: {error_msg}", file=sys.stderr)
+                return 1
+        
+        if not args.quiet:
+            print(f"Resource profile: {args.profile} ({cpus} CPUs, {memory} memory)")
+            if args.cpus or args.memory:
+                print("  (customized via command-line arguments)")
             print()
 
         with DoclingContainer(
@@ -961,6 +1109,8 @@ def main() -> int:
             args.port,
             timeout=timeout,
             keep_container=DEBUG,
+            memory=memory,
+            cpus=cpus,
         ) as container:
             # Convert files
             conversion_start = time.time()
@@ -1020,6 +1170,60 @@ def main() -> int:
                                 f"{progress} {input_file.name} ✗ ({format_duration(elapsed)})"
                             )
                             print(f"    Error: {error_msg}", file=sys.stderr)
+                            
+                            # Check if it's a connection error and retrieve logs
+                            is_connection_error = "Connection refused" in error_msg or "Connection aborted" in error_msg or "RemoteDisconnected" in error_msg
+                            if is_connection_error:
+                                container_alive = container.is_ready()
+                                if container_alive:
+                                    print(
+                                        "    Connection lost (server may have crashed and restarted)",
+                                        file=sys.stderr,
+                                    )
+                                else:
+                                    print(
+                                        "    Container crashed while processing file",
+                                        file=sys.stderr,
+                                    )
+                                    print(
+                                        "    File may be too complex, large, or malformed",
+                                        file=sys.stderr,
+                                    )
+                                
+                                # Always show logs for connection errors
+                                print("    Retrieving container logs...", file=sys.stderr)
+                                logs, log_error = container.get_logs(tail=50)
+                                if logs:
+                                    print("    Container logs (last 50 lines):", file=sys.stderr)
+                                    for line in logs.strip().split("\n"):
+                                        if line.strip():
+                                            print(f"      {line}", file=sys.stderr)
+                                elif log_error:
+                                    print(f"    Error retrieving logs: {log_error}", file=sys.stderr)
+                                else:
+                                    print("    No logs available (container may have been removed)", file=sys.stderr)
+                                
+                                # Restart container if it crashed
+                                if not container_alive:
+                                    print("    Container crashed - attempting to restart...", file=sys.stderr)
+                                    try:
+                                        # Stop and remove the dead container
+                                        container.stop()
+                                        container.remove()
+                                        # Generate new container name to avoid conflicts
+                                        import uuid
+                                        container.container_name = f"mdify-serve-{uuid.uuid4().hex[:8]}"
+                                        # Start a new one
+                                        container.start(timeout=120)
+                                        print("    Container restarted successfully", file=sys.stderr)
+                                        print("    Continuing with next file...", file=sys.stderr)
+                                    except Exception as restart_error:
+                                        print(f"    Failed to restart container: {restart_error}", file=sys.stderr)
+                                        if DEBUG:
+                                            import traceback
+                                            traceback.print_exc()
+                                        print("    Stopping remaining conversions", file=sys.stderr)
+                                        break
                 except Exception as e:
                     elapsed = time.time() - start_time
                     failed_count += 1
@@ -1031,6 +1235,10 @@ def main() -> int:
                     error_msg = str(e)
                     is_connection_error = "Connection refused" in error_msg or "Connection aborted" in error_msg or "RemoteDisconnected" in error_msg
                     
+                    if DEBUG:
+                        print(f"    DEBUG: Exception caught: {type(e).__name__}", file=sys.stderr)
+                        print(f"    DEBUG: is_connection_error={is_connection_error}", file=sys.stderr)
+                    
                     if is_connection_error:
                         container_alive = container.is_ready()
                         if not args.quiet:
@@ -1078,9 +1286,27 @@ def main() -> int:
                             if not container_alive:
                                 print("    Stopping remaining conversions", file=sys.stderr)
 
-                        # Stop processing if container is dead
+                        # Restart container if it crashed
                         if not container_alive:
-                            break
+                            print("    Container crashed - attempting to restart...", file=sys.stderr)
+                            try:
+                                # Stop and remove the dead container
+                                container.stop()
+                                container.remove()
+                                # Generate new container name to avoid conflicts
+                                import uuid
+                                container.container_name = f"mdify-serve-{uuid.uuid4().hex[:8]}"
+                                # Start a new one
+                                container.start(timeout=120)
+                                print("    Container restarted successfully", file=sys.stderr)
+                                print("    Continuing with next file...", file=sys.stderr)
+                            except Exception as restart_error:
+                                print(f"    Failed to restart container: {restart_error}", file=sys.stderr)
+                                if DEBUG:
+                                    import traceback
+                                    traceback.print_exc()
+                                print("    Stopping remaining conversions", file=sys.stderr)
+                                break
                     else:
                         # Non-connection error
                         if not args.quiet:

{mdify_cli-2.11.6 → mdify_cli-2.11.8}/mdify/container.py

@@ -27,6 +27,8 @@ class DoclingContainer:
         port: int = 5001,
         timeout: int = 1200,
         keep_container: bool = False,
+        memory: Optional[str] = None,
+        cpus: Optional[int] = None,
     ):
         """Initialize container manager.
 
@@ -36,12 +38,16 @@ class DoclingContainer:
             port: Host port to bind (default: 5001)
             timeout: Conversion timeout in seconds (default: 1200)
             keep_container: If True, do not auto-remove container (preserve logs)
+            memory: Memory limit (e.g., "2g", "512m"). None for no limit.
+            cpus: Number of CPUs to allocate. None for no limit.
         """
         self.runtime = runtime
         self.image = image
         self.port = port
         self.timeout = timeout
         self.keep_container = keep_container
+        self.memory = memory
+        self.cpus = cpus
         self.container_name = f"mdify-serve-{uuid.uuid4().hex[:8]}"
         self.container_id: Optional[str] = None
 
@@ -110,6 +116,15 @@ class DoclingContainer:
         ]
         if not self.keep_container:
             cmd.insert(3, "--rm")  # Auto-remove on stop
+        
+        # Add resource limits if specified
+        if self.cpus:
+            cmd.insert(3, str(self.cpus))
+            cmd.insert(3, "--cpus")
+        
+        if self.memory:
+            cmd.insert(3, self.memory)
+            cmd.insert(3, "-m")
 
         try:
             result = subprocess.run(cmd, capture_output=True, text=True, check=True)
@@ -135,6 +150,15 @@ class DoclingContainer:
                 check=False,
             )
 
+    def remove(self) -> None:
+        """Remove container. Safe to call multiple times."""
+        if self.container_name:
+            subprocess.run(
+                [self.runtime, "rm", "-f", self.container_name],
+                capture_output=True,
+                check=False,
+            )
+
     def get_logs(self, tail: int = 50) -> tuple[str, str]:
         """Get container logs for debugging.
 
@@ -148,8 +172,17 @@ class DoclingContainer:
             return ("", "No container name set")
         
         try:
+            import os
+            runtime_name = os.path.basename(self.runtime)
+            
+            # Apple Container uses -n instead of --tail
+            if runtime_name == "container":
+                cmd = [self.runtime, "logs", "-n", str(tail), self.container_name]
+            else:
+                cmd = [self.runtime, "logs", "--tail", str(tail), self.container_name]
+            
             result = subprocess.run(
-                [self.runtime, "logs", "--tail", str(tail), self.container_name],
+                cmd,
                 capture_output=True,
                 text=True,
                 check=False,

{mdify_cli-2.11.6 → mdify_cli-2.11.8/mdify_cli.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mdify-cli
-Version: 2.11.6
+Version: 2.11.8
 Summary: Convert PDFs and document images into structured Markdown for LLM workflows
 Author: tiroq
 License-Expression: MIT

{mdify_cli-2.11.6 → mdify_cli-2.11.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mdify-cli"
-version = "2.11.6"
+version = "2.11.8"
 description = "Convert PDFs and document images into structured Markdown for LLM workflows"
 readme = "README.md"
 requires-python = ">=3.8"