mdify-cli 2.11.9__py3-none-any.whl → 3.0.0__py3-none-any.whl

mdify/ssh/transfer.py

@@ -0,0 +1,297 @@
+"""File transfer and progress tracking for SSH."""
+
+import gzip
+import hashlib
+import logging
+from pathlib import Path
+from typing import Callable
+from mdify.ssh.models import TransferSession
+
+logger = logging.getLogger(__name__)
+
+
+class FileTransferManager:
+    """Manages file transfers with compression and progress tracking."""
+    
+    def __init__(
+        self,
+        ssh_client,
+        compression_threshold: int = 1024 * 1024,  # 1MB
+        chunk_size: int = 64 * 1024,  # 64KB
+        verify_checksum: bool = True
+    ):
+        """Initialize file transfer manager.
+        
+        Parameters:
+            ssh_client: Connected AsyncSSHClient instance
+            compression_threshold: Compress files larger than this (bytes)
+            chunk_size: Read/write chunk size for progress updates
+            verify_checksum: Calculate SHA256 checksums
+        """
+        self.ssh_client = ssh_client
+        self.compression_threshold = compression_threshold
+        self.chunk_size = chunk_size
+        self.verify_checksum = verify_checksum
+    
+    async def upload_file(
+        self,
+        local_path: str,
+        remote_path: str,
+        progress_callback: Callable[[TransferSession], None] | None = None,
+        overwrite: bool = False,
+        compress: bool | None = None
+    ) -> TransferSession:
+        """Upload file to remote server.
+        
+        Parameters:
+            local_path: Local file path
+            remote_path: Remote destination path
+            progress_callback: Called with TransferSession after each chunk
+            overwrite: Allow overwriting existing files
+            compress: Force compression (None = auto-detect by size)
+            
+        Returns:
+            TransferSession with transfer results
+            
+        Raises:
+            FileNotFoundError: Local file doesn't exist
+            FileExistsError: Remote file exists and overwrite=False
+        """
+        local_file = Path(local_path)
+        if not local_file.exists():
+            raise FileNotFoundError(f"Local file not found: {local_path}")
+        
+        file_size = local_file.stat().st_size
+        
+        # Auto-detect compression
+        if compress is None:
+            compress = file_size > self.compression_threshold
+        
+        session = TransferSession(
+            local_path=local_path,
+            remote_path=remote_path,
+            direction="upload",
+            total_bytes=file_size,
+            status="in_progress"
+        )
+        
+        try:
+            # Check if remote file exists
+            if not overwrite:
+                stdout, stderr, code = await self.ssh_client.run_command(
+                    f"test -f {remote_path}"
+                )
+                if code == 0:
+                    raise FileExistsError(f"Remote file exists: {remote_path}")
+            
+            # Prepare target path (content is streamed directly from local_file)
+            if compress:
+                logger.debug(f"Compressing {local_file.name} for upload...")
+                session.status = "in_progress"
+                await self._compress_file(local_file)
+                actual_remote_path = f"{remote_path}.gz"
+                session.status = "in_progress"
+            else:
+                actual_remote_path = remote_path
+            
+            # Upload via SFTP
+            async with self.ssh_client.connection.start_sftp_client() as sftp:
+                # Write file with progress tracking
+                bytes_written = 0
+                with open(local_file, 'rb') as local_fp:
+                    async with await sftp.open(actual_remote_path, 'wb') as remote_fp:
+                        while True:
+                            chunk = local_fp.read(self.chunk_size)
+                            if not chunk:
+                                break
+                            
+                            await remote_fp.write(chunk)
+                            bytes_written += len(chunk)
+                            session.update_progress(bytes_written)
+                            
+                            if progress_callback:
+                                progress_callback(session)
+            
+            # Verify checksum if enabled
+            if self.verify_checksum:
+                await self._verify_upload_checksum(
+                    local_file, actual_remote_path, session
+                )
+            
+            session.complete()
+            logger.info(f"Upload complete: {local_path} → {actual_remote_path}")
+            return session
+            
+        except Exception as e:
+            session.fail(e)
+            logger.error(f"Upload failed: {e}")
+            raise
+    
+    async def download_file(
+        self,
+        remote_path: str,
+        local_path: str,
+        progress_callback: Callable[[TransferSession], None] | None = None,
+        overwrite: bool = False
+    ) -> TransferSession:
+        """Download file from remote server.
+        
+        Parameters:
+            remote_path: Remote file path
+            local_path: Local destination path
+            progress_callback: Called with TransferSession after each chunk
+            overwrite: Allow overwriting existing files
+            
+        Returns:
+            TransferSession with transfer results
+        """
+        local_file = Path(local_path)
+        
+        # Check if local file exists
+        if local_file.exists() and not overwrite:
+            raise FileExistsError(f"Local file exists: {local_path}")
+        
+        session = TransferSession(
+            remote_path=remote_path,
+            local_path=local_path,
+            direction="download",
+            status="in_progress"
+        )
+        
+        try:
+            # Get remote file size
+            stdout, stderr, code = await self.ssh_client.run_command(
+                f"wc -c < {remote_path}"
+            )
+            if code == 0:
+                try:
+                    session.total_bytes = int(stdout.strip())
+                except ValueError:
+                    session.total_bytes = 0
+            
+            # Download via SFTP
+            async with self.ssh_client.connection.start_sftp_client() as sftp:
+                bytes_read = 0
+                with open(local_file, 'wb') as local_fp:
+                    async with await sftp.open(remote_path, 'rb') as remote_fp:
+                        while True:
+                            chunk = await remote_fp.read(self.chunk_size)
+                            if not chunk:
+                                break
+                            
+                            local_fp.write(chunk)
+                            bytes_read += len(chunk)
+                            session.update_progress(bytes_read)
+                            
+                            if progress_callback:
+                                progress_callback(session)
+            
+            session.complete()
+            logger.info(f"Download complete: {remote_path} → {local_path}")
+            return session
+            
+        except Exception as e:
+            session.fail(e)
+            logger.error(f"Download failed: {e}")
+            raise
+    
+    async def _compress_file(self, file_path: Path) -> bytes:
+        """Compress file for transfer.
+        
+        Parameters:
+            file_path: Path to file to compress
+            
+        Returns:
+            Compressed file data
+        """
+        with open(file_path, 'rb') as f_in:
+            compressed = gzip.compress(f_in.read())
+        return compressed
+    
+    async def _verify_upload_checksum(
+        self,
+        local_file: Path,
+        remote_path: str,
+        session: TransferSession
+    ) -> None:
+        """Verify uploaded file checksum.
+        
+        Parameters:
+            local_file: Local file path
+            remote_path: Remote file path
+            session: Transfer session for error reporting
+            
+        Raises:
+            ValueError: Checksum mismatch
+        """
+        # Calculate local checksum
+        local_sha256 = hashlib.sha256()
+        with open(local_file, 'rb') as f:
+            for chunk in iter(lambda: f.read(self.chunk_size), b''):
+                local_sha256.update(chunk)
+        local_checksum = local_sha256.hexdigest()
+        
+        # Calculate remote checksum
+        stdout, stderr, code = await self.ssh_client.run_command(
+            f"sha256sum {remote_path} | awk '{{print $1}}'"
+        )
+        
+        if code == 0:
+            remote_checksum = stdout.strip()
+            
+            if local_checksum != remote_checksum:
+                raise ValueError(
+                    f"Checksum mismatch: {local_checksum} != {remote_checksum}"
+                )
+            
+            logger.debug(f"Checksum verified: {local_checksum}")
+        else:
+            logger.warning(f"Could not verify checksum: {stderr}")
+
+
+class ProgressBar:
+    """Simple progress bar display."""
+    
+    def __init__(self, total_bytes: int, debug_mode: bool = False):
+        """Initialize progress bar.
+        
+        Parameters:
+            total_bytes: Total bytes to transfer
+            debug_mode: Enable detailed logging
+        """
+        self.total_bytes = total_bytes
+        self.debug_mode = debug_mode
+        self.last_update = 0
+    
+    def update(self, session: TransferSession) -> str:
+        """Format progress display.
+        
+        Parameters:
+            session: TransferSession with current progress
+            
+        Returns:
+            Formatted progress string
+        """
+        if session.total_bytes == 0:
+            return ""
+        
+        percent = 100 * session.transferred_bytes / session.total_bytes
+        bar_width = 40
+        filled = int(bar_width * session.transferred_bytes / session.total_bytes)
+        bar = "█" * filled + "░" * (bar_width - filled)
+        
+        speed_str = f"{session.avg_speed_mbps:.1f}MB/s"
+        eta_str = f"ETA {session.eta_seconds}s" if session.eta_seconds else "Computing..."
+        
+        return f"[{bar}] {percent:3.0f}% {speed_str} {eta_str}"
+    
+    def log_chunk(self, chunk_num: int, bytes_transferred: int, speed_mbps: float) -> None:
+        """Log chunk transfer (debug mode).
+        
+        Parameters:
+            chunk_num: Chunk number
+            bytes_transferred: Total bytes transferred so far
+            speed_mbps: Current speed in MB/s
+        """
+        if self.debug_mode:
+            logger.debug(f"Chunk {chunk_num}: {bytes_transferred} bytes ({speed_mbps:.1f}MB/s)")

{mdify_cli-2.11.9.dist-info → mdify_cli-3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mdify-cli
-Version: 2.11.9
+Version: 3.0.0
 Summary: Convert PDFs and document images into structured Markdown for LLM workflows
 Author: tiroq
 License-Expression: MIT
@@ -14,17 +14,17 @@ Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: End Users/Desktop
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Topic :: Text Processing :: Markup :: Markdown
 Classifier: Topic :: Utilities
-Requires-Python: >=3.8
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: requests
+Requires-Dist: asyncssh>=2.10.0
+Requires-Dist: pyyaml>=6.0
 Provides-Extra: dev
 Requires-Dist: pytest>=7.0; extra == "dev"
 Dynamic: license-file
@@ -120,6 +120,72 @@ mdify --gpu documents/*.pdf
 
 Requires NVIDIA GPU with CUDA support and nvidia-container-toolkit.
 
+### 🚀 Remote Server Execution (SSH)
+
+**NEW:** Convert documents on remote servers via SSH to offload resource-intensive processing:
+
+```bash
+# Basic remote conversion
+mdify document.pdf --remote-host server.example.com
+
+# Use SSH config alias
+mdify document.pdf --remote-host production
+
+# With custom configuration
+mdify docs/*.pdf --remote-host 192.168.1.100 \
+  --remote-user admin \
+  --remote-key ~/.ssh/id_rsa
+
+# Validate remote server before processing
+mdify document.pdf --remote-host server --remote-validate-only
+```
+
+**How it works:**
+1. Connects to remote server via SSH
+2. Validates remote resources (disk space, memory, Docker/Podman)
+3. Uploads files via SFTP
+4. Starts remote container automatically
+5. Converts documents on remote server
+6. Downloads results via SFTP
+7. Cleans up remote files and stops container
+
+**Requirements:**
+- SSH key authentication (password auth not supported for security)
+- Docker or Podman installed on remote server
+- Minimum 5GB disk space and 2GB RAM on remote
+
+**SSH Configuration:**
+
+Create `~/.mdify/remote.conf` for reusable settings:
+```yaml
+host: production.example.com
+port: 22
+username: deploy
+key_file: ~/.ssh/deploy_key
+work_dir: /tmp/mdify-remote
+container_runtime: docker
+timeout: 30
+```
+
+Or use existing `~/.ssh/config`:
+```
+Host production
+  HostName 192.168.1.100
+  User deploy
+  Port 2222
+  IdentityFile ~/.ssh/deploy_key
+```
+
+Then simply: `mdify doc.pdf --remote-host production`
+
+**Configuration Precedence** (highest to lowest):
+1. CLI arguments (`--remote-*`)
+2. `~/.mdify/remote.conf`
+3. `~/.ssh/config`
+4. Built-in defaults
+
+See the [SSH Remote Server Guide](#ssh-remote-server-options) below for all options.
+
 ### ⚠️ PII Masking (Deprecated)
 
 The `--mask` flag is deprecated and will be ignored in this version. PII masking functionality was available in older versions using a custom runtime but is not supported with the current docling-serve backend.
@@ -158,6 +224,24 @@ The first conversion takes longer (~30-60s) as the container loads ML models int
 | `--check-update` | Check for available updates and exit |
 | `--version` | Show version and exit |
 
+### SSH Remote Server Options
+
+| Option | Description |
+| ------ | ----------- |
+| `--remote-host HOST` | SSH hostname or IP (required for remote mode) |
+| `--remote-port PORT` | SSH port (default: 22) |
+| `--remote-user USER` | SSH username (uses ~/.ssh/config or current user) |
+| `--remote-key PATH` | SSH private key file path |
+| `--remote-key-passphrase PASS` | SSH key passphrase |
+| `--remote-timeout SEC` | SSH connection timeout in seconds (default: 30) |
+| `--remote-work-dir DIR` | Remote working directory (default: /tmp/mdify-remote) |
+| `--remote-runtime RT` | Remote container runtime: docker or podman (auto-detected) |
+| `--remote-config PATH` | Path to mdify remote config file (default: ~/.mdify/remote.conf) |
+| `--remote-skip-ssh-config` | Don't load settings from ~/.ssh/config |
+| `--remote-skip-validation` | Skip remote resource validation (not recommended) |
+| `--remote-validate-only` | Validate remote server and exit (dry run) |
+| `--remote-debug` | Enable detailed SSH debug logging |
+
 ### Container Runtime Selection
 
 mdify automatically detects and uses the best available container runtime. The detection order differs by platform:
@@ -292,6 +376,110 @@ Or if installed via pip:
 pip uninstall mdify-cli
 ```
 
+## Troubleshooting
+
+### SSH Remote Server Issues
+
+**Connection Refused**
+
+```
+Error: SSH connection failed: Connection refused (host:22)
+```
+
+- Verify SSH server is running on remote: `ssh user@host`
+- Check firewall allows port 22 (or custom SSH port)
+- Verify hostname/IP is correct
+
+**Authentication Failed**
+
+```
+Error: SSH authentication failed
+```
+
+- Use SSH key authentication (password auth not supported)
+- Verify key file exists: `ls -l ~/.ssh/id_rsa`
+- Check key permissions: `chmod 600 ~/.ssh/id_rsa`
+- Test SSH manually: `ssh -i ~/.ssh/id_rsa user@host`
+- Add key to ssh-agent: `ssh-add ~/.ssh/id_rsa`
+
+**Remote Container Runtime Not Found**
+
+```
+Error: Container runtime not available: docker/podman
+```
+
+- Install Docker on remote: `sudo apt install docker.io` (Ubuntu/Debian)
+- Or install Podman: `sudo dnf install podman` (Fedora/RHEL)
+- Add user to docker group: `sudo usermod -aG docker $USER`
+- Verify remote Docker running: `ssh user@host docker ps`
+
+**Insufficient Remote Resources**
+
+```
+Warning: Less than 5GB available on remote
+```
+
+- Free up disk space on remote server
+- Use `--remote-work-dir` to specify different partition
+- Use `--remote-skip-validation` to bypass check (not recommended)
+
+**File Transfer Timeout**
+
+```
+Error: File transfer timeout
+```
+
+- Increase timeout: `--remote-timeout 120`
+- Check network bandwidth and stability
+- Try smaller files first to verify connection
+
+**Container Health Check Fails**
+
+```
+Error: Container failed to become healthy within 60 seconds
+```
+
+- Check remote Docker logs: `ssh user@host docker logs mdify-remote-<id>`
+- Verify port 5001 not in use: `ssh user@host netstat -tuln | grep 5001`
+- Try different port: `--port 5002`
+
+**SSH Config Not Loaded**
+
+If using SSH config alias but getting connection errors:
+
+```bash
+# Verify SSH config is valid
+cat ~/.ssh/config
+
+# Test SSH config works
+ssh your-alias
+
+# Use explicit connection if needed
+mdify doc.pdf --remote-host 192.168.1.100 --remote-user admin
+```
+
+**Permission Denied on Remote**
+
+```
+Error: Work directory not writable: /tmp/mdify-remote
+```
+
+- SSH to remote and check permissions: `ssh user@host ls -ld /tmp`
+- Use directory in your home: `--remote-work-dir ~/mdify-temp`
+- Fix permissions: `ssh user@host chmod 777 /tmp/mdify-remote`
+
+**Debug Mode**
+
+Enable detailed logging for troubleshooting:
+
+```bash
+# Debug SSH operations
+mdify doc.pdf --remote-host server --remote-debug
+
+# Debug local operations
+MDIFY_DEBUG=1 mdify doc.pdf
+```
+
 ## Development
 
 ### Task automation

mdify_cli-3.0.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+assets/mdify.png,sha256=qUj7WXWqNwpI2KNXOW79XJwqFqa-UI0JEkmt1mmy4Rg,1820418
+mdify/__init__.py,sha256=kKpiV423cq23fbsR9zhi3l8ZpZmWGRJsjyMG9G5v670,90
+mdify/__main__.py,sha256=bhpJ00co6MfaVOdH4XLoW04NtLYDa_oJK7ODzfLrn9M,143
+mdify/cli.py,sha256=xU_HWrOVYB67GRGz77l2ToLWlh2OM1FPgZjZDLT3MVk,73705
+mdify/container.py,sha256=BjL5ZR__n1i_WHifXKllTPoqO7IuOUdPDo5esuNg0Iw,8213
+mdify/docling_client.py,sha256=xuQR6sC1v3EPloOSwExoHCqT4uUxE8myYq-Yeby3C2I,7975
+mdify/ssh/__init__.py,sha256=SmRWgwEvAQZ_ARHlKTb9QDPwVAcz6dvPUks2pZFWLAU,271
+mdify/ssh/client.py,sha256=MNMBrL5Xk2rFo28Ytw80hWX2vQ3_CXlIL4VathNtK-I,14873
+mdify/ssh/models.py,sha256=IGAf5EfpZuBS2lIGzxmIsl8f44bXg4a8wk4BW9JWKEQ,17275
+mdify/ssh/remote_container.py,sha256=kmScAlmHI9rJLKliYcYQXDZHF7PYYiD-_rRV-S0fffM,8462
+mdify/ssh/transfer.py,sha256=aZZgylDjoqx6PEpaMu2zxkDF04w7btiOnMExmtt922A,10574
+mdify_cli-3.0.0.dist-info/licenses/LICENSE,sha256=NWM66Uv-XuSMKaU-gaPmvfyk4WgE6zcIPr78wyg6GAo,1065
+mdify_cli-3.0.0.dist-info/METADATA,sha256=HAui-xr9LYP9yJGtFT0g9oVk3NGJnOmo5pyvQx0qb-Q,14766
+mdify_cli-3.0.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+mdify_cli-3.0.0.dist-info/entry_points.txt,sha256=0Xki8f5lADQUtwdt6Eq_FEaieI6Byhk8UE7BuDhChMg,41
+mdify_cli-3.0.0.dist-info/top_level.txt,sha256=qltzf7h8owHq7dxCdfCkSHY8gT21hn1_E8P-VWS_OKM,6
+mdify_cli-3.0.0.dist-info/RECORD,,

mdify_cli-2.11.9.dist-info/RECORD

@@ -1,12 +0,0 @@
-assets/mdify.png,sha256=qUj7WXWqNwpI2KNXOW79XJwqFqa-UI0JEkmt1mmy4Rg,1820418
-mdify/__init__.py,sha256=x2PxT1laVq9WFwgXBDy1TJ_qCOBN4cxlmLYbSBcb7qA,91
-mdify/__main__.py,sha256=bhpJ00co6MfaVOdH4XLoW04NtLYDa_oJK7ODzfLrn9M,143
-mdify/cli.py,sha256=Mv3ClwC84fkorZgwM1IqGMvZ0-hT_V77qhHo2p0ueCU,49638
-mdify/container.py,sha256=ARdFs-TOSh5vHGtBJ0CppfpZFaiprIuRdQ5wDH0NfrY,8377
-mdify/docling_client.py,sha256=xuQR6sC1v3EPloOSwExoHCqT4uUxE8myYq-Yeby3C2I,7975
-mdify_cli-2.11.9.dist-info/licenses/LICENSE,sha256=NWM66Uv-XuSMKaU-gaPmvfyk4WgE6zcIPr78wyg6GAo,1065
-mdify_cli-2.11.9.dist-info/METADATA,sha256=NHwtbgGo2CAPqZIOT7ebPk5mTwsmxRBo5pg0l71xenE,9623
-mdify_cli-2.11.9.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-mdify_cli-2.11.9.dist-info/entry_points.txt,sha256=0Xki8f5lADQUtwdt6Eq_FEaieI6Byhk8UE7BuDhChMg,41
-mdify_cli-2.11.9.dist-info/top_level.txt,sha256=qltzf7h8owHq7dxCdfCkSHY8gT21hn1_E8P-VWS_OKM,6
-mdify_cli-2.11.9.dist-info/RECORD,,