remdb 0.3.0__py3-none-any.whl

rem/cli/commands/mcp.py

@@ -0,0 +1,66 @@
+"""
+REM MCP server command.
+
+Run the FastMCP server in standalone mode (stdio transport for Claude Desktop).
+
+Usage:
+    rem mcp                 # Run MCP server in stdio mode
+    rem mcp --http          # Run in HTTP mode (for testing)
+"""
+
+import click
+from loguru import logger
+
+
+@click.command("mcp")
+@click.option(
+    "--http",
+    is_flag=True,
+    help="Run in HTTP mode instead of stdio (for testing)",
+)
+@click.option(
+    "--host",
+    default="0.0.0.0",
+    help="Host to bind to (HTTP mode only)",
+)
+@click.option(
+    "--port",
+    default=8001,
+    type=int,
+    help="Port to listen on (HTTP mode only)",
+)
+def mcp_command(http: bool, host: str, port: int):
+    """
+    Run the REM MCP server.
+
+    By default, runs in stdio mode for Claude Desktop integration.
+    Use --http for testing in HTTP mode.
+
+    Examples:
+        # Stdio mode (for Claude Desktop)
+        rem mcp
+
+        # HTTP mode (for testing)
+        rem mcp --http --port 8001
+    """
+    from rem.api.mcp_router.server import create_mcp_server
+
+    # Stdio mode is local (allows local file paths), HTTP mode is remote (s3/https only)
+    mcp = create_mcp_server(is_local=not http)
+
+    if http:
+        # HTTP mode for testing
+        logger.info(f"Starting REM MCP server in HTTP mode at http://{host}:{port}")
+        import uvicorn
+
+        mcp_app = mcp.get_asgi_app()
+        uvicorn.run(mcp_app, host=host, port=port)
+    else:
+        # Stdio mode for Claude Desktop
+        logger.info("Starting REM MCP server in stdio mode (Claude Desktop)")
+        mcp.run()
+
+
+def register_command(cli_group):
+    """Register the mcp command."""
+    cli_group.add_command(mcp_command)

rem/cli/commands/process.py

@@ -0,0 +1,245 @@
+"""File processing CLI commands."""
+
+import json
+import sys
+from typing import Optional
+
+import click
+from loguru import logger
+
+from rem.services.content import ContentService
+
+
+@click.command(name="ingest")
+@click.argument("file_path", type=click.Path(exists=True))
+@click.option("--user-id", required=True, help="User ID to own the file")
+@click.option("--category", help="Optional file category")
+@click.option("--tags", help="Optional comma-separated tags")
+def process_ingest(
+    file_path: str,
+    user_id: str,
+    category: str | None,
+    tags: str | None,
+):
+    """
+    Ingest a file into REM (storage + parsing + embedding).
+
+    This command performs the full ingestion pipeline:
+    1. Reads the file from the local path.
+    2. Stores it in the configured storage (local/S3).
+    3. Parses the content.
+    4. Chunks and embeds the content into Resources.
+    5. Creates a File entity record.
+
+    Examples:
+        rem process ingest sample.pdf --user-id user-123
+        rem process ingest contract.docx --user-id user-123 --category legal --tags contract,2023
+    """
+    import asyncio
+    from ...services.content import ContentService
+
+    async def _ingest():
+        # Initialize ContentService with repositories for proper resource saving
+        from rem.services.postgres import get_postgres_service
+        from rem.services.postgres.repository import Repository
+        from rem.models.entities import File, Resource
+
+        db = get_postgres_service()
+        if not db:
+            raise RuntimeError("PostgreSQL service not available")
+        await db.connect()
+
+        try:
+            file_repo = Repository(File, "files", db=db)
+            resource_repo = Repository(Resource, "resources", db=db)
+            service = ContentService(file_repo=file_repo, resource_repo=resource_repo)
+
+            tag_list = tags.split(",") if tags else None
+
+            logger.info(f"Ingesting file: {file_path} for user: {user_id}")
+            result = await service.ingest_file(
+                file_uri=file_path,
+                user_id=user_id,
+                category=category,
+                tags=tag_list,
+                is_local_server=True, # CLI is local
+            )
+
+            if result.get("processing_status") == "completed":
+                logger.success(f"File ingested successfully: {result['file_name']}")
+                logger.info(f"File ID: {result['file_id']}")
+                logger.info(f"Resources created: {result['resources_created']}")
+                logger.info(f"Status: {result['processing_status']}")
+            else:
+                logger.error(f"Ingestion failed: {result.get('message', 'Unknown error')}")
+                sys.exit(1)
+
+        except Exception as e:
+            logger.error(f"Error during ingestion: {e}")
+            sys.exit(1)
+        finally:
+            # Wait for global embedding worker to finish queued tasks
+            from rem.services.embeddings.worker import get_global_embedding_worker
+            try:
+                worker = get_global_embedding_worker()
+                if worker and worker.running and not worker.task_queue.empty():
+                    logger.info(f"Waiting for {worker.task_queue.qsize()} embedding tasks to complete...")
+                    # Worker.stop() waits for queue to drain (see worker.py line ~148)
+                    await worker.stop()
+            except RuntimeError:
+                # Worker doesn't exist yet - no tasks queued
+                pass
+
+            await db.disconnect()
+
+    asyncio.run(_ingest())
+
+def register_commands(group: click.Group):
+    """Register process commands."""
+    group.add_command(process_uri)
+    group.add_command(process_files)
+    group.add_command(process_ingest)
+
+
+@click.command(name="uri")
+@click.argument("uri", type=str)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Choice(["json", "text"]),
+    default="json",
+    help="Output format (json or text)",
+)
+@click.option(
+    "--save",
+    "-s",
+    type=click.Path(),
+    help="Save extracted content to file",
+)
+def process_uri(uri: str, output: str, save: str | None):
+    """
+    Process a file URI and extract content (READ-ONLY, no storage).
+
+    **ARCHITECTURE NOTE - Code Path Comparison**:
+
+    This CLI command provides READ-ONLY file processing:
+    - Uses ContentService.process_uri() directly (no file storage, no DB writes)
+    - Returns extracted content to stdout or saves to local file
+    - No File entity created, no Resource chunks stored in database
+    - Useful for testing file parsing without side effects
+
+    Compare with MCP tool 'parse_and_ingest_file' (api/mcp_router/tools.py):
+    - WRITES file to internal storage (~/.rem/fs/ or S3)
+    - Creates File entity in database
+    - Creates Resource chunks via ContentService.process_and_save()
+    - Full ingestion pipeline for searchable content
+
+    **SHARED CODE**: Both use ContentService for file parsing:
+    - CLI: ContentService.process_uri() → extract only
+    - MCP: ContentService.process_and_save() → extract + store chunks
+
+    URI can be:
+    - S3 URI: s3://bucket/key
+    - Local file: /path/to/file.md or ./file.md
+
+    Examples:
+
+        \b
+        # Process local markdown file
+        rem process uri ./README.md
+
+        \b
+        # Process S3 file
+        rem process uri s3://rem/uploads/document.md
+
+        \b
+        # Save to file
+        rem process uri s3://rem/uploads/doc.md -s output.json
+
+        \b
+        # Text-only output
+        rem process uri ./file.md -o text
+    """
+    try:
+        service = ContentService()
+        result = service.process_uri(uri)
+
+        if output == "json":
+            output_data = json.dumps(result, indent=2, default=str)
+        else:
+            # Text-only output
+            output_data = result["content"]
+
+        # Save to file or print to stdout
+        if save:
+            with open(save, "w") as f:
+                f.write(output_data)
+            logger.info(f"Saved to {save}")
+        else:
+            click.echo(output_data)
+
+        sys.exit(0)
+
+    except FileNotFoundError as e:
+        logger.error(f"File not found: {e}")
+        sys.exit(1)
+    except RuntimeError as e:
+        logger.error(f"Processing error: {e}")
+        sys.exit(1)
+    except Exception as e:
+        logger.exception(f"Unexpected error: {e}")
+        sys.exit(1)
+
+
+@click.command(name="files")
+@click.option("--tenant-id", required=True, help="Tenant ID")
+@click.option("--user-id", help="Filter by user ID")
+@click.option("--status", type=click.Choice(["pending", "processing", "completed", "failed"]), help="Filter by status")
+@click.option("--extractor", help="Run files through custom extractor (e.g., cv-parser-v1)")
+@click.option("--limit", type=int, help="Max files to process")
+@click.option("--provider", help="Optional LLM provider override")
+@click.option("--model", help="Optional model override")
+def process_files(
+    tenant_id: str,
+    user_id: Optional[str],
+    status: Optional[str],
+    extractor: Optional[str],
+    limit: Optional[int],
+    provider: Optional[str],
+    model: Optional[str],
+):
+    """Process files with optional custom extractor.
+
+    Query files from the database and optionally run them through
+    a custom extractor to extract domain-specific knowledge.
+
+    Examples:
+
+        \b
+        # List completed files
+        rem process files --tenant-id acme-corp --status completed
+
+        \b
+        # Extract from CV files
+        rem process files --tenant-id acme-corp --extractor cv-parser-v1 --limit 10
+
+        \b
+        # Extract with provider override
+        rem process files --tenant-id acme-corp --extractor contract-analyzer-v1 \\
+            --provider anthropic --model claude-sonnet-4-5
+    """
+    logger.warning("Not implemented yet")
+    logger.info(f"Would process files for tenant: {tenant_id}")
+
+    if user_id:
+        logger.info(f"Filter: user_id={user_id}")
+    if status:
+        logger.info(f"Filter: status={status}")
+    if extractor:
+        logger.info(f"Extractor: {extractor}")
+    if limit:
+        logger.info(f"Limit: {limit} files")
+    if provider:
+        logger.info(f"Provider override: {provider}")
+    if model:
+        logger.info(f"Model override: {model}")

rem/cli/commands/schema.py

@@ -0,0 +1,183 @@
+"""
+Schema generation commands.
+
+Usage:
+    rem db schema generate --models src/rem/models/entities
+    rem db schema validate
+    rem db schema indexes --background
+"""
+
+import asyncio
+from pathlib import Path
+
+import click
+from loguru import logger
+
+from ...settings import settings
+from ...services.postgres.schema_generator import SchemaGenerator
+
+
+@click.command()
+@click.option(
+    "--models",
+    "-m",
+    required=True,
+    type=click.Path(exists=True, path_type=Path),
+    help="Directory containing Pydantic models",
+)
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(path_type=Path),
+    default="install_models.sql",
+    help="Output SQL file (default: install_models.sql)",
+)
+@click.option(
+    "--output-dir",
+    type=click.Path(path_type=Path),
+    default=None,
+    help=f"Base output directory (default: {settings.sql_dir})",
+)
+def generate(models: Path, output: Path, output_dir: Path | None):
+    """
+    Generate database schema from Pydantic models.
+
+    Scans the specified directory for Pydantic models and generates:
+    - CREATE TABLE statements
+    - Embeddings tables (embeddings_<table>)
+    - KV_STORE triggers for cache maintenance
+    - Indexes (foreground only)
+
+    Output is written to src/rem/sql/install_models.sql by default.
+
+    Example:
+        rem db schema generate --models src/rem/models/entities
+
+    This creates:
+    - src/rem/sql/install_models.sql - Entity tables and triggers
+    - src/rem/sql/background_indexes.sql - HNSW indexes (apply after data load)
+
+    After generation, apply with:
+        rem db migrate
+    """
+    click.echo(f"Discovering models in {models}")
+
+    # Use settings.sql_dir if not provided
+    actual_output_dir = output_dir or Path(settings.sql_dir)
+    generator = SchemaGenerator(output_dir=actual_output_dir)
+
+    # Generate schema
+    try:
+        schema_sql = asyncio.run(generator.generate_from_directory(models, output_file=output.name))
+
+        click.echo(f"✓ Schema generated: {len(generator.schemas)} tables")
+        click.echo(f"✓ Written to: {actual_output_dir / output.name}")
+
+        # Generate background indexes
+        background_indexes = generator.generate_background_indexes()
+        if background_indexes:
+            bg_file = actual_output_dir / "background_indexes.sql"
+            bg_file.write_text(background_indexes)
+            click.echo(f"✓ Background indexes: {bg_file}")
+
+        # Summary
+        click.echo("\nGenerated tables:")
+        for table_name, schema in generator.schemas.items():
+            embeddable = len(schema.get("embeddable_fields", []))
+            embed_status = f"({embeddable} embeddable fields)" if embeddable else "(no embeddings)"
+            click.echo(f"  - {table_name} {embed_status}")
+
+    except Exception as e:
+        logger.exception("Schema generation failed")
+        click.echo(f"✗ Error: {e}", err=True)
+        raise click.Abort()
+
+
+@click.command()
+@click.option(
+    "--models",
+    "-m",
+    required=True,
+    type=click.Path(exists=True, path_type=Path),
+    help="Directory containing Pydantic models",
+)
+def validate(models: Path):
+    """
+    Validate Pydantic models for schema generation.
+
+    Checks:
+    - Models can be loaded
+    - Models have suitable entity_key fields
+    - Fields with embeddings are properly configured
+    """
+    click.echo(f"Validating models in {models}")
+
+    generator = SchemaGenerator()
+    discovered = generator.discover_models(models)
+
+    if not discovered:
+        click.echo("✗ No models found", err=True)
+        raise click.Abort()
+
+    click.echo(f"✓ Discovered {len(discovered)} models")
+
+    errors: list[str] = []
+    warnings: list[str] = []
+
+    for model_name, model in discovered.items():
+        table_name = generator.infer_table_name(model)
+        entity_key = generator.infer_entity_key_field(model)
+
+        # Check for entity_key
+        if entity_key == "id":
+            warnings.append(f"{model_name}: No natural key field, using 'id'")
+
+        # Check for embeddable fields
+        # TODO: Implement should_embed_field check
+        embeddable: list[str] = []  # Placeholder - needs implementation
+
+        click.echo(f"  {model_name} -> {table_name} (key: {entity_key})")
+
+    if warnings:
+        click.echo("\nWarnings:")
+        for warning in warnings:
+            click.echo(click.style(f"  ⚠ {warning}", fg="yellow"))
+
+    if errors:
+        click.echo("\nErrors:")
+        for error in errors:
+            click.echo(click.style(f"  ✗ {error}", fg="red"))
+        raise click.Abort()
+
+    click.echo("\n✓ All models valid")
+
+
+@click.command()
+@click.option(
+    "--output",
+    "-o",
+    type=click.Path(path_type=Path),
+    default=None,
+    help=f"Output file for background indexes (default: {settings.sql_dir}/background_indexes.sql)",
+)
+def indexes(output: Path):
+    """
+    Generate SQL for background index creation.
+
+    Creates HNSW vector indexes that should be run CONCURRENTLY
+    after initial data load to avoid blocking writes.
+    """
+    click.echo("Generating background index SQL")
+
+    generator = SchemaGenerator()
+
+    # Load existing schemas (would need to be persisted or regenerated)
+    click.echo(click.style("⚠ Note: This requires schemas to be generated first", fg="yellow"))
+    click.echo(click.style("⚠ Run 'rem db schema generate' before 'rem db schema indexes'", fg="yellow"))
+
+
+def register_commands(schema_group):
+    """Register all schema commands."""
+    schema_group.add_command(generate)
+    schema_group.add_command(validate)
+    schema_group.add_command(indexes)

rem/cli/commands/serve.py

@@ -0,0 +1,106 @@
+"""
+REM API server command.
+
+Start the FastAPI server with uvicorn.
+
+Usage:
+    rem serve                   # Start API server with default settings
+    rem serve --host 0.0.0.0    # Bind to all interfaces
+    rem serve --port 8080       # Use custom port
+    rem serve --reload          # Enable auto-reload (development)
+    rem serve --workers 4       # Production mode with workers
+"""
+
+import click
+from loguru import logger
+
+
+@click.command("serve")
+@click.option(
+    "--host",
+    default=None,
+    help="Host to bind to (overrides config/env)",
+)
+@click.option(
+    "--port",
+    default=None,
+    type=int,
+    help="Port to listen on (overrides config/env)",
+)
+@click.option(
+    "--reload",
+    is_flag=True,
+    default=None,
+    help="Enable auto-reload for development",
+)
+@click.option(
+    "--workers",
+    default=None,
+    type=int,
+    help="Number of worker processes (production mode)",
+)
+@click.option(
+    "--log-level",
+    default=None,
+    type=click.Choice(["critical", "error", "warning", "info", "debug", "trace"]),
+    help="Logging level",
+)
+def serve_command(
+    host: str | None,
+    port: int | None,
+    reload: bool | None,
+    workers: int | None,
+    log_level: str | None,
+):
+    """
+    Start the REM API server.
+
+    The server will load configuration from:
+    1. Environment variables (highest priority)
+    2. ~/.rem/config.yaml (user configuration)
+    3. Default values (lowest priority)
+
+    Examples:
+        rem serve                       # Use settings from config
+        rem serve --host 0.0.0.0        # Bind to all interfaces
+        rem serve --reload              # Development mode
+        rem serve --workers 4           # Production mode (4 workers)
+    """
+    import uvicorn
+
+    # Import settings to trigger config loading
+    from rem.settings import settings
+
+    # Determine reload/workers (mutually exclusive)
+    use_reload = reload if reload is not None else (settings.api.reload if workers is None else False)
+    use_workers = workers if workers is not None else (settings.api.workers if not use_reload else None)
+
+    # Start server
+    final_host = host or settings.api.host
+    final_port = port or settings.api.port
+    final_log_level = log_level or settings.api.log_level
+
+    logger.info(f"Starting REM API server at http://{final_host}:{final_port}")
+
+    # Call uvicorn.run with explicit parameters to satisfy type checker
+    if use_reload:
+        uvicorn.run(
+            "rem.api.main:app",
+            host=final_host,
+            port=final_port,
+            log_level=final_log_level,
+            reload=True,
+        )
+    else:
+        uvicorn.run(
+            "rem.api.main:app",
+            host=final_host,
+            port=final_port,
+            log_level=final_log_level,
+            workers=use_workers,
+        )
+
+
+def register_command(cli_group):
+    """Register the serve command."""
+    cli_group.add_command(serve_command)