remdb 0.2.6__py3-none-any.whl

rem/cli/dreaming.py

@@ -0,0 +1,363 @@
+"""
+REM Dreaming CLI - Memory indexing and insight extraction.
+
+Command-line interface for running dreaming workers to build the
+REM knowledge graph through user model updates, moment construction,
+and resource affinity operations.
+
+Commands:
+- user-model: Update user profiles from activity
+- moments: Extract temporal narratives from resources
+- affinity: Build semantic relationships between resources
+- custom: Run custom extractors on user's resources/sessions
+- full: Run complete dreaming workflow (all operations)
+
+Usage Examples:
+```bash
+# Update user model for specific user
+rem-dreaming user-model --user-id=user-123
+
+# Extract moments with custom lookback
+rem-dreaming moments --user-id=user-123 --lookback-hours=48
+
+# Build resource affinity (semantic mode, fast)
+rem-dreaming affinity --user-id=user-123
+
+# Build resource affinity (LLM mode, intelligent but expensive)
+rem-dreaming affinity --user-id=user-123 --use-llm --limit=100
+
+# Run custom extractor on user's data
+rem-dreaming custom --user-id=user-123 --extractor cv-parser-v1
+rem-dreaming custom --user-id=user-123 --extractor contract-analyzer-v1 --lookback-hours=168
+
+# Run full workflow for user
+rem-dreaming full --user-id=user-123
+
+# Process all active users (daily cron)
+rem-dreaming full --all-users
+
+# Process with custom REM API endpoint
+rem-dreaming full --user-id=user-123 --rem-api-url=http://localhost:8000
+```
+
+Environment Variables:
+- REM_API_URL: REM API endpoint (default: http://rem-api:8000)
+- REM_EMBEDDING_PROVIDER: Embedding provider (default: text-embedding-3-small)
+- REM_DEFAULT_MODEL: LLM model (default: gpt-4o)
+- REM_LOOKBACK_HOURS: Default lookback window (default: 24)
+- OPENAI_API_KEY: OpenAI API key
+
+Exit Codes:
+- 0: Success
+- 1: Validation error (missing required args)
+- 2: Execution error (worker failed)
+"""
+
+import asyncio
+import os
+import sys
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.progress import Progress, SpinnerColumn, TextColumn
+
+from rem.workers.dreaming import (
+    AffinityMode,
+    DreamingWorker,
+    TaskType,
+)
+
+app = typer.Typer(
+    name="rem-dreaming",
+    help="REM dreaming worker for memory indexing",
+    add_completion=False,
+)
+console = Console()
+
+
+def get_worker() -> DreamingWorker:
+    """Create dreaming worker from environment."""
+    return DreamingWorker(
+        rem_api_url=os.getenv("REM_API_URL", "http://rem-api:8000"),
+        embedding_provider=os.getenv(
+            "REM_EMBEDDING_PROVIDER", "text-embedding-3-small"
+        ),
+        default_model=os.getenv("REM_DEFAULT_MODEL", "gpt-4o"),
+        lookback_hours=int(os.getenv("REM_LOOKBACK_HOURS", "24")),
+    )
+
+
+@app.command()
+def user_model(
+    user_id: str = typer.Option(..., help="User ID to process"),
+    max_sessions: int = typer.Option(100, help="Max sessions to analyze"),
+    max_moments: int = typer.Option(20, help="Max moments to include"),
+    max_resources: int = typer.Option(20, help="Max resources to include"),
+):
+    """
+    Update user model from recent activity.
+
+    Reads recent sessions, moments, and resources to generate
+    a comprehensive user profile summary using LLM analysis.
+    """
+
+    async def run():
+        worker = get_worker()
+        try:
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                console=console,
+            ) as progress:
+                task = progress.add_task(
+                    f"Updating user model for {user_id}...", total=None
+                )
+
+                result = await worker.update_user_model(
+                    user_id=user_id,
+                    max_sessions=max_sessions,
+                    max_moments=max_moments,
+                    max_resources=max_resources,
+                )
+
+                progress.update(task, completed=True)
+                console.print(f"[green]✓[/green] User model updated")
+                console.print(result)
+        except Exception as e:
+            console.print(f"[red]✗[/red] Failed: {e}", style="red")
+            sys.exit(2)
+        finally:
+            await worker.close()
+
+    asyncio.run(run())
+
+
+@app.command()
+def moments(
+    user_id: str = typer.Option(..., help="User ID to process"),
+    lookback_hours: Optional[int] = typer.Option(
+        None, help="Hours to look back (default: from env)"
+    ),
+    limit: Optional[int] = typer.Option(None, help="Max resources to process"),
+):
+    """
+    Extract moments from resources.
+
+    Analyzes recent resources to identify temporal narratives
+    (meetings, coding sessions, conversations) and creates
+    Moment entities with temporal boundaries and metadata.
+    """
+
+    async def run():
+        worker = get_worker()
+        try:
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                console=console,
+            ) as progress:
+                task = progress.add_task(
+                    f"Constructing moments for {user_id}...", total=None
+                )
+
+                result = await worker.construct_moments(
+                    user_id=user_id,
+                    lookback_hours=lookback_hours,
+                    limit=limit,
+                )
+
+                progress.update(task, completed=True)
+                console.print(f"[green]✓[/green] Moments constructed")
+                console.print(result)
+        except Exception as e:
+            console.print(f"[red]✗[/red] Failed: {e}", style="red")
+            sys.exit(2)
+        finally:
+            await worker.close()
+
+    asyncio.run(run())
+
+
+@app.command()
+def affinity(
+    user_id: str = typer.Option(..., help="User ID to process"),
+    use_llm: bool = typer.Option(
+        False, "--use-llm", help="Use LLM mode (expensive, use --limit)"
+    ),
+    lookback_hours: Optional[int] = typer.Option(
+        None, help="Hours to look back (default: from env)"
+    ),
+    limit: Optional[int] = typer.Option(
+        None, help="Max resources to process (REQUIRED for LLM mode)"
+    ),
+):
+    """
+    Build resource affinity graph.
+
+    Creates semantic relationships between resources using either
+    vector similarity (fast, default) or LLM analysis (intelligent but expensive).
+
+    Semantic Mode (default):
+    - Fast vector similarity search
+    - No LLM calls, just embedding cosine similarity
+    - Good for frequent updates
+
+    LLM Mode (--use-llm):
+    - Intelligent relationship assessment
+    - Expensive: ALWAYS use --limit to control costs
+    - Good for deep analysis (weekly or monthly)
+
+    Example:
+        # Semantic mode (fast, cheap)
+        rem-dreaming affinity --user-id=user-123
+
+        # LLM mode (intelligent, expensive)
+        rem-dreaming affinity --user-id=user-123 --use-llm --limit=100
+    """
+    if use_llm and not limit:
+        console.print(
+            "[red]Error:[/red] --limit is REQUIRED when using --use-llm to control costs",
+            style="red",
+        )
+        sys.exit(1)
+
+    async def run():
+        worker = get_worker()
+        try:
+            mode = AffinityMode.LLM if use_llm else AffinityMode.SEMANTIC
+            mode_str = "LLM" if use_llm else "semantic"
+
+            with Progress(
+                SpinnerColumn(),
+                TextColumn("[progress.description]{task.description}"),
+                console=console,
+            ) as progress:
+                task = progress.add_task(
+                    f"Building {mode_str} affinity for {user_id}...", total=None
+                )
+
+                result = await worker.build_affinity(
+                    user_id=user_id,
+                    mode=mode,
+                    lookback_hours=lookback_hours,
+                    limit=limit,
+                )
+
+                progress.update(task, completed=True)
+                console.print(f"[green]✓[/green] Resource affinity built ({mode_str} mode)")
+                console.print(result)
+        except Exception as e:
+            console.print(f"[red]✗[/red] Failed: {e}", style="red")
+            sys.exit(2)
+        finally:
+            await worker.close()
+
+    asyncio.run(run())
+
+
+@app.command()
+def full(
+    user_id: Optional[str] = typer.Option(None, help="User ID (or --all-users)"),
+    all_users: bool = typer.Option(
+        False, "--all-users", help="Process all active users"
+    ),
+    use_llm_affinity: bool = typer.Option(
+        False, "--use-llm-affinity", help="Use LLM mode for affinity (expensive)"
+    ),
+    lookback_hours: Optional[int] = typer.Option(
+        None, help="Hours to look back (default: from env)"
+    ),
+):
+    """
+    Run complete dreaming workflow.
+
+    Executes all dreaming operations in sequence:
+    1. Update user model
+    2. Construct moments
+    3. Build resource affinity
+
+    Recommended for daily cron execution.
+
+    Examples:
+        # Process single user
+        rem-dreaming full --user-id=user-123
+
+        # Process all active users (daily cron)
+        rem-dreaming full --all-users
+
+        # Use LLM affinity mode (expensive)
+        rem-dreaming full --user-id=user-123 --use-llm-affinity
+    """
+    if not user_id and not all_users:
+        console.print(
+            "[red]Error:[/red] Either --user-id or --all-users is required",
+            style="red",
+        )
+        sys.exit(1)
+
+    if user_id and all_users:
+        console.print(
+            "[red]Error:[/red] Cannot use both --user-id and --all-users",
+            style="red",
+        )
+        sys.exit(1)
+
+    async def run():
+        worker = get_worker()
+        try:
+            if all_users:
+                with Progress(
+                    SpinnerColumn(),
+                    TextColumn("[progress.description]{task.description}"),
+                    console=console,
+                ) as progress:
+                    task = progress.add_task("Processing all users...", total=None)
+
+                    results = await worker.process_all_users(
+                        task_type=TaskType.FULL,
+                        use_llm_affinity=use_llm_affinity,
+                        lookback_hours=lookback_hours,
+                    )
+
+                    progress.update(task, completed=True)
+                    console.print(
+                        f"[green]✓[/green] Processed {len(results)} users"
+                    )
+                    for result in results:
+                        console.print(result)
+            else:
+                with Progress(
+                    SpinnerColumn(),
+                    TextColumn("[progress.description]{task.description}"),
+                    console=console,
+                ) as progress:
+                    task = progress.add_task(
+                        f"Running full workflow for {user_id}...", total=None
+                    )
+
+                    result = await worker.process_full(
+                        user_id=user_id,
+                        use_llm_affinity=use_llm_affinity,
+                        lookback_hours=lookback_hours,
+                    )
+
+                    progress.update(task, completed=True)
+                    console.print(f"[green]✓[/green] Full workflow completed")
+                    console.print(result)
+        except Exception as e:
+            console.print(f"[red]✗[/red] Failed: {e}", style="red")
+            sys.exit(2)
+        finally:
+            await worker.close()
+
+    asyncio.run(run())
+
+
+def main():
+    """Entry point."""
+    app()
+
+
+if __name__ == "__main__":
+    main()

rem/cli/main.py

@@ -0,0 +1,88 @@
+"""
+REM CLI entry point.
+
+Usage:
+    rem db schema generate --models src/rem/models/entities
+    rem db schema validate
+    rem db migrate up
+    rem dev run-server
+"""
+
+import sys
+from pathlib import Path
+
+import click
+from loguru import logger
+
+
+@click.group()
+@click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging")
+def cli(verbose: bool):
+    """REM - Resources Entities Moments system CLI."""
+    if verbose:
+        logger.remove()
+        logger.add(sys.stderr, level="DEBUG")
+    else:
+        logger.remove()
+        logger.add(sys.stderr, level="INFO")
+
+
+@cli.group()
+def db():
+    """Database operations (schema, migrate, status, etc.)."""
+    pass
+
+
+@db.group()
+def schema():
+    """Database schema management commands."""
+    pass
+
+
+@cli.group()
+def dev():
+    """Development utilities."""
+    pass
+
+
+@cli.group()
+def process():
+    """File processing commands."""
+    pass
+
+
+@cli.group()
+def dreaming():
+    """Memory indexing and knowledge extraction."""
+    pass
+
+
+# Register commands
+from .commands.schema import register_commands as register_schema_commands
+from .commands.db import register_commands as register_db_commands
+from .commands.process import register_commands as register_process_commands
+from .commands.ask import register_command as register_ask_command
+from .commands.dreaming import register_commands as register_dreaming_commands
+from .commands.experiments import experiments as experiments_group
+from .commands.configure import register_command as register_configure_command
+from .commands.serve import register_command as register_serve_command
+from .commands.mcp import register_command as register_mcp_command
+
+register_schema_commands(schema)
+register_db_commands(db)
+register_process_commands(process)
+register_dreaming_commands(dreaming)
+register_ask_command(cli)
+register_configure_command(cli)
+register_serve_command(cli)
+register_mcp_command(cli)
+cli.add_command(experiments_group)
+
+
+def main():
+    """Main entry point for CLI."""
+    cli()
+
+
+if __name__ == "__main__":
+    main()

rem/config.py

@@ -0,0 +1,237 @@
+"""
+REM Configuration Management.
+
+Provides persistent configuration in ~/.rem/config.yaml with environment variable overrides.
+
+Configuration Precedence (highest to lowest):
+1. Environment variables (POSTGRES__CONNECTION_STRING, etc.)
+2. ~/.rem/config.yaml (user configuration)
+3. Default values (from settings.py)
+
+File Format (~/.rem/config.yaml):
+    postgres:
+      connection_string: postgresql://user:pass@localhost:5432/rem
+      pool_min_size: 5
+      pool_max_size: 20
+
+    llm:
+      default_model: anthropic:claude-sonnet-4-5-20250929
+      openai_api_key: sk-...
+      anthropic_api_key: sk-ant-...
+
+    s3:
+      bucket_name: rem-storage
+      region: us-east-1
+      endpoint_url: http://localhost:9000
+
+    # Additional custom environment variables
+    env:
+      MY_CUSTOM_VAR: value
+
+Usage:
+    from rem.config import load_config, get_config_path, ensure_config_dir
+
+    # Load configuration and merge with environment
+    config = load_config()
+
+    # Get configuration file path
+    config_path = get_config_path()
+
+    # Ensure ~/.rem directory exists
+    ensure_config_dir()
+"""
+
+import os
+from pathlib import Path
+from typing import Any
+
+import yaml
+from loguru import logger
+
+
+def get_rem_home() -> Path:
+    """
+    Get REM home directory (~/.rem).
+
+    Returns:
+        Path to ~/.rem directory
+    """
+    return Path.home() / ".rem"
+
+
+def ensure_config_dir() -> Path:
+    """
+    Ensure ~/.rem directory exists.
+
+    Returns:
+        Path to ~/.rem directory
+    """
+    rem_home = get_rem_home()
+    rem_home.mkdir(exist_ok=True, mode=0o700)  # User-only permissions
+    return rem_home
+
+
+def get_config_path() -> Path:
+    """
+    Get path to configuration file (~/.rem/config.yaml).
+
+    Returns:
+        Path to configuration file
+    """
+    return get_rem_home() / "config.yaml"
+
+
+def config_exists() -> bool:
+    """
+    Check if configuration file exists.
+
+    Returns:
+        True if ~/.rem/config.yaml exists
+    """
+    return get_config_path().exists()
+
+
+def load_config() -> dict[str, Any]:
+    """
+    Load configuration from ~/.rem/config.yaml.
+
+    Returns:
+        Configuration dictionary (empty if file doesn't exist)
+    """
+    config_path = get_config_path()
+
+    if not config_path.exists():
+        logger.debug(f"Configuration file not found: {config_path}")
+        return {}
+
+    try:
+        with open(config_path, "r") as f:
+            config = yaml.safe_load(f) or {}
+            logger.debug(f"Loaded configuration from {config_path}")
+            return config
+    except Exception as e:
+        logger.warning(f"Failed to load configuration from {config_path}: {e}")
+        return {}
+
+
+def save_config(config: dict[str, Any]) -> None:
+    """
+    Save configuration to ~/.rem/config.yaml.
+
+    Args:
+        config: Configuration dictionary to save
+    """
+    ensure_config_dir()
+    config_path = get_config_path()
+
+    try:
+        with open(config_path, "w") as f:
+            yaml.dump(config, f, default_flow_style=False, sort_keys=False)
+            logger.info(f"Configuration saved to {config_path}")
+    except Exception as e:
+        logger.error(f"Failed to save configuration to {config_path}: {e}")
+        raise
+
+
+def merge_config_to_env(config: dict[str, Any]) -> None:
+    """
+    Merge configuration file into environment variables.
+
+    This allows Pydantic Settings to pick up values from the config file
+    by setting environment variables before settings initialization.
+
+    Precedence:
+    - Existing environment variables are NOT overwritten
+    - Only sets env vars if they don't already exist
+
+    Args:
+        config: Configuration dictionary from ~/.rem/config.yaml
+
+    Example:
+        config = {"postgres": {"connection_string": "postgresql://..."}}
+        merge_config_to_env(config)
+        # Sets POSTGRES__CONNECTION_STRING if not already set
+    """
+    # Handle custom env vars first
+    if "env" in config:
+        for key, value in config["env"].items():
+            if key not in os.environ:
+                os.environ[key] = str(value)
+                logger.debug(f"Set env var from config: {key}")
+
+    # Convert nested config to environment variables
+    for section, values in config.items():
+        if section == "env":
+            continue  # Already handled
+
+        if not isinstance(values, dict):
+            continue
+
+        for key, value in values.items():
+            # Convert to environment variable format (SECTION__KEY)
+            env_key = f"{section.upper()}__{key.upper()}"
+
+            # Only set if not already in environment
+            if env_key not in os.environ:
+                os.environ[env_key] = str(value)
+                logger.debug(f"Set env var from config: {env_key}")
+
+
+def validate_config(config: dict[str, Any]) -> list[str]:
+    """
+    Validate configuration for required fields.
+
+    Args:
+        config: Configuration dictionary
+
+    Returns:
+        List of validation error messages (empty if valid)
+    """
+    errors = []
+
+    # Postgres connection is required
+    postgres = config.get("postgres", {})
+    if not postgres.get("connection_string"):
+        errors.append("PostgreSQL connection string is required (postgres.connection_string)")
+
+    # Validate connection string format
+    conn_str = postgres.get("connection_string", "")
+    if conn_str and not conn_str.startswith("postgresql://"):
+        errors.append("PostgreSQL connection string must start with 'postgresql://'")
+
+    return errors
+
+
+def get_default_config() -> dict[str, Any]:
+    """
+    Get default configuration template for new installations.
+
+    Returns:
+        Default configuration dictionary
+    """
+    return {
+        "postgres": {
+            "connection_string": "postgresql://rem:rem@localhost:5432/rem",
+            "pool_min_size": 5,
+            "pool_max_size": 20,
+        },
+        "llm": {
+            "default_model": "anthropic:claude-sonnet-4-5-20250929",
+            "default_temperature": 0.5,
+            # API keys will be prompted for in wizard
+            # "openai_api_key": "",
+            # "anthropic_api_key": "",
+        },
+        "s3": {
+            "bucket_name": "rem-storage",
+            "region": "us-east-1",
+            # Optional fields
+            # "endpoint_url": "http://localhost:9000",  # For MinIO
+            # "access_key_id": "",
+            # "secret_access_key": "",
+        },
+        "env": {
+            # Custom environment variables
+            # "MY_VAR": "value",
+        },
+    }