remdb 0.3.103__py3-none-any.whl → 0.3.118__py3-none-any.whl

rem/cli/commands/configure.py

@@ -405,9 +405,7 @@ def configure_command(install: bool, claude_desktop: bool, show: bool, edit: boo
 
                 # Create a context for the command and invoke it
                 ctx = click.Context(migrate)
-                ctx.invoke(migrate, install_only=False, models_only=False,
-                          background_indexes=False, connection=None,
-                          sql_dir=Path("rem/sql"))
+                ctx.invoke(migrate, background_indexes=False)
 
                 click.echo("✅ Database installation complete")
 

rem/cli/commands/db.py

@@ -98,168 +98,119 @@ def calculate_checksum(file_path: Path) -> str:
 
 
 @click.command()
-@click.option(
-    "--install",
-    "install_only",
-    is_flag=True,
-    help="Apply only install.sql (extensions and infrastructure)",
-)
-@click.option(
-    "--models", "models_only", is_flag=True, help="Apply only install_models.sql (entity tables)"
-)
 @click.option(
     "--background-indexes",
     is_flag=True,
-    help="Apply background indexes (HNSW for vectors)",
-)
-@click.option(
-    "--connection",
-    "-c",
-    help="PostgreSQL connection string (overrides environment)",
+    help="Also apply background HNSW indexes (run after data load)",
 )
-@click.option(
-    "--sql-dir",
-    type=click.Path(exists=True, path_type=Path),
-    default=None,
-    help="Directory containing SQL files (defaults to package SQL dir)",
-)
-def migrate(
-    install_only: bool,
-    models_only: bool,
-    background_indexes: bool,
-    connection: str | None,
-    sql_dir: Path | None,
-):
+def migrate(background_indexes: bool):
     """
-    Apply database migrations.
+    Apply standard database migrations (001_install + 002_install_models).
 
-    By default, applies both install.sql and install_models.sql.
-    Use flags to apply specific migrations.
+    This is a convenience command for initial setup. It applies:
+    1. 001_install.sql - Core infrastructure (extensions, kv_store)
+    2. 002_install_models.sql - Entity tables from registered models
+
+    For incremental changes, use the diff-based workflow instead:
+        rem db schema generate  # Regenerate from models
+        rem db diff             # Check what changed
+        rem db apply <file>     # Apply changes
 
     Examples:
-        rem db migrate                     # Apply all
-        rem db migrate --install           # Core infrastructure only
-        rem db migrate --models            # Entity tables only
-        rem db migrate --background-indexes  # Background HNSW indexes
+        rem db migrate                    # Initial setup
+        rem db migrate --background-indexes  # Include HNSW indexes
     """
-    asyncio.run(_migrate_async(install_only, models_only, background_indexes, connection, sql_dir))
+    asyncio.run(_migrate_async(background_indexes))
 
 
-async def _migrate_async(
-    install_only: bool,
-    models_only: bool,
-    background_indexes: bool,
-    connection: str | None,
-    sql_dir: Path | None,
-):
+async def _migrate_async(background_indexes: bool):
     """Async implementation of migrate command."""
-    from ...services.postgres import get_postgres_service
-    # Find SQL directory - use package SQL if not specified
-    if sql_dir is None:
-        import importlib.resources
-        try:
-            # Python 3.9+
-            sql_ref = importlib.resources.files("rem") / "sql"
-            sql_dir = Path(str(sql_ref))
-        except AttributeError:
-            # Fallback: try to find sql dir relative to package
-            import rem
-            package_dir = Path(rem.__file__).parent.parent
-            sql_dir = package_dir / "sql"
-            if not sql_dir.exists():
-                # Last resort: current directory
-                sql_dir = Path("sql")
+    from ...settings import settings
+    from ...utils.sql_paths import (
+        get_package_sql_dir,
+        get_user_sql_dir,
+        list_all_migrations,
+    )
 
+    click.echo()
     click.echo("REM Database Migration")
     click.echo("=" * 60)
-    click.echo(f"SQL Directory: {sql_dir}")
+
+    # Find package SQL directory
+    try:
+        package_sql_dir = get_package_sql_dir()
+        click.echo(f"Package SQL: {package_sql_dir}")
+    except FileNotFoundError as e:
+        click.secho(f"✗ {e}", fg="red")
+        raise click.Abort()
+
+    # Check for user migrations
+    user_sql_dir = get_user_sql_dir()
+    if user_sql_dir:
+        click.echo(f"User SQL: {user_sql_dir}")
+
+    # Get all migrations (package + user)
+    all_migrations = list_all_migrations()
+
+    if not all_migrations:
+        click.secho("✗ No migration files found", fg="red")
+        raise click.Abort()
+
+    click.echo(f"Found {len(all_migrations)} migration(s)")
     click.echo()
 
-    # Discover migrations from migrations/ directory
-    migrations_dir = sql_dir / "migrations"
+    # Add background indexes if requested
+    migrations_to_apply = [(f, f.stem) for f in all_migrations]
 
     if background_indexes:
-        # Special case: background indexes
-        migrations = [("background_indexes.sql", "Background Indexes")]
-    elif install_only or models_only:
-        # Find specific migration
-        target_prefix = "001" if install_only else "002"
-        migration_files = sorted(migrations_dir.glob(f"{target_prefix}_*.sql"))
-        if migration_files:
-            migrations = [(f"migrations/{f.name}", f.stem.replace("_", " ").title()) for f in migration_files]
+        bg_indexes = package_sql_dir / "background_indexes.sql"
+        if bg_indexes.exists():
+            migrations_to_apply.append((bg_indexes, "Background Indexes"))
         else:
-            migrations = []
-    else:
-        # Default: discover and apply all migrations in sorted order
-        migration_files = sorted(migrations_dir.glob("*.sql"))
-        migrations = [(f"migrations/{f.name}", f.stem.replace("_", " ").title()) for f in migration_files]
-
-    # Check files exist
-    for filename, description in migrations:
-        file_path = sql_dir / filename
+            click.secho("⚠ background_indexes.sql not found, skipping", fg="yellow")
+
+    # Check all files exist (they should, but verify)
+    for file_path, description in migrations_to_apply:
         if not file_path.exists():
-            if filename == "install_models.sql":
-                click.secho(f"✗ {filename} not found", fg="red")
+            click.secho(f"✗ {file_path.name} not found", fg="red")
+            if "002" in file_path.name:
                 click.echo()
                 click.secho("Generate it first with:", fg="yellow")
-                click.secho("  rem db schema generate --models src/rem/models/entities", fg="yellow")
-                raise click.Abort()
-            else:
-                click.secho(f"✗ {filename} not found", fg="red")
-                raise click.Abort()
+                click.secho("  rem db schema generate", fg="yellow")
+            raise click.Abort()
 
-    # Connect to database
-    db = get_postgres_service()
-    if not db:
-        click.secho("Error: PostgreSQL is disabled in settings.", fg="red")
-        raise click.Abort()
-        
-    await db.connect()
+    # Apply each migration
+    import psycopg
+    conn_str = settings.postgres.connection_string
+    total_time = 0.0
 
-    try:
-        # Apply migrations
-        total_time = 0.0
-        all_success = True
+    for file_path, description in migrations_to_apply:
+        click.echo(f"Applying: {file_path.name}")
 
-        for filename, description in migrations:
-            file_path = sql_dir / filename
-            checksum = calculate_checksum(file_path)
+        sql_content = file_path.read_text(encoding="utf-8")
+        start_time = time.time()
 
-            click.echo(f"Applying: {description} ({filename})")
-            click.echo(f"  Checksum: {checksum[:16]}...")
+        try:
+            with psycopg.connect(conn_str) as conn:
+                with conn.cursor() as cur:
+                    cur.execute(sql_content)
+                conn.commit()
 
-            success, output, exec_time = await run_sql_file_async(file_path, db)
+            exec_time = (time.time() - start_time) * 1000
             total_time += exec_time
+            click.secho(f"  ✓ Applied in {exec_time:.0f}ms", fg="green")
 
-            if success:
-                click.secho(f"  ✓ Applied in {exec_time:.0f}ms", fg="green")
-                # Show any NOTICE messages from the output
-                for line in output.split("\n"):
-                    if "NOTICE:" in line or "✓" in line:
-                        notice = line.split("NOTICE:")[-1].strip()
-                        if notice:
-                            click.echo(f"    {notice}")
-            else:
-                click.secho(f"  ✗ Failed", fg="red")
-                click.echo()
-                click.secho("Error output:", fg="red")
-                click.secho(output, fg="red")
-                all_success = False
-                break
-
-            click.echo()
-
-        # Summary
-        click.echo("=" * 60)
-        if all_success:
-            click.secho(f"✓ All migrations applied successfully", fg="green")
-            click.echo(f"  Total time: {total_time:.0f}ms")
-        else:
-            click.secho(f"✗ Migration failed", fg="red")
+        except Exception as e:
+            click.secho(f"  ✗ Failed: {e}", fg="red")
             raise click.Abort()
 
-    finally:
-        await db.disconnect()
+        click.echo()
+
+    click.echo("=" * 60)
+    click.secho("✓ All migrations applied", fg="green")
+    click.echo(f"  Total time: {total_time:.0f}ms")
+    click.echo()
+    click.echo("Next: verify with 'rem db diff'")
 
 
 @click.command()
@@ -382,7 +333,7 @@ def rebuild_cache(connection: str | None):
 
 @click.command()
 @click.argument("file_path", type=click.Path(exists=True, path_type=Path))
-@click.option("--user-id", default=None, help="User ID for loaded data (default: from settings)")
+@click.option("--user-id", default=None, help="User ID to scope data privately (default: public/shared)")
 @click.option("--dry-run", is_flag=True, help="Show what would be loaded without loading")
 def load(file_path: Path, user_id: str | None, dry_run: bool):
     """
@@ -397,17 +348,13 @@ def load(file_path: Path, user_id: str | None, dry_run: bool):
 
     Examples:
         rem db load rem/tests/data/graph_seed.yaml
-        rem db load data.yaml --user-id my-user
+        rem db load data.yaml --user-id my-user  # Private to user
         rem db load data.yaml --dry-run
     """
-    from ...settings import settings
-
-    # Resolve user_id from settings if not provided
-    effective_user_id = user_id or settings.test.effective_user_id
-    asyncio.run(_load_async(file_path, effective_user_id, dry_run))
+    asyncio.run(_load_async(file_path, user_id, dry_run))
 
 
-async def _load_async(file_path: Path, user_id: str, dry_run: bool):
+async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
     """Async implementation of load command."""
     import yaml
     from ...models.core.inline_edge import InlineEdge
@@ -415,7 +362,8 @@ async def _load_async(file_path: Path, user_id: str, dry_run: bool):
     from ...services.postgres import get_postgres_service
 
     logger.info(f"Loading data from: {file_path}")
-    logger.info(f"User ID: {user_id}")
+    scope_msg = f"user: {user_id}" if user_id else "public"
+    logger.info(f"Scope: {scope_msg}")
 
     # Load YAML file
     with open(file_path) as f:
@@ -489,11 +437,12 @@ async def _load_async(file_path: Path, user_id: str, dry_run: bool):
             model_class = MODEL_MAP[table_name]  # Type is inferred from MODEL_MAP
 
             for row_data in rows:
-                # Add user_id and tenant_id if not already present
-                # This allows seed files to specify explicit owners
-                if "user_id" not in row_data:
+                # Add user_id and tenant_id only if explicitly provided
+                # Default is public (None) - data is shared/visible to all
+                # Pass --user-id to scope data privately to a specific user
+                if "user_id" not in row_data and user_id is not None:
                     row_data["user_id"] = user_id
-                if "tenant_id" not in row_data:
+                if "tenant_id" not in row_data and user_id is not None:
                     row_data["tenant_id"] = row_data.get("user_id", user_id)
 
                 # Convert graph_edges to InlineEdge format if present
@@ -531,9 +480,271 @@ async def _load_async(file_path: Path, user_id: str, dry_run: bool):
         await pg.disconnect()
 
 
+@click.command()
+@click.option(
+    "--check",
+    is_flag=True,
+    help="Exit with non-zero status if drift detected (for CI)",
+)
+@click.option(
+    "--generate",
+    is_flag=True,
+    help="Generate incremental migration file from diff",
+)
+@click.option(
+    "--strategy",
+    "-s",
+    type=click.Choice(["additive", "full", "safe"]),
+    default="additive",
+    help="Migration strategy: additive (no drops, default), full (all changes), safe (additive + type widenings)",
+)
+@click.option(
+    "--models",
+    "-m",
+    type=click.Path(exists=True, path_type=Path),
+    default=None,
+    help="Directory containing Pydantic models (default: auto-detect)",
+)
+@click.option(
+    "--output-dir",
+    "-o",
+    type=click.Path(path_type=Path),
+    default=None,
+    help="Output directory for generated migration (default: sql/migrations)",
+)
+@click.option(
+    "--message",
+    default="schema_update",
+    help="Migration message/description (used in filename)",
+)
+def diff(
+    check: bool,
+    generate: bool,
+    strategy: str,
+    models: Path | None,
+    output_dir: Path | None,
+    message: str,
+):
+    """
+    Compare database schema against Pydantic models.
+
+    Uses Alembic autogenerate to detect differences between:
+    - Your Pydantic models (the target schema)
+    - The current database (what's actually deployed)
+
+    Strategies:
+        additive  Only ADD columns/tables/indexes (safe, no data loss) [default]
+        full      All changes including DROPs (use with caution)
+        safe      Additive + safe column type changes (widenings only)
+
+    Examples:
+        rem db diff                        # Show additive changes only
+        rem db diff --strategy full        # Show all changes including drops
+        rem db diff --generate             # Create migration file
+        rem db diff --check                # CI mode: exit 1 if drift
+
+    Workflow:
+        1. Develop locally, modify Pydantic models
+        2. Run 'rem db diff' to see changes
+        3. Run 'rem db diff --generate' to create migration
+        4. Review generated SQL, then 'rem db apply <file>'
+    """
+    asyncio.run(_diff_async(check, generate, strategy, models, output_dir, message))
+
+
+async def _diff_async(
+    check: bool,
+    generate: bool,
+    strategy: str,
+    models: Path | None,
+    output_dir: Path | None,
+    message: str,
+):
+    """Async implementation of diff command."""
+    from ...services.postgres.diff_service import DiffService
+
+    click.echo()
+    click.echo("REM Schema Diff")
+    click.echo("=" * 60)
+    click.echo(f"Strategy: {strategy}")
+
+    # Initialize diff service
+    diff_service = DiffService(models_dir=models, strategy=strategy)
+
+    try:
+        # Compute diff
+        click.echo("Comparing Pydantic models against database...")
+        click.echo()
+
+        result = diff_service.compute_diff()
+
+        if not result.has_changes:
+            click.secho("✓ No schema drift detected", fg="green")
+            click.echo("  Database matches Pydantic models")
+            if result.filtered_count > 0:
+                click.echo()
+                click.secho(f"  ({result.filtered_count} destructive change(s) hidden by '{strategy}' strategy)", fg="yellow")
+                click.echo("  Use --strategy full to see all changes")
+            return
+
+        # Show changes
+        click.secho(f"⚠ Schema drift detected: {result.change_count} change(s)", fg="yellow")
+        if result.filtered_count > 0:
+            click.secho(f"   ({result.filtered_count} destructive change(s) hidden by '{strategy}' strategy)", fg="yellow")
+        click.echo()
+        click.echo("Changes:")
+        for line in result.summary:
+            if line.startswith("+"):
+                click.secho(f"  {line}", fg="green")
+            elif line.startswith("-"):
+                click.secho(f"  {line}", fg="red")
+            elif line.startswith("~"):
+                click.secho(f"  {line}", fg="yellow")
+            else:
+                click.echo(f"  {line}")
+        click.echo()
+
+        # Generate migration if requested
+        if generate:
+            # Determine output directory
+            if output_dir is None:
+                import importlib.resources
+                try:
+                    sql_ref = importlib.resources.files("rem") / "sql" / "migrations"
+                    output_dir = Path(str(sql_ref))
+                except AttributeError:
+                    import rem
+                    package_dir = Path(rem.__file__).parent.parent
+                    output_dir = package_dir / "sql" / "migrations"
+
+            click.echo(f"Generating migration to: {output_dir}")
+            migration_path = diff_service.generate_migration_file(output_dir, message)
+
+            if migration_path:
+                click.secho(f"✓ Migration generated: {migration_path.name}", fg="green")
+                click.echo()
+                click.echo("Next steps:")
+                click.echo("  1. Review the generated SQL file")
+                click.echo("  2. Run: rem db apply <file>")
+            else:
+                click.echo("No migration file generated (no changes)")
+
+        # CI check mode
+        if check:
+            click.echo()
+            click.secho("✗ Schema drift detected (--check mode)", fg="red")
+            raise SystemExit(1)
+
+    except SystemExit:
+        raise
+    except Exception as e:
+        click.secho(f"✗ Error: {e}", fg="red")
+        logger.exception("Diff failed")
+        raise click.Abort()
+
+
+@click.command()
+@click.argument("sql_file", type=click.Path(exists=True, path_type=Path))
+@click.option(
+    "--log/--no-log",
+    default=True,
+    help="Log migration to rem_migrations table (default: yes)",
+)
+@click.option(
+    "--dry-run",
+    is_flag=True,
+    help="Show SQL that would be executed without running it",
+)
+def apply(sql_file: Path, log: bool, dry_run: bool):
+    """
+    Apply a SQL file directly to the database.
+
+    This is the simple, code-as-source-of-truth approach:
+    - Pydantic models define the schema
+    - `rem db diff` detects drift
+    - `rem db diff --generate` creates migration SQL
+    - `rem db apply <file>` runs it
+
+    Examples:
+        rem db apply migrations/004_add_field.sql
+        rem db apply --dry-run migrations/004_add_field.sql
+        rem db apply --no-log migrations/004_add_field.sql
+    """
+    asyncio.run(_apply_async(sql_file, log, dry_run))
+
+
+async def _apply_async(sql_file: Path, log: bool, dry_run: bool):
+    """Async implementation of apply command."""
+    from ...services.postgres import get_postgres_service
+
+    click.echo()
+    click.echo(f"Applying: {sql_file.name}")
+    click.echo("=" * 60)
+
+    # Read SQL content
+    sql_content = sql_file.read_text(encoding="utf-8")
+
+    if dry_run:
+        click.echo()
+        click.echo("SQL to execute (dry run):")
+        click.echo("-" * 40)
+        click.echo(sql_content)
+        click.echo("-" * 40)
+        click.echo()
+        click.secho("Dry run - no changes made", fg="yellow")
+        return
+
+    # Execute SQL
+    db = get_postgres_service()
+    if not db:
+        click.secho("✗ Could not connect to database", fg="red")
+        raise click.Abort()
+
+    start_time = time.time()
+
+    try:
+        import psycopg
+        from ...settings import settings
+
+        conn_str = settings.postgres.connection_string
+
+        with psycopg.connect(conn_str) as conn:
+            with conn.cursor() as cur:
+                cur.execute(sql_content)
+            conn.commit()
+
+            # Log to rem_migrations if requested
+            if log:
+                checksum = calculate_checksum(sql_file)
+                with conn.cursor() as cur:
+                    cur.execute(
+                        """
+                        INSERT INTO rem_migrations (name, type, checksum, applied_by)
+                        VALUES (%s, 'diff', %s, CURRENT_USER)
+                        ON CONFLICT (name) DO UPDATE SET
+                            applied_at = CURRENT_TIMESTAMP,
+                            checksum = EXCLUDED.checksum
+                        """,
+                        (sql_file.name, checksum[:16]),
+                    )
+                conn.commit()
+
+        execution_time = (time.time() - start_time) * 1000
+        click.secho(f"✓ Applied successfully in {execution_time:.0f}ms", fg="green")
+
+        if log:
+            click.echo(f"  Logged to rem_migrations as '{sql_file.name}'")
+
+    except Exception as e:
+        click.secho(f"✗ Failed: {e}", fg="red")
+        raise click.Abort()
+
+
 def register_commands(db_group):
     """Register all db commands."""
     db_group.add_command(migrate)
     db_group.add_command(status)
     db_group.add_command(rebuild_cache, name="rebuild-cache")
     db_group.add_command(load)
+    db_group.add_command(diff)
+    db_group.add_command(apply)

rem/cli/commands/process.py

@@ -12,12 +12,12 @@ from rem.services.content import ContentService
 
 @click.command(name="ingest")
 @click.argument("file_path", type=click.Path(exists=True))
-@click.option("--user-id", required=True, help="User ID to own the file")
+@click.option("--user-id", default=None, help="User ID to scope file privately (default: public/shared)")
 @click.option("--category", help="Optional file category")
 @click.option("--tags", help="Optional comma-separated tags")
 def process_ingest(
     file_path: str,
-    user_id: str,
+    user_id: str | None,
     category: str | None,
     tags: str | None,
 ):
@@ -32,8 +32,9 @@ def process_ingest(
     5. Creates a File entity record.
 
     Examples:
-        rem process ingest sample.pdf --user-id user-123
-        rem process ingest contract.docx --user-id user-123 --category legal --tags contract,2023
+        rem process ingest sample.pdf
+        rem process ingest contract.docx --category legal --tags contract,2023
+        rem process ingest agent.yaml  # Auto-detects kind=agent, saves to schemas table
     """
     import asyncio
     from ...services.content import ContentService
@@ -56,7 +57,8 @@ def process_ingest(
 
             tag_list = tags.split(",") if tags else None
 
-            logger.info(f"Ingesting file: {file_path} for user: {user_id}")
+            scope_msg = f"user: {user_id}" if user_id else "public"
+            logger.info(f"Ingesting file: {file_path} ({scope_msg})")
             result = await service.ingest_file(
                 file_uri=file_path,
                 user_id=user_id,
@@ -65,11 +67,15 @@ def process_ingest(
                 is_local_server=True, # CLI is local
             )
 
-            if result.get("processing_status") == "completed":
-                logger.success(f"File ingested successfully: {result['file_name']}")
+            # Handle schema ingestion (agents/evaluators)
+            if result.get("schema_name"):
+                logger.success(f"Schema ingested: {result['schema_name']} (kind={result.get('kind', 'agent')})")
+                logger.info(f"Version: {result.get('version', '1.0.0')}")
+            # Handle file ingestion
+            elif result.get("processing_status") == "completed":
+                logger.success(f"File ingested: {result['file_name']}")
                 logger.info(f"File ID: {result['file_id']}")
                 logger.info(f"Resources created: {result['resources_created']}")
-                logger.info(f"Status: {result['processing_status']}")
             else:
                 logger.error(f"Ingestion failed: {result.get('message', 'Unknown error')}")
                 sys.exit(1)