remdb 0.3.0__py3-none-any.whl → 0.3.114__py3-none-any.whl

rem/cli/commands/experiments.py

@@ -578,8 +578,7 @@ def run(
     from rem.services.git import GitService
     from rem.services.phoenix import PhoenixClient
     from rem.agentic.providers.phoenix import create_evaluator_from_schema
-    from datetime import datetime
-    import pandas as pd
+    from rem.utils.date_utils import utc_now, to_iso, format_timestamp_for_experiment
     import os
 
     try:
@@ -615,36 +614,22 @@ def run(
             click.echo(f"  Mode: DRY RUN (no data will be saved)")
         click.echo()
 
-        # Load agent schema from Git or filesystem
+        # Load agent schema using centralized schema loader
         agent_name = config.agent_schema_ref.name
         agent_version = config.agent_schema_ref.version
 
         click.echo(f"Loading agent schema: {agent_name} (version: {agent_version or 'latest'})")
 
-        # Try Git first, fallback to filesystem
-        agent_schema = None
-        try:
-            git_svc = GitService()
-            agent_schema = git_svc.load_schema(agent_name, version=agent_version)
-            click.echo(f"✓ Loaded agent schema from Git")
-        except Exception as e:
-            logger.debug(f"Git not available, trying filesystem: {e}")
+        from rem.utils.schema_loader import load_agent_schema
 
-            # Fallback to local filesystem
-            from rem.services.fs import FS
-            fs = FS()
-
-            schema_path = f"schemas/agents/{agent_name}.yaml"
-            try:
-                agent_schema = fs.read(schema_path)
-                click.echo(f"✓ Loaded agent schema from filesystem")
-            except Exception as fs_error:
-                logger.error(f"Failed to load agent schema: Git: {e}, FS: {fs_error}")
-                click.echo(f"Error: Could not load agent schema '{agent_name}'")
-                click.echo(f"  Tried Git: {e}")
-                click.echo(f"  Tried filesystem: {schema_path}")
-                click.echo(f"  Make sure the schema exists")
-                raise click.Abort()
+        try:
+            agent_schema = load_agent_schema(agent_name)
+            click.echo(f"✓ Loaded agent schema: {agent_name}")
+        except FileNotFoundError as e:
+            logger.error(f"Failed to load agent schema: {e}")
+            click.echo(f"Error: Could not load agent schema '{agent_name}'")
+            click.echo(f"  {e}")
+            raise click.Abort()
 
         # Create agent function from schema
         from rem.agentic.providers.pydantic_ai import create_agent
@@ -683,73 +668,85 @@ def run(
                 return {"output": serialized}
             return serialized if isinstance(serialized, dict) else {"output": str(serialized)}
 
-        # Load evaluator schema
+        # Load evaluator schema using centralized schema loader
         evaluator_name = config.evaluator_schema_ref.name
         evaluator_version = config.evaluator_schema_ref.version
 
-        # Resolve evaluator path (evaluators are organized by agent name)
-        evaluator_schema_path = f"rem/schemas/evaluators/{agent_name}/{evaluator_name}.yaml"
-
         click.echo(f"Loading evaluator: {evaluator_name} for agent {agent_name}")
 
-        try:
-            evaluator_fn = create_evaluator_from_schema(
-                evaluator_schema_path=evaluator_schema_path,
-                model_name=None,  # Use default from schema
-            )
-            click.echo(f"✓ Loaded evaluator schema")
-        except Exception as e:
-            logger.warning(f"Failed to load evaluator: {e}")
-            click.echo(f"Error: Could not load evaluator schema")
-            click.echo(f"  Path: {evaluator_schema_path}")
-            click.echo(f"  Make sure the schema exists")
+        # Try multiple evaluator path patterns (agent-specific, then generic)
+        evaluator_paths_to_try = [
+            f"{agent_name}/{evaluator_name}",  # e.g., hello-world/default
+            f"{agent_name}-{evaluator_name}",  # e.g., hello-world-default
+            evaluator_name,                     # e.g., default (generic)
+        ]
+
+        evaluator_fn = None
+        evaluator_load_error = None
+
+        for evaluator_path in evaluator_paths_to_try:
+            try:
+                evaluator_fn = create_evaluator_from_schema(
+                    evaluator_schema_path=evaluator_path,
+                    model_name=None,  # Use default from schema
+                )
+                click.echo(f"✓ Loaded evaluator schema: {evaluator_path}")
+                break
+            except FileNotFoundError as e:
+                evaluator_load_error = e
+                logger.debug(f"Evaluator not found at {evaluator_path}: {e}")
+                continue
+            except Exception as e:
+                evaluator_load_error = e
+                logger.warning(f"Failed to load evaluator from {evaluator_path}: {e}")
+                continue
+
+        if evaluator_fn is None:
+            click.echo(f"Error: Could not load evaluator schema '{evaluator_name}'")
+            click.echo(f"  Tried paths: {evaluator_paths_to_try}")
+            if evaluator_load_error:
+                click.echo(f"  Last error: {evaluator_load_error}")
             raise click.Abort()
 
-        # Load dataset
+        # Load dataset using Polars
+        import polars as pl
+
         click.echo(f"Loading dataset: {list(config.datasets.keys())[0]}")
         dataset_ref = list(config.datasets.values())[0]
 
         if dataset_ref.location.value == "git":
-            # Load from Git
+            # Load from Git (local filesystem)
             dataset_path = Path(base_path) / name / dataset_ref.path
             if not dataset_path.exists():
                 click.echo(f"Error: Dataset not found: {dataset_path}")
                 raise click.Abort()
 
             if dataset_ref.format == "csv":
-                dataset_df = pd.read_csv(dataset_path)
+                dataset_df = pl.read_csv(dataset_path)
             elif dataset_ref.format == "parquet":
-                dataset_df = pd.read_parquet(dataset_path)
+                dataset_df = pl.read_parquet(dataset_path)
             elif dataset_ref.format == "jsonl":
-                dataset_df = pd.read_json(dataset_path, lines=True)
+                dataset_df = pl.read_ndjson(dataset_path)
             else:
                 click.echo(f"Error: Format '{dataset_ref.format}' not yet supported")
                 raise click.Abort()
         elif dataset_ref.location.value in ["s3", "hybrid"]:
             # Load from S3 using FS provider
             from rem.services.fs import FS
+            from io import BytesIO
 
             fs = FS()
 
             try:
                 if dataset_ref.format == "csv":
                     content = fs.read(dataset_ref.path)
-                    from io import StringIO
-                    dataset_df = pd.read_csv(StringIO(content))
+                    dataset_df = pl.read_csv(BytesIO(content.encode() if isinstance(content, str) else content))
                 elif dataset_ref.format == "parquet":
-                    # For parquet, we need binary read
-                    import tempfile
-                    with tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) as tmp:
-                        tmp_path = tmp.name
-                        # Download via FS
-                        content_bytes = fs.read(dataset_ref.path)
-                        tmp.write(content_bytes)
-                    dataset_df = pd.read_parquet(tmp_path)
-                    Path(tmp_path).unlink()  # Clean up temp file
+                    content_bytes = fs.read(dataset_ref.path)
+                    dataset_df = pl.read_parquet(BytesIO(content_bytes if isinstance(content_bytes, bytes) else content_bytes.encode()))
                 elif dataset_ref.format == "jsonl":
                     content = fs.read(dataset_ref.path)
-                    from io import StringIO
-                    dataset_df = pd.read_json(StringIO(content), lines=True)
+                    dataset_df = pl.read_ndjson(BytesIO(content.encode() if isinstance(content, str) else content))
                 else:
                     click.echo(f"Error: Format '{dataset_ref.format}' not yet supported")
                     raise click.Abort()
@@ -793,13 +790,13 @@ def run(
 
             client = PhoenixClient(config=phoenix_config)
 
-            experiment_name = f"{config.name}-{datetime.now().strftime('%Y%m%d-%H%M%S')}"
+            experiment_name = f"{config.name}-{format_timestamp_for_experiment()}"
 
             click.echo(f"\n⏳ Running experiment: {experiment_name}")
             click.echo(f"   This may take several minutes...")
 
             experiment = client.run_experiment(
-                dataset=dataset_df,  # type: ignore[arg-type]
+                dataset=dataset_df,
                 task=task_fn,
                 evaluators=[evaluator_fn],
                 experiment_name=experiment_name,
@@ -809,12 +806,15 @@ def run(
                     "evaluator": config.evaluator_schema_ref.name,
                     "experiment_config": config.name,
                     **config.metadata
-                }
+                },
+                # Smart column detection for DataFrame -> Phoenix Dataset conversion
+                input_keys=["input"] if "input" in dataset_df.columns else None,
+                output_keys=["expected_output"] if "expected_output" in dataset_df.columns else None,
             )
 
             # Update experiment status
             config.status = ExperimentStatus.COMPLETED
-            config.last_run_at = datetime.now()
+            config.last_run_at = utc_now()
             if not version:  # Only save if not loading from Git
                 config.save(base_path)
 
@@ -835,7 +835,7 @@ def run(
                         "agent": config.agent_schema_ref.name,
                         "evaluator": config.evaluator_schema_ref.name,
                         "dataset_size": len(dataset_df),
-                        "completed_at": datetime.now().isoformat(),
+                        "completed_at": to_iso(utc_now()),
                         "phoenix_url": getattr(experiment, "url", None),
                         "task_runs": len(exp_data.get("task_runs", [])),
                     }
@@ -1015,20 +1015,24 @@ def dataset_add(
             --output-keys expected_label,expected_type
     """
     from rem.services.phoenix import PhoenixClient
-    import pandas as pd
+    import polars as pl
 
     try:
         client = PhoenixClient()
 
-        # Load CSV
-        df = pd.read_csv(from_csv)
+        # Load CSV with Polars
+        df = pl.read_csv(from_csv)
+        records = df.to_dicts()
 
         # Extract data
-        inputs = cast(list[dict[str, Any]], df[input_keys.split(",")].to_dict("records"))
-        outputs = cast(list[dict[str, Any]], df[output_keys.split(",")].to_dict("records"))
+        input_cols = input_keys.split(",")
+        output_cols = output_keys.split(",")
+        inputs = [{k: row.get(k) for k in input_cols} for row in records]
+        outputs = [{k: row.get(k) for k in output_cols} for row in records]
         metadata = None
         if metadata_keys:
-            metadata = cast(list[dict[str, Any]], df[metadata_keys.split(",")].to_dict("records"))
+            meta_cols = metadata_keys.split(",")
+            metadata = [{k: row.get(k) for k in meta_cols} for row in records]
 
         # Add to dataset
         dataset = client.add_examples_to_dataset(
@@ -1269,12 +1273,12 @@ def trace_list(
         rem experiments trace list --project rem-agents --days 7 --limit 50
     """
     from rem.services.phoenix import PhoenixClient
-    from datetime import datetime, timedelta
+    from rem.utils.date_utils import days_ago
 
     try:
         client = PhoenixClient()
 
-        start_time = datetime.now() - timedelta(days=days)
+        start_time = days_ago(days)
 
         traces_df = client.get_traces(
             project_name=project,

rem/cli/commands/process.py

@@ -12,12 +12,12 @@ from rem.services.content import ContentService
 
 @click.command(name="ingest")
 @click.argument("file_path", type=click.Path(exists=True))
-@click.option("--user-id", required=True, help="User ID to own the file")
+@click.option("--user-id", default=None, help="User ID to scope file privately (default: public/shared)")
 @click.option("--category", help="Optional file category")
 @click.option("--tags", help="Optional comma-separated tags")
 def process_ingest(
     file_path: str,
-    user_id: str,
+    user_id: str | None,
     category: str | None,
     tags: str | None,
 ):
@@ -32,8 +32,9 @@ def process_ingest(
     5. Creates a File entity record.
 
     Examples:
-        rem process ingest sample.pdf --user-id user-123
-        rem process ingest contract.docx --user-id user-123 --category legal --tags contract,2023
+        rem process ingest sample.pdf
+        rem process ingest contract.docx --category legal --tags contract,2023
+        rem process ingest agent.yaml  # Auto-detects kind=agent, saves to schemas table
     """
     import asyncio
     from ...services.content import ContentService
@@ -56,7 +57,8 @@ def process_ingest(
 
             tag_list = tags.split(",") if tags else None
 
-            logger.info(f"Ingesting file: {file_path} for user: {user_id}")
+            scope_msg = f"user: {user_id}" if user_id else "public"
+            logger.info(f"Ingesting file: {file_path} ({scope_msg})")
             result = await service.ingest_file(
                 file_uri=file_path,
                 user_id=user_id,
@@ -65,11 +67,15 @@ def process_ingest(
                 is_local_server=True, # CLI is local
             )
 
-            if result.get("processing_status") == "completed":
-                logger.success(f"File ingested successfully: {result['file_name']}")
+            # Handle schema ingestion (agents/evaluators)
+            if result.get("schema_name"):
+                logger.success(f"Schema ingested: {result['schema_name']} (kind={result.get('kind', 'agent')})")
+                logger.info(f"Version: {result.get('version', '1.0.0')}")
+            # Handle file ingestion
+            elif result.get("processing_status") == "completed":
+                logger.success(f"File ingested: {result['file_name']}")
                 logger.info(f"File ID: {result['file_id']}")
                 logger.info(f"Resources created: {result['resources_created']}")
-                logger.info(f"Status: {result['processing_status']}")
             else:
                 logger.error(f"Ingestion failed: {result.get('message', 'Unknown error')}")
                 sys.exit(1)
@@ -192,15 +198,13 @@ def process_uri(uri: str, output: str, save: str | None):
 
 
 @click.command(name="files")
-@click.option("--tenant-id", required=True, help="Tenant ID")
-@click.option("--user-id", help="Filter by user ID")
+@click.option("--user-id", default=None, help="User ID (default: from settings)")
 @click.option("--status", type=click.Choice(["pending", "processing", "completed", "failed"]), help="Filter by status")
 @click.option("--extractor", help="Run files through custom extractor (e.g., cv-parser-v1)")
 @click.option("--limit", type=int, help="Max files to process")
 @click.option("--provider", help="Optional LLM provider override")
 @click.option("--model", help="Optional model override")
 def process_files(
-    tenant_id: str,
     user_id: Optional[str],
     status: Optional[str],
     extractor: Optional[str],
@@ -217,19 +221,22 @@ def process_files(
 
         \b
         # List completed files
-        rem process files --tenant-id acme-corp --status completed
+        rem process files --status completed
 
         \b
         # Extract from CV files
-        rem process files --tenant-id acme-corp --extractor cv-parser-v1 --limit 10
+        rem process files --extractor cv-parser-v1 --limit 10
 
         \b
         # Extract with provider override
-        rem process files --tenant-id acme-corp --extractor contract-analyzer-v1 \\
+        rem process files --extractor contract-analyzer-v1 \\
             --provider anthropic --model claude-sonnet-4-5
     """
+    from ...settings import settings
+    effective_user_id = user_id or settings.test.effective_user_id
+
     logger.warning("Not implemented yet")
-    logger.info(f"Would process files for tenant: {tenant_id}")
+    logger.info(f"Would process files for user: {effective_user_id}")
 
     if user_id:
         logger.info(f"Filter: user_id={user_id}")

rem/cli/commands/scaffold.py

@@ -0,0 +1,47 @@
+"""
+Scaffold command - generate project structure for REM-based applications.
+
+TODO: Implement this command to generate:
+- my_app/main.py (entry point with create_app)
+- my_app/models.py (example CoreModel subclass)
+- my_app/routers/ (example FastAPI router)
+- schemas/agents/ (example agent schema)
+- schemas/evaluators/ (example evaluator)
+- sql/migrations/ (empty migrations directory)
+- pyproject.toml (with remdb dependency)
+- README.md (basic usage instructions)
+
+Usage:
+    rem scaffold my-app
+    rem scaffold my-app --with-examples  # Include example models/routers/tools
+"""
+
+import click
+
+
+@click.command()
+@click.argument("name")
+@click.option("--with-examples", is_flag=True, help="Include example code")
+def scaffold(name: str, with_examples: bool) -> None:
+    """
+    Generate a new REM-based project structure.
+
+    NAME is the project directory name to create.
+    """
+    click.echo(f"TODO: Scaffold command not yet implemented")
+    click.echo(f"Would create project: {name}")
+    click.echo(f"With examples: {with_examples}")
+    click.echo()
+    click.echo("For now, manually create this structure:")
+    click.echo(f"""
+{name}/
+├── {name.replace('-', '_')}/
+│   ├── main.py           # Entry point (create_app + extensions)
+│   ├── models.py         # Custom models (inherit CoreModel)
+│   └── routers/          # Custom FastAPI routers
+├── schemas/
+│   ├── agents/           # Custom agent YAML schemas
+│   └── evaluators/       # Custom evaluator schemas
+├── sql/migrations/       # Custom SQL migrations
+└── pyproject.toml
+""")

rem/cli/commands/schema.py

@@ -8,6 +8,7 @@ Usage:
 """
 
 import asyncio
+import importlib
 from pathlib import Path
 
 import click
@@ -17,66 +18,113 @@ from ...settings import settings
 from ...services.postgres.schema_generator import SchemaGenerator
 
 
+def _import_model_modules() -> list[str]:
+    """
+    Import modules specified in MODELS__IMPORT_MODULES setting.
+
+    This ensures downstream models decorated with @rem.register_model
+    are registered before schema generation.
+
+    Returns:
+        List of successfully imported module names
+    """
+    imported = []
+    for module_name in settings.models.module_list:
+        try:
+            importlib.import_module(module_name)
+            imported.append(module_name)
+            logger.debug(f"Imported model module: {module_name}")
+        except ImportError as e:
+            logger.warning(f"Failed to import model module '{module_name}': {e}")
+            click.echo(
+                click.style(f"  ⚠ Could not import '{module_name}': {e}", fg="yellow"),
+                err=True,
+            )
+    return imported
+
+
 @click.command()
-@click.option(
-    "--models",
-    "-m",
-    required=True,
-    type=click.Path(exists=True, path_type=Path),
-    help="Directory containing Pydantic models",
-)
 @click.option(
     "--output",
     "-o",
     type=click.Path(path_type=Path),
-    default="install_models.sql",
-    help="Output SQL file (default: install_models.sql)",
+    default="002_install_models.sql",
+    help="Output SQL file (default: 002_install_models.sql)",
 )
 @click.option(
     "--output-dir",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Base output directory (default: {settings.sql_dir})",
+    help=f"Base output directory (default: {settings.sql_dir}/migrations)",
 )
-def generate(models: Path, output: Path, output_dir: Path | None):
+def generate(output: Path, output_dir: Path | None):
     """
-    Generate database schema from Pydantic models.
+    Generate database schema from registered Pydantic models.
 
-    Scans the specified directory for Pydantic models and generates:
+    Uses the model registry (core models + user-registered models) to generate:
     - CREATE TABLE statements
     - Embeddings tables (embeddings_<table>)
     - KV_STORE triggers for cache maintenance
     - Indexes (foreground only)
 
-    Output is written to src/rem/sql/install_models.sql by default.
+    Output is written to src/rem/sql/migrations/002_install_models.sql by default.
 
     Example:
-        rem db schema generate --models src/rem/models/entities
+        rem db schema generate
+
+    To register custom models in downstream apps:
+
+    1. Create models with @rem.register_model decorator:
+
+        # models/__init__.py
+        import rem
+        from rem.models.core import CoreModel
+
+        @rem.register_model
+        class MyEntity(CoreModel):
+            name: str
+
+    2. Set MODELS__IMPORT_MODULES in your .env:
+
+        MODELS__IMPORT_MODULES=models
+
+    3. Run schema generation:
+
+        rem db schema generate
 
     This creates:
-    - src/rem/sql/install_models.sql - Entity tables and triggers
+    - src/rem/sql/migrations/002_install_models.sql - Entity tables and triggers
     - src/rem/sql/background_indexes.sql - HNSW indexes (apply after data load)
 
-    After generation, apply with:
-        rem db migrate
+    After generation, verify with:
+        rem db diff
     """
-    click.echo(f"Discovering models in {models}")
+    from ...registry import get_model_registry
+
+    # Import downstream model modules to trigger @rem.register_model decorators
+    imported_modules = _import_model_modules()
+    if imported_modules:
+        click.echo(f"Imported model modules: {', '.join(imported_modules)}")
+
+    registry = get_model_registry()
+    models = registry.get_models(include_core=True)
+    click.echo(f"Generating schema from {len(models)} registered models")
 
-    # Use settings.sql_dir if not provided
-    actual_output_dir = output_dir or Path(settings.sql_dir)
+    # Default to migrations directory
+    actual_output_dir = output_dir or Path(settings.sql_dir) / "migrations"
     generator = SchemaGenerator(output_dir=actual_output_dir)
 
-    # Generate schema
+    # Generate schema from registry
     try:
-        schema_sql = asyncio.run(generator.generate_from_directory(models, output_file=output.name))
+        schema_sql = asyncio.run(generator.generate_from_registry(output_file=output.name))
 
         click.echo(f"✓ Schema generated: {len(generator.schemas)} tables")
         click.echo(f"✓ Written to: {actual_output_dir / output.name}")
 
-        # Generate background indexes
+        # Generate background indexes in parent sql dir
         background_indexes = generator.generate_background_indexes()
         if background_indexes:
-            bg_file = actual_output_dir / "background_indexes.sql"
+            bg_file = Path(settings.sql_dir) / "background_indexes.sql"
             bg_file.write_text(background_indexes)
             click.echo(f"✓ Background indexes: {bg_file}")
 
@@ -94,48 +142,46 @@ def generate(models: Path, output: Path, output_dir: Path | None):
 
 
 @click.command()
-@click.option(
-    "--models",
-    "-m",
-    required=True,
-    type=click.Path(exists=True, path_type=Path),
-    help="Directory containing Pydantic models",
-)
-def validate(models: Path):
+def validate():
     """
-    Validate Pydantic models for schema generation.
+    Validate registered Pydantic models for schema generation.
 
     Checks:
-    - Models can be loaded
+    - Models can be loaded from registry
     - Models have suitable entity_key fields
     - Fields with embeddings are properly configured
+
+    Set MODELS__IMPORT_MODULES to include custom models from downstream apps.
     """
-    click.echo(f"Validating models in {models}")
+    from ...registry import get_model_registry
 
-    generator = SchemaGenerator()
-    discovered = generator.discover_models(models)
+    # Import downstream model modules to trigger @rem.register_model decorators
+    imported_modules = _import_model_modules()
+    if imported_modules:
+        click.echo(f"Imported model modules: {', '.join(imported_modules)}")
 
-    if not discovered:
-        click.echo("✗ No models found", err=True)
-        raise click.Abort()
+    registry = get_model_registry()
+    models = registry.get_models(include_core=True)
+
+    click.echo(f"Validating {len(models)} registered models")
 
-    click.echo(f"✓ Discovered {len(discovered)} models")
+    if not models:
+        click.echo("✗ No models found in registry", err=True)
+        raise click.Abort()
 
+    generator = SchemaGenerator()
     errors: list[str] = []
     warnings: list[str] = []
 
-    for model_name, model in discovered.items():
-        table_name = generator.infer_table_name(model)
-        entity_key = generator.infer_entity_key_field(model)
+    for model_name, ext in models.items():
+        model = ext.model
+        table_name = ext.table_name or generator.infer_table_name(model)
+        entity_key = ext.entity_key_field or generator.infer_entity_key_field(model)
 
         # Check for entity_key
         if entity_key == "id":
             warnings.append(f"{model_name}: No natural key field, using 'id'")
 
-        # Check for embeddable fields
-        # TODO: Implement should_embed_field check
-        embeddable: list[str] = []  # Placeholder - needs implementation
-
         click.echo(f"  {model_name} -> {table_name} (key: {entity_key})")
 
     if warnings: