remdb 0.3.114__py3-none-any.whl → 0.3.127__py3-none-any.whl

rem/cli/commands/configure.py

@@ -405,9 +405,7 @@ def configure_command(install: bool, claude_desktop: bool, show: bool, edit: boo
 
                 # Create a context for the command and invoke it
                 ctx = click.Context(migrate)
-                ctx.invoke(migrate, install_only=False, models_only=False,
-                          background_indexes=False, connection=None,
-                          sql_dir=Path("rem/sql"))
+                ctx.invoke(migrate, background_indexes=False)
 
                 click.echo("✅ Database installation complete")
 

rem/cli/commands/db.py

@@ -126,29 +126,51 @@ def migrate(background_indexes: bool):
 async def _migrate_async(background_indexes: bool):
     """Async implementation of migrate command."""
     from ...settings import settings
+    from ...utils.sql_paths import (
+        get_package_sql_dir,
+        get_user_sql_dir,
+        list_all_migrations,
+    )
 
     click.echo()
     click.echo("REM Database Migration")
     click.echo("=" * 60)
 
-    # Find SQL directory
-    sql_dir = Path(settings.sql_dir)
-    migrations_dir = sql_dir / "migrations"
+    # Find package SQL directory
+    try:
+        package_sql_dir = get_package_sql_dir()
+        click.echo(f"Package SQL: {package_sql_dir}")
+    except FileNotFoundError as e:
+        click.secho(f"✗ {e}", fg="red")
+        raise click.Abort()
+
+    # Check for user migrations
+    user_sql_dir = get_user_sql_dir()
+    if user_sql_dir:
+        click.echo(f"User SQL: {user_sql_dir}")
+
+    # Get all migrations (package + user)
+    all_migrations = list_all_migrations()
+
+    if not all_migrations:
+        click.secho("✗ No migration files found", fg="red")
+        raise click.Abort()
 
-    click.echo(f"SQL Directory: {sql_dir}")
+    click.echo(f"Found {len(all_migrations)} migration(s)")
     click.echo()
 
-    # Standard migration files
-    migrations = [
-        (migrations_dir / "001_install.sql", "Core Infrastructure"),
-        (migrations_dir / "002_install_models.sql", "Entity Tables"),
-    ]
+    # Add background indexes if requested
+    migrations_to_apply = [(f, f.stem) for f in all_migrations]
 
     if background_indexes:
-        migrations.append((sql_dir / "background_indexes.sql", "Background Indexes"))
-
-    # Check files exist
-    for file_path, description in migrations:
+        bg_indexes = package_sql_dir / "background_indexes.sql"
+        if bg_indexes.exists():
+            migrations_to_apply.append((bg_indexes, "Background Indexes"))
+        else:
+            click.secho("⚠ background_indexes.sql not found, skipping", fg="yellow")
+
+    # Check all files exist (they should, but verify)
+    for file_path, description in migrations_to_apply:
         if not file_path.exists():
             click.secho(f"✗ {file_path.name} not found", fg="red")
             if "002" in file_path.name:
@@ -162,8 +184,8 @@ async def _migrate_async(background_indexes: bool):
     conn_str = settings.postgres.connection_string
     total_time = 0.0
 
-    for file_path, description in migrations:
-        click.echo(f"Applying: {description} ({file_path.name})")
+    for file_path, description in migrations_to_apply:
+        click.echo(f"Applying: {file_path.name}")
 
         sql_content = file_path.read_text(encoding="utf-8")
         start_time = time.time()
@@ -469,6 +491,13 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
     is_flag=True,
     help="Generate incremental migration file from diff",
 )
+@click.option(
+    "--strategy",
+    "-s",
+    type=click.Choice(["additive", "full", "safe"]),
+    default="additive",
+    help="Migration strategy: additive (no drops, default), full (all changes), safe (additive + type widenings)",
+)
 @click.option(
     "--models",
     "-m",
@@ -491,6 +520,7 @@ async def _load_async(file_path: Path, user_id: str | None, dry_run: bool):
 def diff(
     check: bool,
     generate: bool,
+    strategy: str,
     models: Path | None,
     output_dir: Path | None,
     message: str,
@@ -502,23 +532,30 @@ def diff(
     - Your Pydantic models (the target schema)
     - The current database (what's actually deployed)
 
+    Strategies:
+        additive  Only ADD columns/tables/indexes (safe, no data loss) [default]
+        full      All changes including DROPs (use with caution)
+        safe      Additive + safe column type changes (widenings only)
+
     Examples:
-        rem db diff                    # Show what would change
-        rem db diff --check            # CI mode: exit 1 if drift detected
-        rem db diff --generate         # Create migration file from diff
+        rem db diff                        # Show additive changes only
+        rem db diff --strategy full        # Show all changes including drops
+        rem db diff --generate             # Create migration file
+        rem db diff --check                # CI mode: exit 1 if drift
 
     Workflow:
         1. Develop locally, modify Pydantic models
         2. Run 'rem db diff' to see changes
         3. Run 'rem db diff --generate' to create migration
-        4. Review generated SQL, then 'rem db migrate'
+        4. Review generated SQL, then 'rem db apply <file>'
     """
-    asyncio.run(_diff_async(check, generate, models, output_dir, message))
+    asyncio.run(_diff_async(check, generate, strategy, models, output_dir, message))
 
 
 async def _diff_async(
     check: bool,
     generate: bool,
+    strategy: str,
     models: Path | None,
     output_dir: Path | None,
     message: str,
@@ -529,9 +566,10 @@ async def _diff_async(
     click.echo()
     click.echo("REM Schema Diff")
     click.echo("=" * 60)
+    click.echo(f"Strategy: {strategy}")
 
     # Initialize diff service
-    diff_service = DiffService(models_dir=models)
+    diff_service = DiffService(models_dir=models, strategy=strategy)
 
     try:
         # Compute diff
@@ -543,10 +581,16 @@ async def _diff_async(
         if not result.has_changes:
             click.secho("✓ No schema drift detected", fg="green")
             click.echo("  Database matches Pydantic models")
+            if result.filtered_count > 0:
+                click.echo()
+                click.secho(f"  ({result.filtered_count} destructive change(s) hidden by '{strategy}' strategy)", fg="yellow")
+                click.echo("  Use --strategy full to see all changes")
             return
 
         # Show changes
         click.secho(f"⚠ Schema drift detected: {result.change_count} change(s)", fg="yellow")
+        if result.filtered_count > 0:
+            click.secho(f"   ({result.filtered_count} destructive change(s) hidden by '{strategy}' strategy)", fg="yellow")
         click.echo()
         click.echo("Changes:")
         for line in result.summary:
@@ -581,7 +625,7 @@ async def _diff_async(
                 click.echo()
                 click.echo("Next steps:")
                 click.echo("  1. Review the generated SQL file")
-                click.echo("  2. Run: rem db migrate")
+                click.echo("  2. Run: rem db apply <file>")
             else:
                 click.echo("No migration file generated (no changes)")
 

rem/cli/commands/experiments.py

@@ -514,6 +514,159 @@ def show(name: str, base_path: Optional[str]):
         raise click.Abort()
 
 
+# =============================================================================
+# VIBES MODE HELPER
+# =============================================================================
+
+
+def _run_vibes_mode(
+    config: Any,
+    dataset_df: Any,
+    task_fn: Any,
+    base_path: str,
+    limit: Optional[int],
+    evaluator_schema_path: Path,
+) -> None:
+    """Run experiment in vibes mode - execute agent and export for AI evaluation.
+
+    Vibes mode runs the agent on each example and saves results to a JSONL file.
+    The AI assistant (e.g., Claude Code) then acts as the judge using the
+    evaluator schema to evaluate results.
+
+    Args:
+        config: ExperimentConfig object
+        dataset_df: Polars DataFrame with ground truth examples
+        task_fn: Function to run agent on each example
+        base_path: Base directory for experiments
+        limit: Optional limit on number of examples to process
+        evaluator_schema_path: Path to the evaluator schema YAML file
+    """
+    from rem.utils.date_utils import format_timestamp_for_experiment, utc_now, to_iso
+    import json
+
+    # Apply limit if specified
+    if limit:
+        dataset_df = dataset_df.head(limit)
+        click.echo(f"  (Limited to {limit} examples)")
+
+    # Create results directory
+    timestamp = format_timestamp_for_experiment()
+    results_dir = Path(base_path) / config.name / "results" / timestamp
+    results_dir.mkdir(parents=True, exist_ok=True)
+
+    click.echo(f"\n⏳ Running agent on {len(dataset_df)} examples...")
+    click.echo(f"   Results will be saved to: {results_dir}")
+    click.echo()
+
+    # Run agent on each example and collect results
+    results = []
+    records = dataset_df.to_dicts()
+
+    for i, record in enumerate(records, 1):
+        example_id = record.get("id", i)
+        click.echo(f"  [{i}/{len(records)}] Processing example {example_id}...", nl=False)
+
+        try:
+            # Prepare input for agent
+            input_text = record.get("text", record.get("input", record.get("query", "")))
+            example_input = {"query": input_text} if isinstance(input_text, str) else input_text
+
+            # Run agent
+            output = task_fn({"input": example_input})
+
+            result = {
+                "id": example_id,
+                "input": input_text,
+                "ground_truth": record.get("ground_truth", record.get("expected_output", "")),
+                "category": record.get("category", ""),
+                "agent_output": output,
+                "status": "success",
+            }
+            click.echo(" ✓")
+
+        except Exception as e:
+            result = {
+                "id": example_id,
+                "input": record.get("text", record.get("input", "")),
+                "ground_truth": record.get("ground_truth", record.get("expected_output", "")),
+                "category": record.get("category", ""),
+                "agent_output": None,
+                "status": "error",
+                "error": str(e),
+            }
+            click.echo(f" ✗ ({e})")
+
+        results.append(result)
+
+    # Save results to JSONL
+    results_file = results_dir / "vibes-results.jsonl"
+    with open(results_file, "w") as f:
+        for result in results:
+            f.write(json.dumps(result) + "\n")
+
+    # Copy evaluator schema to results dir for easy reference
+    import shutil
+    evaluator_copy = results_dir / "evaluator-schema.yaml"
+    shutil.copy(evaluator_schema_path, evaluator_copy)
+
+    # Save run metadata
+    run_info = {
+        "experiment": config.name,
+        "agent": config.agent_schema_ref.name,
+        "evaluator": config.evaluator_schema_ref.name,
+        "mode": "vibes",
+        "timestamp": timestamp,
+        "total_examples": len(records),
+        "successful": len([r for r in results if r["status"] == "success"]),
+        "failed": len([r for r in results if r["status"] == "error"]),
+        "completed_at": to_iso(utc_now()),
+    }
+
+    run_info_file = results_dir / "run-info.json"
+    with open(run_info_file, "w") as f:
+        json.dump(run_info, f, indent=2)
+
+    # Print summary and instructions
+    success_count = run_info["successful"]
+    fail_count = run_info["failed"]
+
+    click.echo(f"\n{'=' * 60}")
+    click.echo(f"VIBES MODE COMPLETE")
+    click.echo(f"{'=' * 60}")
+    click.echo(f"\nResults: {success_count} successful, {fail_count} failed")
+    click.echo(f"\nFiles saved to: {results_dir}/")
+    click.echo(f"  - vibes-results.jsonl    (agent outputs)")
+    click.echo(f"  - evaluator-schema.yaml  (evaluation criteria)")
+    click.echo(f"  - run-info.json          (run metadata)")
+
+    click.echo(f"\n{'=' * 60}")
+    click.echo(f"NEXT STEP: Ask your AI assistant to evaluate")
+    click.echo(f"{'=' * 60}")
+    click.echo(f"""
+Copy this prompt to Claude Code or your AI assistant:
+
+    Please evaluate the experiment results in:
+    {results_dir}/
+
+    Read the vibes-results.jsonl file and evaluate each example
+    using the evaluator schema in evaluator-schema.yaml.
+
+    For each example, provide:
+    1. extracted_classification
+    2. exact_match (vs ground_truth)
+    3. semantic_match
+    4. reasoning_quality_score
+    5. overall_score
+    6. pass/fail
+
+    Then provide summary metrics:
+    - Exact match accuracy
+    - Semantic match accuracy
+    - Average overall score
+    - Pass rate
+""")
+
+
 # =============================================================================
 # RUN COMMAND
 # =============================================================================
@@ -524,6 +677,8 @@ def show(name: str, base_path: Optional[str]):
 @click.option("--base-path", help="Base directory for experiments (default: EXPERIMENTS_HOME or 'experiments')")
 @click.option("--version", help="Git tag version to load (e.g., 'experiments/my-exp/v1.0.0')")
 @click.option("--dry-run", is_flag=True, help="Test on small subset without saving")
+@click.option("--only-vibes", is_flag=True, help="Run agent locally, export results for AI evaluation (no Phoenix)")
+@click.option("--limit", "-n", type=int, help="Limit number of examples to evaluate (useful with --only-vibes)")
 @click.option("--update-prompts", is_flag=True, help="Update prompts in Phoenix before running")
 @click.option("--phoenix-url", help="Phoenix server URL (overrides PHOENIX_BASE_URL env var)")
 @click.option("--phoenix-api-key", help="Phoenix API key (overrides PHOENIX_API_KEY env var)")
@@ -532,14 +687,45 @@ def run(
     base_path: Optional[str],
     version: Optional[str],
     dry_run: bool,
+    only_vibes: bool,
+    limit: Optional[int],
     update_prompts: bool,
     phoenix_url: Optional[str],
     phoenix_api_key: Optional[str],
 ):
-    """Run an experiment using Phoenix provider.
+    """Run an experiment using Phoenix provider or local vibes mode.
 
     Loads configuration, executes agent and evaluator, saves results.
 
+    Vibes Mode (--only-vibes):
+        Run agent locally without Phoenix infrastructure. Agent outputs are saved
+        to a JSONL file along with the evaluator schema. Your AI assistant (e.g.,
+        Claude Code) then acts as the judge to evaluate results.
+
+        This enables seamless switching between:
+        - Local evaluation: Quick iteration with AI-as-judge
+        - Phoenix evaluation: Production metrics and dashboards
+
+        Usage:
+            rem experiments run my-experiment --only-vibes
+            rem experiments run my-experiment --only-vibes --limit 5
+
+        The command will:
+        1. Run the agent on each ground-truth example
+        2. Save results to results/{timestamp}/vibes-results.jsonl
+        3. Print the evaluator prompt and schema
+        4. Instruct you to ask your AI assistant to evaluate
+
+        Example workflow with Claude Code:
+            $ rem experiments run mental-health-classifier --only-vibes --limit 3
+            # ... agent runs ...
+            # Results saved to: .experiments/mental-health-classifier/results/20241203-143022/
+
+            # Then ask Claude Code:
+            "Please evaluate the experiment results in
+             .experiments/mental-health-classifier/results/20241203-143022/
+             using the evaluator schema provided"
+
     Phoenix Connection:
         Commands respect PHOENIX_BASE_URL and PHOENIX_API_KEY environment variables.
         Defaults to localhost:6006 for local development.
@@ -562,6 +748,12 @@ def run(
         # Run experiment with latest schemas
         rem experiments run hello-world-validation
 
+        # Quick local evaluation (vibes mode)
+        rem experiments run hello-world-validation --only-vibes
+
+        # Vibes mode with limited examples
+        rem experiments run hello-world-validation --only-vibes --limit 5
+
         # Run specific version
         rem experiments run hello-world-validation \\
             --version experiments/hello-world-validation/v1.0.0
@@ -674,35 +866,47 @@ def run(
 
         click.echo(f"Loading evaluator: {evaluator_name} for agent {agent_name}")
 
-        # Try multiple evaluator path patterns (agent-specific, then generic)
-        evaluator_paths_to_try = [
-            f"{agent_name}/{evaluator_name}",  # e.g., hello-world/default
-            f"{agent_name}-{evaluator_name}",  # e.g., hello-world-default
-            evaluator_name,                     # e.g., default (generic)
-        ]
+        # Find evaluator schema file path
+        from rem.utils.schema_loader import get_evaluator_schema_path
+
+        evaluator_schema_path = get_evaluator_schema_path(evaluator_name)
+        if not evaluator_schema_path or not evaluator_schema_path.exists():
+            click.echo(f"Error: Could not find evaluator schema '{evaluator_name}'")
+            raise click.Abort()
+
+        click.echo(f"✓ Found evaluator schema: {evaluator_schema_path}")
 
+        # For Phoenix mode, also load evaluator function
         evaluator_fn = None
-        evaluator_load_error = None
+        if not only_vibes:
+            # Try multiple evaluator path patterns (agent-specific, then generic)
+            evaluator_paths_to_try = [
+                f"{agent_name}/{evaluator_name}",  # e.g., hello-world/default
+                f"{agent_name}-{evaluator_name}",  # e.g., hello-world-default
+                evaluator_name,                     # e.g., default (generic)
+            ]
 
-        for evaluator_path in evaluator_paths_to_try:
-            try:
-                evaluator_fn = create_evaluator_from_schema(
-                    evaluator_schema_path=evaluator_path,
-                    model_name=None,  # Use default from schema
-                )
-                click.echo(f"✓ Loaded evaluator schema: {evaluator_path}")
-                break
-            except FileNotFoundError as e:
-                evaluator_load_error = e
-                logger.debug(f"Evaluator not found at {evaluator_path}: {e}")
-                continue
-            except Exception as e:
-                evaluator_load_error = e
-                logger.warning(f"Failed to load evaluator from {evaluator_path}: {e}")
-                continue
+            evaluator_load_error = None
 
-        if evaluator_fn is None:
-            click.echo(f"Error: Could not load evaluator schema '{evaluator_name}'")
+            for evaluator_path in evaluator_paths_to_try:
+                try:
+                    evaluator_fn = create_evaluator_from_schema(
+                        evaluator_schema_path=evaluator_path,
+                        model_name=None,  # Use default from schema
+                    )
+                    click.echo(f"✓ Loaded evaluator function: {evaluator_path}")
+                    break
+                except FileNotFoundError as e:
+                    evaluator_load_error = e
+                    logger.debug(f"Evaluator not found at {evaluator_path}: {e}")
+                    continue
+                except Exception as e:
+                    evaluator_load_error = e
+                    logger.warning(f"Failed to load evaluator from {evaluator_path}: {e}")
+                    continue
+
+        if evaluator_fn is None and not only_vibes:
+            click.echo(f"Error: Could not load evaluator function '{evaluator_name}'")
             click.echo(f"  Tried paths: {evaluator_paths_to_try}")
             if evaluator_load_error:
                 click.echo(f"  Last error: {evaluator_load_error}")
@@ -769,6 +973,18 @@ def run(
             # TODO: Implement prompt updating
             click.echo("⚠  --update-prompts not yet implemented")
 
+        # Vibes mode: run agent and export for AI evaluation
+        if only_vibes:
+            _run_vibes_mode(
+                config=config,
+                dataset_df=dataset_df,
+                task_fn=task_fn,
+                base_path=base_path,
+                limit=limit,
+                evaluator_schema_path=evaluator_schema_path,
+            )
+            return
+
         # Run experiment via Phoenix
         if not dry_run:
             # Create Phoenix client with optional overrides

rem/cli/commands/schema.py

@@ -16,6 +16,7 @@ from loguru import logger
 
 from ...settings import settings
 from ...services.postgres.schema_generator import SchemaGenerator
+from ...utils.sql_paths import get_package_sql_dir, get_package_migrations_dir
 
 
 def _import_model_modules() -> list[str]:
@@ -55,7 +56,7 @@ def _import_model_modules() -> list[str]:
     "--output-dir",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Base output directory (default: {settings.sql_dir}/migrations)",
+    help="Base output directory (default: package sql/migrations)",
 )
 def generate(output: Path, output_dir: Path | None):
     """
@@ -110,8 +111,8 @@ def generate(output: Path, output_dir: Path | None):
     models = registry.get_models(include_core=True)
     click.echo(f"Generating schema from {len(models)} registered models")
 
-    # Default to migrations directory
-    actual_output_dir = output_dir or Path(settings.sql_dir) / "migrations"
+    # Default to package migrations directory
+    actual_output_dir = output_dir or get_package_migrations_dir()
     generator = SchemaGenerator(output_dir=actual_output_dir)
 
     # Generate schema from registry
@@ -124,7 +125,7 @@ def generate(output: Path, output_dir: Path | None):
         # Generate background indexes in parent sql dir
         background_indexes = generator.generate_background_indexes()
         if background_indexes:
-            bg_file = Path(settings.sql_dir) / "background_indexes.sql"
+            bg_file = get_package_sql_dir() / "background_indexes.sql"
             bg_file.write_text(background_indexes)
             click.echo(f"✓ Background indexes: {bg_file}")
 
@@ -204,7 +205,7 @@ def validate():
     "-o",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Output file for background indexes (default: {settings.sql_dir}/background_indexes.sql)",
+    help="Output file for background indexes (default: package sql/background_indexes.sql)",
 )
 def indexes(output: Path):
     """

rem/config.py

@@ -95,9 +95,16 @@ def load_config() -> dict[str, Any]:
     """
     Load configuration from ~/.rem/config.yaml.
 
+    Set REM_SKIP_CONFIG=1 to skip loading the config file (useful when using .env files).
+
     Returns:
-        Configuration dictionary (empty if file doesn't exist)
+        Configuration dictionary (empty if file doesn't exist or skipped)
     """
+    # Allow skipping config file via environment variable
+    if os.environ.get("REM_SKIP_CONFIG", "").lower() in ("1", "true", "yes"):
+        logger.debug("Skipping config file (REM_SKIP_CONFIG is set)")
+        return {}
+
     config_path = get_config_path()
 
     if not config_path.exists():