remdb 0.3.127__py3-none-any.whl → 0.3.172__py3-none-any.whl

rem/cli/commands/experiments.py

@@ -63,6 +63,7 @@ def experiments():
 @experiments.command("create")
 @click.argument("name")
 @click.option("--agent", "-a", required=True, help="Agent schema name (e.g., 'cv-parser')")
+@click.option("--task", "-t", default="general", help="Task name for organizing experiments (e.g., 'risk-assessment')")
 @click.option("--evaluator", "-e", default="default", help="Evaluator schema name (default: 'default')")
 @click.option("--description", "-d", help="Experiment description")
 @click.option("--dataset-location", type=click.Choice(["git", "s3", "hybrid"]), default="git",
@@ -74,6 +75,7 @@ def experiments():
 def create(
     name: str,
     agent: str,
+    task: str,
     evaluator: str,
     description: Optional[str],
     dataset_location: str,
@@ -123,19 +125,17 @@ def create(
         # Resolve base path: CLI arg > EXPERIMENTS_HOME env var > default "experiments"
         if base_path is None:
             base_path = os.getenv("EXPERIMENTS_HOME", "experiments")
-        # Build dataset reference
+        # Build dataset reference (format auto-detected from file extension)
         if dataset_location == "git":
             dataset_ref = DatasetReference(
                 location=DatasetLocation.GIT,
                 path="ground-truth/dataset.csv",
-                format="csv",
                 description="Ground truth Q&A dataset for evaluation"
             )
         else:  # s3 or hybrid
             dataset_ref = DatasetReference(
                 location=DatasetLocation(dataset_location),
                 path=f"s3://rem-experiments/{name}/datasets/ground_truth.parquet",
-                format="parquet",
                 schema_path="datasets/schema.yaml" if dataset_location == "hybrid" else None,
                 description="Ground truth dataset for evaluation"
             )
@@ -170,7 +170,8 @@ def create(
         # Create experiment config
         config = ExperimentConfig(
             name=name,
-            description=description or f"Evaluation experiment for {agent} agent",
+            task=task,
+            description=description or f"Evaluation experiment for {agent} agent ({task} task)",
             agent_schema_ref=SchemaReference(
                 name=agent,
                 version=None,  # Use latest by default
@@ -912,58 +913,61 @@ def run(
                 click.echo(f"  Last error: {evaluator_load_error}")
             raise click.Abort()
 
-        # Load dataset using Polars
-        import polars as pl
+        # Validate evaluator credentials before running expensive agent tasks
+        if evaluator_fn is not None and not only_vibes:
+            from rem.agentic.providers.phoenix import validate_evaluator_credentials
+
+            click.echo("Validating evaluator credentials...")
+            is_valid, error_msg = validate_evaluator_credentials()
+            if not is_valid:
+                click.echo(click.style(f"\n⚠️  Evaluator validation failed: {error_msg}", fg="yellow"))
+                click.echo("\nOptions:")
+                click.echo("  1. Fix the credentials issue and re-run")
+                click.echo("  2. Run with --only-vibes to skip LLM evaluation")
+                click.echo("  3. Use --evaluator-model to specify a different model")
+                raise click.Abort()
+            click.echo("✓ Evaluator credentials validated")
+
+        # Load dataset using read_dataframe utility (auto-detects format from extension)
+        from rem.utils.files import read_dataframe
 
         click.echo(f"Loading dataset: {list(config.datasets.keys())[0]}")
         dataset_ref = list(config.datasets.values())[0]
 
-        if dataset_ref.location.value == "git":
-            # Load from Git (local filesystem)
-            dataset_path = Path(base_path) / name / dataset_ref.path
-            if not dataset_path.exists():
-                click.echo(f"Error: Dataset not found: {dataset_path}")
-                raise click.Abort()
-
-            if dataset_ref.format == "csv":
-                dataset_df = pl.read_csv(dataset_path)
-            elif dataset_ref.format == "parquet":
-                dataset_df = pl.read_parquet(dataset_path)
-            elif dataset_ref.format == "jsonl":
-                dataset_df = pl.read_ndjson(dataset_path)
-            else:
-                click.echo(f"Error: Format '{dataset_ref.format}' not yet supported")
-                raise click.Abort()
-        elif dataset_ref.location.value in ["s3", "hybrid"]:
-            # Load from S3 using FS provider
-            from rem.services.fs import FS
-            from io import BytesIO
+        try:
+            if dataset_ref.location.value == "git":
+                # Load from Git (local filesystem)
+                dataset_path = Path(base_path) / name / dataset_ref.path
+                if not dataset_path.exists():
+                    click.echo(f"Error: Dataset not found: {dataset_path}")
+                    raise click.Abort()
 
-            fs = FS()
+                dataset_df = read_dataframe(dataset_path)
 
-            try:
-                if dataset_ref.format == "csv":
-                    content = fs.read(dataset_ref.path)
-                    dataset_df = pl.read_csv(BytesIO(content.encode() if isinstance(content, str) else content))
-                elif dataset_ref.format == "parquet":
-                    content_bytes = fs.read(dataset_ref.path)
-                    dataset_df = pl.read_parquet(BytesIO(content_bytes if isinstance(content_bytes, bytes) else content_bytes.encode()))
-                elif dataset_ref.format == "jsonl":
-                    content = fs.read(dataset_ref.path)
-                    dataset_df = pl.read_ndjson(BytesIO(content.encode() if isinstance(content, str) else content))
-                else:
-                    click.echo(f"Error: Format '{dataset_ref.format}' not yet supported")
-                    raise click.Abort()
+            elif dataset_ref.location.value in ["s3", "hybrid"]:
+                # Load from S3 using FS provider
+                from rem.services.fs import FS
 
+                fs = FS()
+                content = fs.read(dataset_ref.path)
+                # Ensure we have bytes
+                if isinstance(content, str):
+                    content = content.encode()
+                dataset_df = read_dataframe(content, filename=dataset_ref.path)
                 click.echo(f"✓ Loaded dataset from S3")
-            except Exception as e:
-                logger.error(f"Failed to load dataset from S3: {e}")
-                click.echo(f"Error: Could not load dataset from S3")
-                click.echo(f"  Path: {dataset_ref.path}")
-                click.echo(f"  Format: {dataset_ref.format}")
+
+            else:
+                click.echo(f"Error: Unknown dataset location: {dataset_ref.location.value}")
                 raise click.Abort()
-        else:
-            click.echo(f"Error: Unknown dataset location: {dataset_ref.location.value}")
+
+        except ValueError as e:
+            # Unsupported format error from read_dataframe
+            click.echo(f"Error: {e}")
+            raise click.Abort()
+        except Exception as e:
+            logger.error(f"Failed to load dataset: {e}")
+            click.echo(f"Error: Could not load dataset")
+            click.echo(f"  Path: {dataset_ref.path}")
             raise click.Abort()
 
         click.echo(f"✓ Loaded dataset: {len(dataset_df)} examples")
@@ -1283,7 +1287,7 @@ def prompt():
 @click.option("--system-prompt", "-s", required=True, help="System prompt text")
 @click.option("--description", "-d", help="Prompt description")
 @click.option("--model-provider", default="OPENAI", help="Model provider (OPENAI, ANTHROPIC)")
-@click.option("--model-name", "-m", help="Model name (e.g., gpt-4o, claude-sonnet-4-5)")
+@click.option("--model-name", "-m", help="Model name (e.g., gpt-4.1, claude-sonnet-4-5)")
 @click.option("--type", "-t", "prompt_type", default="Agent", help="Prompt type (Agent or Evaluator)")
 def prompt_create(
     name: str,
@@ -1299,7 +1303,7 @@ def prompt_create(
         # Create agent prompt
         rem experiments prompt create hello-world \\
             --system-prompt "You are a helpful assistant." \\
-            --model-name gpt-4o
+            --model-name gpt-4.1
 
         # Create evaluator prompt
         rem experiments prompt create correctness-evaluator \\
@@ -1317,7 +1321,7 @@ def prompt_create(
     try:
         # Set default model if not specified
         if not model_name:
-            model_name = "gpt-4o" if model_provider == "OPENAI" else "claude-sonnet-4-5-20250929"
+            model_name = "gpt-4.1" if model_provider == "OPENAI" else "claude-sonnet-4-5-20250929"
 
         # Get config
         phoenix_client = PhoenixClient()
@@ -1520,3 +1524,175 @@ def trace_list(
         logger.error(f"Failed to list traces: {e}")
         click.echo(f"Error: {e}", err=True)
         raise click.Abort()
+
+
+# =============================================================================
+# EXPORT COMMAND
+# =============================================================================
+
+
+@experiments.command("export")
+@click.argument("name")
+@click.option("--base-path", help="Base directory for experiments (default: EXPERIMENTS_HOME or 'experiments')")
+@click.option("--bucket", "-b", help="S3 bucket name (default: DATA_LAKE__BUCKET_NAME)")
+@click.option("--version", "-v", default="v0", help="Data lake version prefix (default: v0)")
+@click.option("--plan", is_flag=True, help="Show what would be exported without uploading")
+@click.option("--include-results", is_flag=True, help="Include results directory in export")
+def export(
+    name: str,
+    base_path: Optional[str],
+    bucket: Optional[str],
+    version: str,
+    plan: bool,
+    include_results: bool,
+):
+    """Export experiment to S3 data lake.
+
+    Exports experiment configuration, ground truth, and optionally results
+    to the S3 data lake following the convention:
+
+        s3://{bucket}/{version}/datasets/calibration/experiments/{agent}/{task}/
+
+    The export includes:
+    - experiment.yaml (configuration)
+    - README.md (documentation)
+    - ground-truth/ (evaluation datasets)
+    - seed-data/ (optional seed data)
+    - results/ (optional, with --include-results)
+
+    Examples:
+        # Preview what would be exported
+        rem experiments export my-experiment --plan
+
+        # Export to configured data lake bucket
+        rem experiments export my-experiment
+
+        # Export to specific bucket
+        rem experiments export my-experiment --bucket siggy-data
+
+        # Include results in export
+        rem experiments export my-experiment --include-results
+
+        # Export with custom version prefix
+        rem experiments export my-experiment --version v1
+    """
+    from rem.models.core.experiment import ExperimentConfig
+    from rem.settings import settings
+    from rem.services.fs.s3_provider import S3Provider
+    import os
+    import json
+
+    try:
+        # Resolve base path
+        if base_path is None:
+            base_path = os.getenv("EXPERIMENTS_HOME", "experiments")
+
+        # Load experiment configuration
+        config_path = Path(base_path) / name / "experiment.yaml"
+        if not config_path.exists():
+            click.echo(f"Experiment not found: {name}")
+            click.echo(f"  Looked in: {config_path}")
+            raise click.Abort()
+
+        config = ExperimentConfig.from_yaml(config_path)
+        click.echo(f"✓ Loaded experiment: {name}")
+
+        # Resolve bucket
+        if bucket is None:
+            bucket = settings.data_lake.bucket_name
+            if bucket is None:
+                click.echo("Error: No S3 bucket configured.")
+                click.echo("  Set DATA_LAKE__BUCKET_NAME environment variable or use --bucket option")
+                raise click.Abort()
+
+        # Build S3 paths
+        s3_base = config.get_s3_export_path(bucket, version)
+        exp_dir = config.get_experiment_dir(base_path)
+
+        # Collect files to export
+        files_to_export = []
+
+        # Always include these files
+        required_files = [
+            ("experiment.yaml", exp_dir / "experiment.yaml"),
+            ("README.md", exp_dir / "README.md"),
+        ]
+
+        for s3_name, local_path in required_files:
+            if local_path.exists():
+                files_to_export.append((s3_name, local_path))
+
+        # Include ground-truth directory
+        ground_truth_dir = exp_dir / "ground-truth"
+        if ground_truth_dir.exists():
+            for f in ground_truth_dir.rglob("*"):
+                if f.is_file():
+                    relative = f.relative_to(exp_dir)
+                    files_to_export.append((str(relative), f))
+
+        # Include seed-data directory
+        seed_data_dir = exp_dir / "seed-data"
+        if seed_data_dir.exists():
+            for f in seed_data_dir.rglob("*"):
+                if f.is_file():
+                    relative = f.relative_to(exp_dir)
+                    files_to_export.append((str(relative), f))
+
+        # Optionally include results
+        if include_results:
+            results_dir = exp_dir / "results"
+            if results_dir.exists():
+                for f in results_dir.rglob("*"):
+                    if f.is_file():
+                        relative = f.relative_to(exp_dir)
+                        files_to_export.append((str(relative), f))
+
+        # Display export plan
+        click.echo(f"\n{'=' * 60}")
+        click.echo(f"EXPORT {'PLAN' if plan else 'TO S3'}")
+        click.echo(f"{'=' * 60}")
+        click.echo(f"\nExperiment: {config.name}")
+        click.echo(f"Agent: {config.agent_schema_ref.name}")
+        click.echo(f"Task: {config.task}")
+        click.echo(f"Evaluator file: {config.get_evaluator_filename()}")
+        click.echo(f"\nDestination: {s3_base}/")
+        click.echo(f"\nFiles to export ({len(files_to_export)}):")
+
+        for s3_name, local_path in files_to_export:
+            s3_uri = f"{s3_base}/{s3_name}"
+            if plan:
+                click.echo(f"  {local_path}")
+                click.echo(f"    → {s3_uri}")
+            else:
+                click.echo(f"  {s3_name}")
+
+        if plan:
+            click.echo(f"\n[PLAN MODE] No files were uploaded.")
+            click.echo(f"Run without --plan to execute the export.")
+            return
+
+        # Execute export
+        click.echo(f"\n⏳ Uploading to S3...")
+        s3 = S3Provider()
+
+        uploaded = 0
+        for s3_name, local_path in files_to_export:
+            s3_uri = f"{s3_base}/{s3_name}"
+            try:
+                s3.copy(str(local_path), s3_uri)
+                uploaded += 1
+                click.echo(f"  ✓ {s3_name}")
+            except Exception as e:
+                click.echo(f"  ✗ {s3_name}: {e}")
+
+        click.echo(f"\n✓ Exported {uploaded}/{len(files_to_export)} files to {s3_base}/")
+
+        # Show next steps
+        click.echo(f"\nNext steps:")
+        click.echo(f"  - View in S3: aws s3 ls {s3_base}/ --recursive")
+        click.echo(f"  - Download: aws s3 sync {s3_base}/ ./{config.agent_schema_ref.name}/{config.task}/")
+
+    except Exception as e:
+        logger.error(f"Failed to export experiment: {e}")
+        click.echo(f"Error: {e}", err=True)
+        raise click.Abort()

rem/cli/commands/session.py

@@ -0,0 +1,336 @@
+"""
+CLI command for viewing and simulating session conversations.
+
+Usage:
+    rem session show <user_id> [--session-id] [--role user|assistant|system]
+    rem session show <user_id> --simulate-next [--save] [--custom-sim-prompt "..."]
+
+Examples:
+    # Show all messages for a user
+    rem session show 11111111-1111-1111-1111-111111111001
+
+    # Show only user messages
+    rem session show 11111111-1111-1111-1111-111111111001 --role user
+
+    # Simulate next user message
+    rem session show 11111111-1111-1111-1111-111111111001 --simulate-next
+
+    # Simulate with custom prompt and save
+    rem session show 11111111-1111-1111-1111-111111111001 --simulate-next --save \
+        --custom-sim-prompt "Respond as an anxious patient"
+"""
+
+import asyncio
+from pathlib import Path
+from typing import Literal
+
+import click
+import yaml
+from loguru import logger
+
+from ...models.entities.user import User
+from ...models.entities.message import Message
+from ...services.postgres import get_postgres_service
+from ...services.postgres.repository import Repository
+from ...settings import settings
+
+
+SIMULATOR_PROMPT = """You are simulating a patient in a mental health conversation.
+
+## Context
+You are continuing a conversation with a clinical evaluation agent. Based on the
+user profile and conversation history below, generate the next realistic patient message.
+
+## User Profile
+{user_profile}
+
+## Conversation History
+{conversation_history}
+
+## Instructions
+- Stay in character as the patient described in the profile
+- Your response should be natural, conversational, and consistent with the patient's presentation
+- Consider the patient's risk level, symptoms, and communication style
+- Do NOT include any metadata or role labels - just the raw message content
+- Keep responses concise (1-3 sentences typical for conversation)
+
+Generate the next patient message:"""
+
+
+async def _load_user_and_messages(
+    user_id: str,
+    session_id: str | None = None,
+    role_filter: str | None = None,
+    limit: int = 100,
+) -> tuple[User | None, list[Message]]:
+    """Load user profile and messages from database."""
+    pg = get_postgres_service()
+    if not pg:
+        logger.error("PostgreSQL not available")
+        return None, []
+
+    await pg.connect()
+
+    try:
+        # Load user
+        user_repo = Repository(User, "users", db=pg)
+        user = await user_repo.get_by_id(user_id, tenant_id="default")
+
+        # Load messages
+        message_repo = Repository(Message, "messages", db=pg)
+        filters = {"user_id": user_id}
+        if session_id:
+            filters["session_id"] = session_id
+
+        messages = await message_repo.find(
+            filters=filters,
+            order_by="created_at ASC",
+            limit=limit,
+        )
+
+        # Filter by role if specified
+        if role_filter:
+            messages = [m for m in messages if m.message_type == role_filter]
+
+        return user, messages
+
+    finally:
+        await pg.disconnect()
+
+
+def _format_user_yaml(user: User | None) -> str:
+    """Format user profile as YAML."""
+    if not user:
+        return "# No user found"
+
+    data = {
+        "id": str(user.id),
+        "name": user.name,
+        "summary": user.summary,
+        "interests": user.interests,
+        "preferred_topics": user.preferred_topics,
+        "metadata": user.metadata,
+    }
+    return yaml.dump(data, default_flow_style=False, allow_unicode=True)
+
+
+def _format_messages_yaml(messages: list[Message]) -> str:
+    """Format messages as YAML."""
+    if not messages:
+        return "# No messages found"
+
+    data = []
+    for msg in messages:
+        data.append({
+            "role": msg.message_type or "unknown",
+            "content": msg.content,
+            "session_id": msg.session_id,
+            "created_at": msg.created_at.isoformat() if msg.created_at else None,
+        })
+    return yaml.dump(data, default_flow_style=False, allow_unicode=True)
+
+
+def _format_conversation_for_llm(messages: list[Message]) -> str:
+    """Format conversation history for LLM context."""
+    lines = []
+    for msg in messages:
+        role = msg.message_type or "unknown"
+        lines.append(f"[{role.upper()}]: {msg.content}")
+    return "\n\n".join(lines) if lines else "(No previous messages)"
+
+
+async def _simulate_next_message(
+    user: User | None,
+    messages: list[Message],
+    custom_prompt: str | None = None,
+) -> str:
+    """Use LLM to simulate the next patient message."""
+    from pydantic_ai import Agent
+
+    # Build context
+    user_profile = _format_user_yaml(user) if user else "Unknown patient"
+    conversation_history = _format_conversation_for_llm(messages)
+
+    # Use custom prompt or default
+    if custom_prompt:
+        # Check if it's a file path
+        if Path(custom_prompt).exists():
+            prompt_template = Path(custom_prompt).read_text()
+        else:
+            prompt_template = custom_prompt
+        # Simple variable substitution
+        prompt = prompt_template.replace("{user_profile}", user_profile)
+        prompt = prompt.replace("{conversation_history}", conversation_history)
+    else:
+        prompt = SIMULATOR_PROMPT.format(
+            user_profile=user_profile,
+            conversation_history=conversation_history,
+        )
+
+    # Create simple agent for simulation
+    agent = Agent(
+        model=settings.llm.default_model,
+        system_prompt="You are a patient simulator. Generate realistic patient responses.",
+    )
+
+    result = await agent.run(prompt)
+    return result.output
+
+
+async def _save_message(
+    user_id: str,
+    session_id: str | None,
+    content: str,
+    role: str = "user",
+) -> Message:
+    """Save a simulated message to the database."""
+    from uuid import uuid4
+
+    pg = get_postgres_service()
+    if not pg:
+        raise RuntimeError("PostgreSQL not available")
+
+    await pg.connect()
+
+    try:
+        message_repo = Repository(Message, "messages", db=pg)
+
+        message = Message(
+            id=uuid4(),
+            user_id=user_id,
+            tenant_id="default",
+            session_id=session_id or str(uuid4()),
+            content=content,
+            message_type=role,
+        )
+
+        await message_repo.upsert(message)
+        return message
+
+    finally:
+        await pg.disconnect()
+
+
+@click.group()
+def session():
+    """Session viewing and simulation commands."""
+    pass
+
+
+@session.command("show")
+@click.argument("user_id")
+@click.option("--session-id", "-s", help="Filter by session ID")
+@click.option(
+    "--role", "-r",
+    type=click.Choice(["user", "assistant", "system", "tool"]),
+    help="Filter messages by role",
+)
+@click.option("--limit", "-l", default=100, help="Max messages to load")
+@click.option("--simulate-next", is_flag=True, help="Simulate the next patient message")
+@click.option("--save", is_flag=True, help="Save simulated message to database")
+@click.option(
+    "--custom-sim-prompt", "-p",
+    help="Custom simulation prompt (text or file path)",
+)
+def show(
+    user_id: str,
+    session_id: str | None,
+    role: str | None,
+    limit: int,
+    simulate_next: bool,
+    save: bool,
+    custom_sim_prompt: str | None,
+):
+    """
+    Show user profile and session messages.
+
+    USER_ID: The user identifier to load.
+
+    Examples:
+
+        # Show user and all messages
+        rem session show 11111111-1111-1111-1111-111111111001
+
+        # Show only assistant responses
+        rem session show 11111111-1111-1111-1111-111111111001 --role assistant
+
+        # Simulate next patient message
+        rem session show 11111111-1111-1111-1111-111111111001 --simulate-next
+
+        # Simulate and save to database
+        rem session show 11111111-1111-1111-1111-111111111001 --simulate-next --save
+    """
+    asyncio.run(_show_async(
+        user_id=user_id,
+        session_id=session_id,
+        role_filter=role,
+        limit=limit,
+        simulate_next=simulate_next,
+        save=save,
+        custom_sim_prompt=custom_sim_prompt,
+    ))
+
+
+async def _show_async(
+    user_id: str,
+    session_id: str | None,
+    role_filter: str | None,
+    limit: int,
+    simulate_next: bool,
+    save: bool,
+    custom_sim_prompt: str | None,
+):
+    """Async implementation of show command."""
+    # Load data
+    user, messages = await _load_user_and_messages(
+        user_id=user_id,
+        session_id=session_id,
+        role_filter=role_filter if not simulate_next else None,  # Need all messages for simulation
+        limit=limit,
+    )
+
+    # Display user profile
+    click.echo("\n# User Profile")
+    click.echo("---")
+    click.echo(_format_user_yaml(user))
+
+    # Display messages (apply filter for display if simulating)
+    display_messages = messages
+    if simulate_next and role_filter:
+        display_messages = [m for m in messages if m.message_type == role_filter]
+
+    click.echo("\n# Messages")
+    click.echo("---")
+    click.echo(_format_messages_yaml(display_messages))
+
+    # Simulate next message if requested
+    if simulate_next:
+        click.echo("\n# Simulated Next Message")
+        click.echo("---")
+
+        try:
+            simulated = await _simulate_next_message(
+                user=user,
+                messages=messages,
+                custom_prompt=custom_sim_prompt,
+            )
+            click.echo(f"role: user")
+            click.echo(f"content: |\n  {simulated}")
+
+            if save:
+                saved_msg = await _save_message(
+                    user_id=user_id,
+                    session_id=session_id,
+                    content=simulated,
+                    role="user",
+                )
+                logger.success(f"Saved message: {saved_msg.id}")
+
+        except Exception as e:
+            logger.error(f"Simulation failed: {e}")
+            raise
+
+
+def register_command(cli_group):
+    """Register the session command group."""
+    cli_group.add_command(session)