remdb 0.3.114__py3-none-any.whl → 0.3.172__py3-none-any.whl

rem/cli/commands/schema.py

@@ -16,6 +16,7 @@ from loguru import logger
 
 from ...settings import settings
 from ...services.postgres.schema_generator import SchemaGenerator
+from ...utils.sql_paths import get_package_sql_dir, get_package_migrations_dir
 
 
 def _import_model_modules() -> list[str]:
@@ -55,7 +56,7 @@ def _import_model_modules() -> list[str]:
     "--output-dir",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Base output directory (default: {settings.sql_dir}/migrations)",
+    help="Base output directory (default: package sql/migrations)",
 )
 def generate(output: Path, output_dir: Path | None):
     """
@@ -110,8 +111,8 @@ def generate(output: Path, output_dir: Path | None):
     models = registry.get_models(include_core=True)
     click.echo(f"Generating schema from {len(models)} registered models")
 
-    # Default to migrations directory
-    actual_output_dir = output_dir or Path(settings.sql_dir) / "migrations"
+    # Default to package migrations directory
+    actual_output_dir = output_dir or get_package_migrations_dir()
     generator = SchemaGenerator(output_dir=actual_output_dir)
 
     # Generate schema from registry
@@ -124,7 +125,7 @@ def generate(output: Path, output_dir: Path | None):
         # Generate background indexes in parent sql dir
         background_indexes = generator.generate_background_indexes()
         if background_indexes:
-            bg_file = Path(settings.sql_dir) / "background_indexes.sql"
+            bg_file = get_package_sql_dir() / "background_indexes.sql"
             bg_file.write_text(background_indexes)
             click.echo(f"✓ Background indexes: {bg_file}")
 
@@ -204,7 +205,7 @@ def validate():
     "-o",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Output file for background indexes (default: {settings.sql_dir}/background_indexes.sql)",
+    help="Output file for background indexes (default: package sql/background_indexes.sql)",
 )
 def indexes(output: Path):
     """

rem/cli/commands/session.py

@@ -0,0 +1,336 @@
+"""
+CLI command for viewing and simulating session conversations.
+
+Usage:
+    rem session show <user_id> [--session-id] [--role user|assistant|system]
+    rem session show <user_id> --simulate-next [--save] [--custom-sim-prompt "..."]
+
+Examples:
+    # Show all messages for a user
+    rem session show 11111111-1111-1111-1111-111111111001
+
+    # Show only user messages
+    rem session show 11111111-1111-1111-1111-111111111001 --role user
+
+    # Simulate next user message
+    rem session show 11111111-1111-1111-1111-111111111001 --simulate-next
+
+    # Simulate with custom prompt and save
+    rem session show 11111111-1111-1111-1111-111111111001 --simulate-next --save \
+        --custom-sim-prompt "Respond as an anxious patient"
+"""
+
+import asyncio
+from pathlib import Path
+from typing import Literal
+
+import click
+import yaml
+from loguru import logger
+
+from ...models.entities.user import User
+from ...models.entities.message import Message
+from ...services.postgres import get_postgres_service
+from ...services.postgres.repository import Repository
+from ...settings import settings
+
+
+SIMULATOR_PROMPT = """You are simulating a patient in a mental health conversation.
+
+## Context
+You are continuing a conversation with a clinical evaluation agent. Based on the
+user profile and conversation history below, generate the next realistic patient message.
+
+## User Profile
+{user_profile}
+
+## Conversation History
+{conversation_history}
+
+## Instructions
+- Stay in character as the patient described in the profile
+- Your response should be natural, conversational, and consistent with the patient's presentation
+- Consider the patient's risk level, symptoms, and communication style
+- Do NOT include any metadata or role labels - just the raw message content
+- Keep responses concise (1-3 sentences typical for conversation)
+
+Generate the next patient message:"""
+
+
+async def _load_user_and_messages(
+    user_id: str,
+    session_id: str | None = None,
+    role_filter: str | None = None,
+    limit: int = 100,
+) -> tuple[User | None, list[Message]]:
+    """Load user profile and messages from database."""
+    pg = get_postgres_service()
+    if not pg:
+        logger.error("PostgreSQL not available")
+        return None, []
+
+    await pg.connect()
+
+    try:
+        # Load user
+        user_repo = Repository(User, "users", db=pg)
+        user = await user_repo.get_by_id(user_id, tenant_id="default")
+
+        # Load messages
+        message_repo = Repository(Message, "messages", db=pg)
+        filters = {"user_id": user_id}
+        if session_id:
+            filters["session_id"] = session_id
+
+        messages = await message_repo.find(
+            filters=filters,
+            order_by="created_at ASC",
+            limit=limit,
+        )
+
+        # Filter by role if specified
+        if role_filter:
+            messages = [m for m in messages if m.message_type == role_filter]
+
+        return user, messages
+
+    finally:
+        await pg.disconnect()
+
+
+def _format_user_yaml(user: User | None) -> str:
+    """Format user profile as YAML."""
+    if not user:
+        return "# No user found"
+
+    data = {
+        "id": str(user.id),
+        "name": user.name,
+        "summary": user.summary,
+        "interests": user.interests,
+        "preferred_topics": user.preferred_topics,
+        "metadata": user.metadata,
+    }
+    return yaml.dump(data, default_flow_style=False, allow_unicode=True)
+
+
+def _format_messages_yaml(messages: list[Message]) -> str:
+    """Format messages as YAML."""
+    if not messages:
+        return "# No messages found"
+
+    data = []
+    for msg in messages:
+        data.append({
+            "role": msg.message_type or "unknown",
+            "content": msg.content,
+            "session_id": msg.session_id,
+            "created_at": msg.created_at.isoformat() if msg.created_at else None,
+        })
+    return yaml.dump(data, default_flow_style=False, allow_unicode=True)
+
+
+def _format_conversation_for_llm(messages: list[Message]) -> str:
+    """Format conversation history for LLM context."""
+    lines = []
+    for msg in messages:
+        role = msg.message_type or "unknown"
+        lines.append(f"[{role.upper()}]: {msg.content}")
+    return "\n\n".join(lines) if lines else "(No previous messages)"
+
+
+async def _simulate_next_message(
+    user: User | None,
+    messages: list[Message],
+    custom_prompt: str | None = None,
+) -> str:
+    """Use LLM to simulate the next patient message."""
+    from pydantic_ai import Agent
+
+    # Build context
+    user_profile = _format_user_yaml(user) if user else "Unknown patient"
+    conversation_history = _format_conversation_for_llm(messages)
+
+    # Use custom prompt or default
+    if custom_prompt:
+        # Check if it's a file path
+        if Path(custom_prompt).exists():
+            prompt_template = Path(custom_prompt).read_text()
+        else:
+            prompt_template = custom_prompt
+        # Simple variable substitution
+        prompt = prompt_template.replace("{user_profile}", user_profile)
+        prompt = prompt.replace("{conversation_history}", conversation_history)
+    else:
+        prompt = SIMULATOR_PROMPT.format(
+            user_profile=user_profile,
+            conversation_history=conversation_history,
+        )
+
+    # Create simple agent for simulation
+    agent = Agent(
+        model=settings.llm.default_model,
+        system_prompt="You are a patient simulator. Generate realistic patient responses.",
+    )
+
+    result = await agent.run(prompt)
+    return result.output
+
+
+async def _save_message(
+    user_id: str,
+    session_id: str | None,
+    content: str,
+    role: str = "user",
+) -> Message:
+    """Save a simulated message to the database."""
+    from uuid import uuid4
+
+    pg = get_postgres_service()
+    if not pg:
+        raise RuntimeError("PostgreSQL not available")
+
+    await pg.connect()
+
+    try:
+        message_repo = Repository(Message, "messages", db=pg)
+
+        message = Message(
+            id=uuid4(),
+            user_id=user_id,
+            tenant_id="default",
+            session_id=session_id or str(uuid4()),
+            content=content,
+            message_type=role,
+        )
+
+        await message_repo.upsert(message)
+        return message
+
+    finally:
+        await pg.disconnect()
+
+
+@click.group()
+def session():
+    """Session viewing and simulation commands."""
+    pass
+
+
+@session.command("show")
+@click.argument("user_id")
+@click.option("--session-id", "-s", help="Filter by session ID")
+@click.option(
+    "--role", "-r",
+    type=click.Choice(["user", "assistant", "system", "tool"]),
+    help="Filter messages by role",
+)
+@click.option("--limit", "-l", default=100, help="Max messages to load")
+@click.option("--simulate-next", is_flag=True, help="Simulate the next patient message")
+@click.option("--save", is_flag=True, help="Save simulated message to database")
+@click.option(
+    "--custom-sim-prompt", "-p",
+    help="Custom simulation prompt (text or file path)",
+)
+def show(
+    user_id: str,
+    session_id: str | None,
+    role: str | None,
+    limit: int,
+    simulate_next: bool,
+    save: bool,
+    custom_sim_prompt: str | None,
+):
+    """
+    Show user profile and session messages.
+
+    USER_ID: The user identifier to load.
+
+    Examples:
+
+        # Show user and all messages
+        rem session show 11111111-1111-1111-1111-111111111001
+
+        # Show only assistant responses
+        rem session show 11111111-1111-1111-1111-111111111001 --role assistant
+
+        # Simulate next patient message
+        rem session show 11111111-1111-1111-1111-111111111001 --simulate-next
+
+        # Simulate and save to database
+        rem session show 11111111-1111-1111-1111-111111111001 --simulate-next --save
+    """
+    asyncio.run(_show_async(
+        user_id=user_id,
+        session_id=session_id,
+        role_filter=role,
+        limit=limit,
+        simulate_next=simulate_next,
+        save=save,
+        custom_sim_prompt=custom_sim_prompt,
+    ))
+
+
+async def _show_async(
+    user_id: str,
+    session_id: str | None,
+    role_filter: str | None,
+    limit: int,
+    simulate_next: bool,
+    save: bool,
+    custom_sim_prompt: str | None,
+):
+    """Async implementation of show command."""
+    # Load data
+    user, messages = await _load_user_and_messages(
+        user_id=user_id,
+        session_id=session_id,
+        role_filter=role_filter if not simulate_next else None,  # Need all messages for simulation
+        limit=limit,
+    )
+
+    # Display user profile
+    click.echo("\n# User Profile")
+    click.echo("---")
+    click.echo(_format_user_yaml(user))
+
+    # Display messages (apply filter for display if simulating)
+    display_messages = messages
+    if simulate_next and role_filter:
+        display_messages = [m for m in messages if m.message_type == role_filter]
+
+    click.echo("\n# Messages")
+    click.echo("---")
+    click.echo(_format_messages_yaml(display_messages))
+
+    # Simulate next message if requested
+    if simulate_next:
+        click.echo("\n# Simulated Next Message")
+        click.echo("---")
+
+        try:
+            simulated = await _simulate_next_message(
+                user=user,
+                messages=messages,
+                custom_prompt=custom_sim_prompt,
+            )
+            click.echo(f"role: user")
+            click.echo(f"content: |\n  {simulated}")
+
+            if save:
+                saved_msg = await _save_message(
+                    user_id=user_id,
+                    session_id=session_id,
+                    content=simulated,
+                    role="user",
+                )
+                logger.success(f"Saved message: {saved_msg.id}")
+
+        except Exception as e:
+            logger.error(f"Simulation failed: {e}")
+            raise
+
+
+def register_command(cli_group):
+    """Register the session command group."""
+    cli_group.add_command(session)

rem/cli/dreaming.py

@@ -43,7 +43,7 @@ rem-dreaming full --user-id=user-123 --rem-api-url=http://localhost:8000
 Environment Variables:
 - REM_API_URL: REM API endpoint (default: http://rem-api:8000)
 - REM_EMBEDDING_PROVIDER: Embedding provider (default: text-embedding-3-small)
-- REM_DEFAULT_MODEL: LLM model (default: gpt-4o)
+- REM_DEFAULT_MODEL: LLM model (default: gpt-4.1)
 - REM_LOOKBACK_HOURS: Default lookback window (default: 24)
 - OPENAI_API_KEY: OpenAI API key
 
@@ -83,7 +83,7 @@ def get_worker() -> DreamingWorker:
         embedding_provider=os.getenv(
             "REM_EMBEDDING_PROVIDER", "text-embedding-3-small"
         ),
-        default_model=os.getenv("REM_DEFAULT_MODEL", "gpt-4o"),
+        default_model=os.getenv("REM_DEFAULT_MODEL", "gpt-4.1"),
         lookback_hours=int(os.getenv("REM_LOOKBACK_HOURS", "24")),
     )
 

rem/cli/main.py

@@ -96,6 +96,7 @@ from .commands.serve import register_command as register_serve_command
 from .commands.mcp import register_command as register_mcp_command
 from .commands.scaffold import scaffold as scaffold_command
 from .commands.cluster import register_commands as register_cluster_commands
+from .commands.session import register_command as register_session_command
 
 register_schema_commands(schema)
 register_db_commands(db)
@@ -108,6 +109,7 @@ register_serve_command(cli)
 register_mcp_command(cli)
 cli.add_command(experiments_group)
 cli.add_command(scaffold_command)
+register_session_command(cli)
 
 
 def main():

rem/config.py

@@ -95,9 +95,16 @@ def load_config() -> dict[str, Any]:
     """
     Load configuration from ~/.rem/config.yaml.
 
+    Set REM_SKIP_CONFIG=1 to skip loading the config file (useful when using .env files).
+
     Returns:
-        Configuration dictionary (empty if file doesn't exist)
+        Configuration dictionary (empty if file doesn't exist or skipped)
     """
+    # Allow skipping config file via environment variable
+    if os.environ.get("REM_SKIP_CONFIG", "").lower() in ("1", "true", "yes"):
+        logger.debug("Skipping config file (REM_SKIP_CONFIG is set)")
+        return {}
+
     config_path = get_config_path()
 
     if not config_path.exists():

rem/models/core/experiment.py

@@ -138,18 +138,14 @@ class DatasetReference(BaseModel):
 
     path: str = Field(
         description=(
-            "Path to dataset:\n"
+            "Path to dataset. Format is inferred from file extension.\n"
+            "Supported: .csv, .tsv, .parquet, .json, .jsonl, .xlsx, .ods, .avro, .ipc\n"
             "- Git: Relative path from experiment root (e.g., 'datasets/ground_truth.csv')\n"
-            "- S3: Full S3 URI (e.g., 's3://bucket/experiments/my-exp/datasets/ground_truth.csv')\n"
+            "- S3: Full S3 URI (e.g., 's3://bucket/experiments/my-exp/datasets/data.parquet')\n"
             "- Hybrid: S3 URI for data, Git path for schema"
         )
     )
 
-    format: Literal["csv", "jsonl", "parquet", "json"] = Field(
-        default="csv",
-        description="Dataset file format"
-    )
-
     schema_path: str | None = Field(
         default=None,
         description=(
@@ -262,8 +258,7 @@ class ExperimentConfig(BaseModel):
     datasets:
       ground_truth:
         location: git
-        path: datasets/ground_truth.csv
-        format: csv
+        path: datasets/ground_truth.csv  # format inferred from extension
     results:
       location: git
       base_path: results/
@@ -288,12 +283,10 @@ class ExperimentConfig(BaseModel):
       ground_truth:
         location: s3
         path: s3://rem-prod/experiments/cv-parser-production/datasets/ground_truth.parquet
-        format: parquet
         schema_path: datasets/schema.yaml  # Schema in Git for documentation
       test_cases:
         location: s3
         path: s3://rem-prod/experiments/cv-parser-production/datasets/test_cases.jsonl
-        format: jsonl
     results:
       location: hybrid
       base_path: s3://rem-prod/experiments/cv-parser-production/results/
@@ -318,6 +311,15 @@ class ExperimentConfig(BaseModel):
         )
     )
 
+    task: str = Field(
+        default="general",
+        description=(
+            "Task name for organizing experiments by purpose.\n"
+            "Used with agent name to form directory: {agent}/{task}/\n"
+            "Examples: 'risk-assessment', 'classification', 'general'"
+        )
+    )
+
     description: str = Field(
         description="Human-readable description of experiment purpose and goals"
     )
@@ -410,6 +412,24 @@ class ExperimentConfig(BaseModel):
 
         return v
 
+    @field_validator("task")
+    @classmethod
+    def validate_task(cls, v: str) -> str:
+        """Validate task name follows conventions."""
+        if not v:
+            return "general"  # Default value
+
+        if not v.islower():
+            raise ValueError("Task name must be lowercase")
+
+        if " " in v:
+            raise ValueError("Task name cannot contain spaces (use hyphens)")
+
+        if not all(c.isalnum() or c == "-" for c in v):
+            raise ValueError("Task name can only contain lowercase letters, numbers, and hyphens")
+
+        return v
+
     @field_validator("tags")
     @classmethod
     def validate_tags(cls, v: list[str]) -> list[str]:
@@ -420,6 +440,15 @@ class ExperimentConfig(BaseModel):
         """Get the experiment directory path."""
         return Path(base_path) / self.name
 
+    def get_agent_task_dir(self, base_path: str = ".experiments") -> Path:
+        """
+        Get the experiment directory path organized by agent/task.
+
+        Returns: Path like .experiments/{agent}/{task}/
+        This is the recommended structure for S3 export compatibility.
+        """
+        return Path(base_path) / self.agent_schema_ref.name / self.task
+
     def get_config_path(self, base_path: str = ".experiments") -> Path:
         """Get the path to experiment.yaml file."""
         return self.get_experiment_dir(base_path) / "experiment.yaml"
@@ -428,6 +457,22 @@ class ExperimentConfig(BaseModel):
         """Get the path to README.md file."""
         return self.get_experiment_dir(base_path) / "README.md"
 
+    def get_evaluator_filename(self) -> str:
+        """
+        Get the evaluator filename with task prefix.
+
+        Returns: {agent_name}-{task}.yaml (e.g., siggy-risk-assessment.yaml)
+        """
+        return f"{self.agent_schema_ref.name}-{self.task}.yaml"
+
+    def get_s3_export_path(self, bucket: str, version: str = "v0") -> str:
+        """
+        Get the S3 path for exporting this experiment.
+
+        Returns: s3://{bucket}/{version}/datasets/calibration/experiments/{agent}/{task}/
+        """
+        return f"s3://{bucket}/{version}/datasets/calibration/experiments/{self.agent_schema_ref.name}/{self.task}"
+
     def to_yaml(self) -> str:
         """Export configuration as YAML string."""
         import yaml
@@ -483,6 +528,7 @@ class ExperimentConfig(BaseModel):
 ## Configuration
 
 **Status**: `{self.status.value}`
+**Task**: `{self.task}`
 **Tags**: {', '.join(f'`{tag}`' for tag in self.tags) if self.tags else 'None'}
 
 ## Agent Schema
@@ -494,6 +540,7 @@ class ExperimentConfig(BaseModel):
 ## Evaluator Schema
 
 - **Name**: `{self.evaluator_schema_ref.name}`
+- **File**: `{self.get_evaluator_filename()}`
 - **Type**: `{self.evaluator_schema_ref.type}`
 
 ## Datasets
@@ -504,7 +551,6 @@ class ExperimentConfig(BaseModel):
 
 - **Location**: `{dataset.location.value}`
 - **Path**: `{dataset.path}`
-- **Format**: `{dataset.format}`
 """
             if dataset.description:
                 readme += f"- **Description**: {dataset.description}\n"
@@ -575,7 +621,6 @@ EXAMPLE_SMALL_EXPERIMENT = ExperimentConfig(
         "ground_truth": DatasetReference(
             location=DatasetLocation.GIT,
             path="datasets/ground_truth.csv",
-            format="csv",
             description="10 manually curated test cases"
         )
     },
@@ -605,7 +650,6 @@ EXAMPLE_LARGE_EXPERIMENT = ExperimentConfig(
         "ground_truth": DatasetReference(
             location=DatasetLocation.S3,
             path="s3://rem-prod/experiments/cv-parser-production/datasets/ground_truth.parquet",
-            format="parquet",
             schema_path="datasets/schema.yaml",
             description="10,000 CV/resume pairs with ground truth extractions"
         )

rem/models/entities/__init__.py

@@ -39,6 +39,7 @@ from .shared_session import (
     SharedWithMeResponse,
     SharedWithMeSummary,
 )
+from .subscriber import Subscriber, SubscriberOrigin, SubscriberStatus
 from .user import User, UserTier
 
 __all__ = [
@@ -56,6 +57,9 @@ __all__ = [
     "FeedbackCategory",
     "User",
     "UserTier",
+    "Subscriber",
+    "SubscriberStatus",
+    "SubscriberOrigin",
     "File",
     "Moment",
     "Schema",

rem/models/entities/ontology.py

@@ -129,7 +129,7 @@ class Ontology(CoreModel):
             file_id="file-uuid-456",
             agent_schema_id="contract-parser-v2",
             provider_name="openai",
-            model_name="gpt-4o",
+            model_name="gpt-4.1",
             extracted_data={
                 "contract_type": "supplier_agreement",
                 "parties": [

rem/models/entities/ontology_config.py

@@ -74,7 +74,7 @@ class OntologyConfig(CoreModel):
             priority=200,  # Higher priority = runs first
             enabled=True,
             provider_name="openai",  # Override default provider
-            model_name="gpt-4o",
+            model_name="gpt-4.1",
             tenant_id="acme-corp",
             tags=["legal", "procurement"]
         )