remdb 0.3.14__py3-none-any.whl → 0.3.157__py3-none-any.whl

rem/cli/commands/process.py

@@ -12,12 +12,12 @@ from rem.services.content import ContentService
 
 @click.command(name="ingest")
 @click.argument("file_path", type=click.Path(exists=True))
-@click.option("--user-id", required=True, help="User ID to own the file")
+@click.option("--user-id", default=None, help="User ID to scope file privately (default: public/shared)")
 @click.option("--category", help="Optional file category")
 @click.option("--tags", help="Optional comma-separated tags")
 def process_ingest(
     file_path: str,
-    user_id: str,
+    user_id: str | None,
     category: str | None,
     tags: str | None,
 ):
@@ -32,8 +32,9 @@ def process_ingest(
     5. Creates a File entity record.
 
     Examples:
-        rem process ingest sample.pdf --user-id user-123
-        rem process ingest contract.docx --user-id user-123 --category legal --tags contract,2023
+        rem process ingest sample.pdf
+        rem process ingest contract.docx --category legal --tags contract,2023
+        rem process ingest agent.yaml  # Auto-detects kind=agent, saves to schemas table
     """
     import asyncio
     from ...services.content import ContentService
@@ -56,7 +57,8 @@ def process_ingest(
 
             tag_list = tags.split(",") if tags else None
 
-            logger.info(f"Ingesting file: {file_path} for user: {user_id}")
+            scope_msg = f"user: {user_id}" if user_id else "public"
+            logger.info(f"Ingesting file: {file_path} ({scope_msg})")
             result = await service.ingest_file(
                 file_uri=file_path,
                 user_id=user_id,
@@ -65,11 +67,15 @@ def process_ingest(
                 is_local_server=True, # CLI is local
             )
 
-            if result.get("processing_status") == "completed":
-                logger.success(f"File ingested successfully: {result['file_name']}")
+            # Handle schema ingestion (agents/evaluators)
+            if result.get("schema_name"):
+                logger.success(f"Schema ingested: {result['schema_name']} (kind={result.get('kind', 'agent')})")
+                logger.info(f"Version: {result.get('version', '1.0.0')}")
+            # Handle file ingestion
+            elif result.get("processing_status") == "completed":
+                logger.success(f"File ingested: {result['file_name']}")
                 logger.info(f"File ID: {result['file_id']}")
                 logger.info(f"Resources created: {result['resources_created']}")
-                logger.info(f"Status: {result['processing_status']}")
             else:
                 logger.error(f"Ingestion failed: {result.get('message', 'Unknown error')}")
                 sys.exit(1)

rem/cli/commands/schema.py

@@ -8,6 +8,7 @@ Usage:
 """
 
 import asyncio
+import importlib
 from pathlib import Path
 
 import click
@@ -15,68 +16,116 @@ from loguru import logger
 
 from ...settings import settings
 from ...services.postgres.schema_generator import SchemaGenerator
+from ...utils.sql_paths import get_package_sql_dir, get_package_migrations_dir
+
+
+def _import_model_modules() -> list[str]:
+    """
+    Import modules specified in MODELS__IMPORT_MODULES setting.
+
+    This ensures downstream models decorated with @rem.register_model
+    are registered before schema generation.
+
+    Returns:
+        List of successfully imported module names
+    """
+    imported = []
+    for module_name in settings.models.module_list:
+        try:
+            importlib.import_module(module_name)
+            imported.append(module_name)
+            logger.debug(f"Imported model module: {module_name}")
+        except ImportError as e:
+            logger.warning(f"Failed to import model module '{module_name}': {e}")
+            click.echo(
+                click.style(f"  ⚠ Could not import '{module_name}': {e}", fg="yellow"),
+                err=True,
+            )
+    return imported
 
 
 @click.command()
-@click.option(
-    "--models",
-    "-m",
-    required=True,
-    type=click.Path(exists=True, path_type=Path),
-    help="Directory containing Pydantic models",
-)
 @click.option(
     "--output",
     "-o",
     type=click.Path(path_type=Path),
-    default="install_models.sql",
-    help="Output SQL file (default: install_models.sql)",
+    default="002_install_models.sql",
+    help="Output SQL file (default: 002_install_models.sql)",
 )
 @click.option(
     "--output-dir",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Base output directory (default: {settings.sql_dir})",
+    help="Base output directory (default: package sql/migrations)",
 )
-def generate(models: Path, output: Path, output_dir: Path | None):
+def generate(output: Path, output_dir: Path | None):
     """
-    Generate database schema from Pydantic models.
+    Generate database schema from registered Pydantic models.
 
-    Scans the specified directory for Pydantic models and generates:
+    Uses the model registry (core models + user-registered models) to generate:
     - CREATE TABLE statements
     - Embeddings tables (embeddings_<table>)
     - KV_STORE triggers for cache maintenance
     - Indexes (foreground only)
 
-    Output is written to src/rem/sql/install_models.sql by default.
+    Output is written to src/rem/sql/migrations/002_install_models.sql by default.
 
     Example:
-        rem db schema generate --models src/rem/models/entities
+        rem db schema generate
+
+    To register custom models in downstream apps:
+
+    1. Create models with @rem.register_model decorator:
+
+        # models/__init__.py
+        import rem
+        from rem.models.core import CoreModel
+
+        @rem.register_model
+        class MyEntity(CoreModel):
+            name: str
+
+    2. Set MODELS__IMPORT_MODULES in your .env:
+
+        MODELS__IMPORT_MODULES=models
+
+    3. Run schema generation:
+
+        rem db schema generate
 
     This creates:
-    - src/rem/sql/install_models.sql - Entity tables and triggers
+    - src/rem/sql/migrations/002_install_models.sql - Entity tables and triggers
     - src/rem/sql/background_indexes.sql - HNSW indexes (apply after data load)
 
-    After generation, apply with:
-        rem db migrate
+    After generation, verify with:
+        rem db diff
     """
-    click.echo(f"Discovering models in {models}")
+    from ...registry import get_model_registry
+
+    # Import downstream model modules to trigger @rem.register_model decorators
+    imported_modules = _import_model_modules()
+    if imported_modules:
+        click.echo(f"Imported model modules: {', '.join(imported_modules)}")
+
+    registry = get_model_registry()
+    models = registry.get_models(include_core=True)
+    click.echo(f"Generating schema from {len(models)} registered models")
 
-    # Use settings.sql_dir if not provided
-    actual_output_dir = output_dir or Path(settings.sql_dir)
+    # Default to package migrations directory
+    actual_output_dir = output_dir or get_package_migrations_dir()
     generator = SchemaGenerator(output_dir=actual_output_dir)
 
-    # Generate schema
+    # Generate schema from registry
     try:
-        schema_sql = asyncio.run(generator.generate_from_directory(models, output_file=output.name))
+        schema_sql = asyncio.run(generator.generate_from_registry(output_file=output.name))
 
         click.echo(f"✓ Schema generated: {len(generator.schemas)} tables")
         click.echo(f"✓ Written to: {actual_output_dir / output.name}")
 
-        # Generate background indexes
+        # Generate background indexes in parent sql dir
         background_indexes = generator.generate_background_indexes()
         if background_indexes:
-            bg_file = actual_output_dir / "background_indexes.sql"
+            bg_file = get_package_sql_dir() / "background_indexes.sql"
             bg_file.write_text(background_indexes)
             click.echo(f"✓ Background indexes: {bg_file}")
 
@@ -94,48 +143,46 @@ def generate(models: Path, output: Path, output_dir: Path | None):
 
 
 @click.command()
-@click.option(
-    "--models",
-    "-m",
-    required=True,
-    type=click.Path(exists=True, path_type=Path),
-    help="Directory containing Pydantic models",
-)
-def validate(models: Path):
+def validate():
     """
-    Validate Pydantic models for schema generation.
+    Validate registered Pydantic models for schema generation.
 
     Checks:
-    - Models can be loaded
+    - Models can be loaded from registry
     - Models have suitable entity_key fields
     - Fields with embeddings are properly configured
+
+    Set MODELS__IMPORT_MODULES to include custom models from downstream apps.
     """
-    click.echo(f"Validating models in {models}")
+    from ...registry import get_model_registry
 
-    generator = SchemaGenerator()
-    discovered = generator.discover_models(models)
+    # Import downstream model modules to trigger @rem.register_model decorators
+    imported_modules = _import_model_modules()
+    if imported_modules:
+        click.echo(f"Imported model modules: {', '.join(imported_modules)}")
 
-    if not discovered:
-        click.echo("✗ No models found", err=True)
-        raise click.Abort()
+    registry = get_model_registry()
+    models = registry.get_models(include_core=True)
+
+    click.echo(f"Validating {len(models)} registered models")
 
-    click.echo(f"✓ Discovered {len(discovered)} models")
+    if not models:
+        click.echo("✗ No models found in registry", err=True)
+        raise click.Abort()
 
+    generator = SchemaGenerator()
     errors: list[str] = []
     warnings: list[str] = []
 
-    for model_name, model in discovered.items():
-        table_name = generator.infer_table_name(model)
-        entity_key = generator.infer_entity_key_field(model)
+    for model_name, ext in models.items():
+        model = ext.model
+        table_name = ext.table_name or generator.infer_table_name(model)
+        entity_key = ext.entity_key_field or generator.infer_entity_key_field(model)
 
         # Check for entity_key
         if entity_key == "id":
             warnings.append(f"{model_name}: No natural key field, using 'id'")
 
-        # Check for embeddable fields
-        # TODO: Implement should_embed_field check
-        embeddable: list[str] = []  # Placeholder - needs implementation
-
         click.echo(f"  {model_name} -> {table_name} (key: {entity_key})")
 
     if warnings:
@@ -158,7 +205,7 @@ def validate(models: Path):
     "-o",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Output file for background indexes (default: {settings.sql_dir}/background_indexes.sql)",
+    help="Output file for background indexes (default: package sql/background_indexes.sql)",
 )
 def indexes(output: Path):
     """

rem/cli/commands/session.py

@@ -0,0 +1,336 @@
+"""
+CLI command for viewing and simulating session conversations.
+
+Usage:
+    rem session show <user_id> [--session-id] [--role user|assistant|system]
+    rem session show <user_id> --simulate-next [--save] [--custom-sim-prompt "..."]
+
+Examples:
+    # Show all messages for a user
+    rem session show 11111111-1111-1111-1111-111111111001
+
+    # Show only user messages
+    rem session show 11111111-1111-1111-1111-111111111001 --role user
+
+    # Simulate next user message
+    rem session show 11111111-1111-1111-1111-111111111001 --simulate-next
+
+    # Simulate with custom prompt and save
+    rem session show 11111111-1111-1111-1111-111111111001 --simulate-next --save \
+        --custom-sim-prompt "Respond as an anxious patient"
+"""
+
+import asyncio
+from pathlib import Path
+from typing import Literal
+
+import click
+import yaml
+from loguru import logger
+
+from ...models.entities.user import User
+from ...models.entities.message import Message
+from ...services.postgres import get_postgres_service
+from ...services.postgres.repository import Repository
+from ...settings import settings
+
+
+SIMULATOR_PROMPT = """You are simulating a patient in a mental health conversation.
+
+## Context
+You are continuing a conversation with a clinical evaluation agent. Based on the
+user profile and conversation history below, generate the next realistic patient message.
+
+## User Profile
+{user_profile}
+
+## Conversation History
+{conversation_history}
+
+## Instructions
+- Stay in character as the patient described in the profile
+- Your response should be natural, conversational, and consistent with the patient's presentation
+- Consider the patient's risk level, symptoms, and communication style
+- Do NOT include any metadata or role labels - just the raw message content
+- Keep responses concise (1-3 sentences typical for conversation)
+
+Generate the next patient message:"""
+
+
+async def _load_user_and_messages(
+    user_id: str,
+    session_id: str | None = None,
+    role_filter: str | None = None,
+    limit: int = 100,
+) -> tuple[User | None, list[Message]]:
+    """Load user profile and messages from database."""
+    pg = get_postgres_service()
+    if not pg:
+        logger.error("PostgreSQL not available")
+        return None, []
+
+    await pg.connect()
+
+    try:
+        # Load user
+        user_repo = Repository(User, "users", db=pg)
+        user = await user_repo.get_by_id(user_id, tenant_id="default")
+
+        # Load messages
+        message_repo = Repository(Message, "messages", db=pg)
+        filters = {"user_id": user_id}
+        if session_id:
+            filters["session_id"] = session_id
+
+        messages = await message_repo.find(
+            filters=filters,
+            order_by="created_at ASC",
+            limit=limit,
+        )
+
+        # Filter by role if specified
+        if role_filter:
+            messages = [m for m in messages if m.message_type == role_filter]
+
+        return user, messages
+
+    finally:
+        await pg.disconnect()
+
+
+def _format_user_yaml(user: User | None) -> str:
+    """Format user profile as YAML."""
+    if not user:
+        return "# No user found"
+
+    data = {
+        "id": str(user.id),
+        "name": user.name,
+        "summary": user.summary,
+        "interests": user.interests,
+        "preferred_topics": user.preferred_topics,
+        "metadata": user.metadata,
+    }
+    return yaml.dump(data, default_flow_style=False, allow_unicode=True)
+
+
+def _format_messages_yaml(messages: list[Message]) -> str:
+    """Format messages as YAML."""
+    if not messages:
+        return "# No messages found"
+
+    data = []
+    for msg in messages:
+        data.append({
+            "role": msg.message_type or "unknown",
+            "content": msg.content,
+            "session_id": msg.session_id,
+            "created_at": msg.created_at.isoformat() if msg.created_at else None,
+        })
+    return yaml.dump(data, default_flow_style=False, allow_unicode=True)
+
+
+def _format_conversation_for_llm(messages: list[Message]) -> str:
+    """Format conversation history for LLM context."""
+    lines = []
+    for msg in messages:
+        role = msg.message_type or "unknown"
+        lines.append(f"[{role.upper()}]: {msg.content}")
+    return "\n\n".join(lines) if lines else "(No previous messages)"
+
+
+async def _simulate_next_message(
+    user: User | None,
+    messages: list[Message],
+    custom_prompt: str | None = None,
+) -> str:
+    """Use LLM to simulate the next patient message."""
+    from pydantic_ai import Agent
+
+    # Build context
+    user_profile = _format_user_yaml(user) if user else "Unknown patient"
+    conversation_history = _format_conversation_for_llm(messages)
+
+    # Use custom prompt or default
+    if custom_prompt:
+        # Check if it's a file path
+        if Path(custom_prompt).exists():
+            prompt_template = Path(custom_prompt).read_text()
+        else:
+            prompt_template = custom_prompt
+        # Simple variable substitution
+        prompt = prompt_template.replace("{user_profile}", user_profile)
+        prompt = prompt.replace("{conversation_history}", conversation_history)
+    else:
+        prompt = SIMULATOR_PROMPT.format(
+            user_profile=user_profile,
+            conversation_history=conversation_history,
+        )
+
+    # Create simple agent for simulation
+    agent = Agent(
+        model=settings.llm.default_model,
+        system_prompt="You are a patient simulator. Generate realistic patient responses.",
+    )
+
+    result = await agent.run(prompt)
+    return result.output
+
+
+async def _save_message(
+    user_id: str,
+    session_id: str | None,
+    content: str,
+    role: str = "user",
+) -> Message:
+    """Save a simulated message to the database."""
+    from uuid import uuid4
+
+    pg = get_postgres_service()
+    if not pg:
+        raise RuntimeError("PostgreSQL not available")
+
+    await pg.connect()
+
+    try:
+        message_repo = Repository(Message, "messages", db=pg)
+
+        message = Message(
+            id=uuid4(),
+            user_id=user_id,
+            tenant_id="default",
+            session_id=session_id or str(uuid4()),
+            content=content,
+            message_type=role,
+        )
+
+        await message_repo.upsert(message)
+        return message
+
+    finally:
+        await pg.disconnect()
+
+
+@click.group()
+def session():
+    """Session viewing and simulation commands."""
+    pass
+
+
+@session.command("show")
+@click.argument("user_id")
+@click.option("--session-id", "-s", help="Filter by session ID")
+@click.option(
+    "--role", "-r",
+    type=click.Choice(["user", "assistant", "system", "tool"]),
+    help="Filter messages by role",
+)
+@click.option("--limit", "-l", default=100, help="Max messages to load")
+@click.option("--simulate-next", is_flag=True, help="Simulate the next patient message")
+@click.option("--save", is_flag=True, help="Save simulated message to database")
+@click.option(
+    "--custom-sim-prompt", "-p",
+    help="Custom simulation prompt (text or file path)",
+)
+def show(
+    user_id: str,
+    session_id: str | None,
+    role: str | None,
+    limit: int,
+    simulate_next: bool,
+    save: bool,
+    custom_sim_prompt: str | None,
+):
+    """
+    Show user profile and session messages.
+
+    USER_ID: The user identifier to load.
+
+    Examples:
+
+        # Show user and all messages
+        rem session show 11111111-1111-1111-1111-111111111001
+
+        # Show only assistant responses
+        rem session show 11111111-1111-1111-1111-111111111001 --role assistant
+
+        # Simulate next patient message
+        rem session show 11111111-1111-1111-1111-111111111001 --simulate-next
+
+        # Simulate and save to database
+        rem session show 11111111-1111-1111-1111-111111111001 --simulate-next --save
+    """
+    asyncio.run(_show_async(
+        user_id=user_id,
+        session_id=session_id,
+        role_filter=role,
+        limit=limit,
+        simulate_next=simulate_next,
+        save=save,
+        custom_sim_prompt=custom_sim_prompt,
+    ))
+
+
+async def _show_async(
+    user_id: str,
+    session_id: str | None,
+    role_filter: str | None,
+    limit: int,
+    simulate_next: bool,
+    save: bool,
+    custom_sim_prompt: str | None,
+):
+    """Async implementation of show command."""
+    # Load data
+    user, messages = await _load_user_and_messages(
+        user_id=user_id,
+        session_id=session_id,
+        role_filter=role_filter if not simulate_next else None,  # Need all messages for simulation
+        limit=limit,
+    )
+
+    # Display user profile
+    click.echo("\n# User Profile")
+    click.echo("---")
+    click.echo(_format_user_yaml(user))
+
+    # Display messages (apply filter for display if simulating)
+    display_messages = messages
+    if simulate_next and role_filter:
+        display_messages = [m for m in messages if m.message_type == role_filter]
+
+    click.echo("\n# Messages")
+    click.echo("---")
+    click.echo(_format_messages_yaml(display_messages))
+
+    # Simulate next message if requested
+    if simulate_next:
+        click.echo("\n# Simulated Next Message")
+        click.echo("---")
+
+        try:
+            simulated = await _simulate_next_message(
+                user=user,
+                messages=messages,
+                custom_prompt=custom_sim_prompt,
+            )
+            click.echo(f"role: user")
+            click.echo(f"content: |\n  {simulated}")
+
+            if save:
+                saved_msg = await _save_message(
+                    user_id=user_id,
+                    session_id=session_id,
+                    content=simulated,
+                    role="user",
+                )
+                logger.success(f"Saved message: {saved_msg.id}")
+
+        except Exception as e:
+            logger.error(f"Simulation failed: {e}")
+            raise
+
+
+def register_command(cli_group):
+    """Register the session command group."""
+    cli_group.add_command(session)

rem/cli/dreaming.py

@@ -43,7 +43,7 @@ rem-dreaming full --user-id=user-123 --rem-api-url=http://localhost:8000
 Environment Variables:
 - REM_API_URL: REM API endpoint (default: http://rem-api:8000)
 - REM_EMBEDDING_PROVIDER: Embedding provider (default: text-embedding-3-small)
-- REM_DEFAULT_MODEL: LLM model (default: gpt-4o)
+- REM_DEFAULT_MODEL: LLM model (default: gpt-4.1)
 - REM_LOOKBACK_HOURS: Default lookback window (default: 24)
 - OPENAI_API_KEY: OpenAI API key
 
@@ -83,7 +83,7 @@ def get_worker() -> DreamingWorker:
         embedding_provider=os.getenv(
             "REM_EMBEDDING_PROVIDER", "text-embedding-3-small"
         ),
-        default_model=os.getenv("REM_DEFAULT_MODEL", "gpt-4o"),
+        default_model=os.getenv("REM_DEFAULT_MODEL", "gpt-4.1"),
         lookback_hours=int(os.getenv("REM_LOOKBACK_HOURS", "24")),
     )