remdb 0.3.7__py3-none-any.whl → 0.3.133__py3-none-any.whl

rem/cli/commands/process.py

@@ -12,12 +12,12 @@ from rem.services.content import ContentService
 
 @click.command(name="ingest")
 @click.argument("file_path", type=click.Path(exists=True))
-@click.option("--user-id", required=True, help="User ID to own the file")
+@click.option("--user-id", default=None, help="User ID to scope file privately (default: public/shared)")
 @click.option("--category", help="Optional file category")
 @click.option("--tags", help="Optional comma-separated tags")
 def process_ingest(
     file_path: str,
-    user_id: str,
+    user_id: str | None,
     category: str | None,
     tags: str | None,
 ):
@@ -32,8 +32,9 @@ def process_ingest(
     5. Creates a File entity record.
 
     Examples:
-        rem process ingest sample.pdf --user-id user-123
-        rem process ingest contract.docx --user-id user-123 --category legal --tags contract,2023
+        rem process ingest sample.pdf
+        rem process ingest contract.docx --category legal --tags contract,2023
+        rem process ingest agent.yaml  # Auto-detects kind=agent, saves to schemas table
     """
     import asyncio
     from ...services.content import ContentService
@@ -56,7 +57,8 @@ def process_ingest(
 
             tag_list = tags.split(",") if tags else None
 
-            logger.info(f"Ingesting file: {file_path} for user: {user_id}")
+            scope_msg = f"user: {user_id}" if user_id else "public"
+            logger.info(f"Ingesting file: {file_path} ({scope_msg})")
             result = await service.ingest_file(
                 file_uri=file_path,
                 user_id=user_id,
@@ -65,11 +67,15 @@ def process_ingest(
                 is_local_server=True, # CLI is local
             )
 
-            if result.get("processing_status") == "completed":
-                logger.success(f"File ingested successfully: {result['file_name']}")
+            # Handle schema ingestion (agents/evaluators)
+            if result.get("schema_name"):
+                logger.success(f"Schema ingested: {result['schema_name']} (kind={result.get('kind', 'agent')})")
+                logger.info(f"Version: {result.get('version', '1.0.0')}")
+            # Handle file ingestion
+            elif result.get("processing_status") == "completed":
+                logger.success(f"File ingested: {result['file_name']}")
                 logger.info(f"File ID: {result['file_id']}")
                 logger.info(f"Resources created: {result['resources_created']}")
-                logger.info(f"Status: {result['processing_status']}")
             else:
                 logger.error(f"Ingestion failed: {result.get('message', 'Unknown error')}")
                 sys.exit(1)
@@ -192,15 +198,13 @@ def process_uri(uri: str, output: str, save: str | None):
 
 
 @click.command(name="files")
-@click.option("--tenant-id", required=True, help="Tenant ID")
-@click.option("--user-id", help="Filter by user ID")
+@click.option("--user-id", default=None, help="User ID (default: from settings)")
 @click.option("--status", type=click.Choice(["pending", "processing", "completed", "failed"]), help="Filter by status")
 @click.option("--extractor", help="Run files through custom extractor (e.g., cv-parser-v1)")
 @click.option("--limit", type=int, help="Max files to process")
 @click.option("--provider", help="Optional LLM provider override")
 @click.option("--model", help="Optional model override")
 def process_files(
-    tenant_id: str,
     user_id: Optional[str],
     status: Optional[str],
     extractor: Optional[str],
@@ -217,19 +221,22 @@ def process_files(
 
         \b
         # List completed files
-        rem process files --tenant-id acme-corp --status completed
+        rem process files --status completed
 
         \b
         # Extract from CV files
-        rem process files --tenant-id acme-corp --extractor cv-parser-v1 --limit 10
+        rem process files --extractor cv-parser-v1 --limit 10
 
         \b
         # Extract with provider override
-        rem process files --tenant-id acme-corp --extractor contract-analyzer-v1 \\
+        rem process files --extractor contract-analyzer-v1 \\
             --provider anthropic --model claude-sonnet-4-5
     """
+    from ...settings import settings
+    effective_user_id = user_id or settings.test.effective_user_id
+
     logger.warning("Not implemented yet")
-    logger.info(f"Would process files for tenant: {tenant_id}")
+    logger.info(f"Would process files for user: {effective_user_id}")
 
     if user_id:
         logger.info(f"Filter: user_id={user_id}")

rem/cli/commands/scaffold.py

@@ -0,0 +1,47 @@
+"""
+Scaffold command - generate project structure for REM-based applications.
+
+TODO: Implement this command to generate:
+- my_app/main.py (entry point with create_app)
+- my_app/models.py (example CoreModel subclass)
+- my_app/routers/ (example FastAPI router)
+- schemas/agents/ (example agent schema)
+- schemas/evaluators/ (example evaluator)
+- sql/migrations/ (empty migrations directory)
+- pyproject.toml (with remdb dependency)
+- README.md (basic usage instructions)
+
+Usage:
+    rem scaffold my-app
+    rem scaffold my-app --with-examples  # Include example models/routers/tools
+"""
+
+import click
+
+
+@click.command()
+@click.argument("name")
+@click.option("--with-examples", is_flag=True, help="Include example code")
+def scaffold(name: str, with_examples: bool) -> None:
+    """
+    Generate a new REM-based project structure.
+
+    NAME is the project directory name to create.
+    """
+    click.echo(f"TODO: Scaffold command not yet implemented")
+    click.echo(f"Would create project: {name}")
+    click.echo(f"With examples: {with_examples}")
+    click.echo()
+    click.echo("For now, manually create this structure:")
+    click.echo(f"""
+{name}/
+├── {name.replace('-', '_')}/
+│   ├── main.py           # Entry point (create_app + extensions)
+│   ├── models.py         # Custom models (inherit CoreModel)
+│   └── routers/          # Custom FastAPI routers
+├── schemas/
+│   ├── agents/           # Custom agent YAML schemas
+│   └── evaluators/       # Custom evaluator schemas
+├── sql/migrations/       # Custom SQL migrations
+└── pyproject.toml
+""")

rem/cli/commands/schema.py

@@ -8,6 +8,7 @@ Usage:
 """
 
 import asyncio
+import importlib
 from pathlib import Path
 
 import click
@@ -15,68 +16,116 @@ from loguru import logger
 
 from ...settings import settings
 from ...services.postgres.schema_generator import SchemaGenerator
+from ...utils.sql_paths import get_package_sql_dir, get_package_migrations_dir
+
+
+def _import_model_modules() -> list[str]:
+    """
+    Import modules specified in MODELS__IMPORT_MODULES setting.
+
+    This ensures downstream models decorated with @rem.register_model
+    are registered before schema generation.
+
+    Returns:
+        List of successfully imported module names
+    """
+    imported = []
+    for module_name in settings.models.module_list:
+        try:
+            importlib.import_module(module_name)
+            imported.append(module_name)
+            logger.debug(f"Imported model module: {module_name}")
+        except ImportError as e:
+            logger.warning(f"Failed to import model module '{module_name}': {e}")
+            click.echo(
+                click.style(f"  ⚠ Could not import '{module_name}': {e}", fg="yellow"),
+                err=True,
+            )
+    return imported
 
 
 @click.command()
-@click.option(
-    "--models",
-    "-m",
-    required=True,
-    type=click.Path(exists=True, path_type=Path),
-    help="Directory containing Pydantic models",
-)
 @click.option(
     "--output",
     "-o",
     type=click.Path(path_type=Path),
-    default="install_models.sql",
-    help="Output SQL file (default: install_models.sql)",
+    default="002_install_models.sql",
+    help="Output SQL file (default: 002_install_models.sql)",
 )
 @click.option(
     "--output-dir",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Base output directory (default: {settings.sql_dir})",
+    help="Base output directory (default: package sql/migrations)",
 )
-def generate(models: Path, output: Path, output_dir: Path | None):
+def generate(output: Path, output_dir: Path | None):
     """
-    Generate database schema from Pydantic models.
+    Generate database schema from registered Pydantic models.
 
-    Scans the specified directory for Pydantic models and generates:
+    Uses the model registry (core models + user-registered models) to generate:
     - CREATE TABLE statements
     - Embeddings tables (embeddings_<table>)
     - KV_STORE triggers for cache maintenance
     - Indexes (foreground only)
 
-    Output is written to src/rem/sql/install_models.sql by default.
+    Output is written to src/rem/sql/migrations/002_install_models.sql by default.
 
     Example:
-        rem db schema generate --models src/rem/models/entities
+        rem db schema generate
+
+    To register custom models in downstream apps:
+
+    1. Create models with @rem.register_model decorator:
+
+        # models/__init__.py
+        import rem
+        from rem.models.core import CoreModel
+
+        @rem.register_model
+        class MyEntity(CoreModel):
+            name: str
+
+    2. Set MODELS__IMPORT_MODULES in your .env:
+
+        MODELS__IMPORT_MODULES=models
+
+    3. Run schema generation:
+
+        rem db schema generate
 
     This creates:
-    - src/rem/sql/install_models.sql - Entity tables and triggers
+    - src/rem/sql/migrations/002_install_models.sql - Entity tables and triggers
     - src/rem/sql/background_indexes.sql - HNSW indexes (apply after data load)
 
-    After generation, apply with:
-        rem db migrate
+    After generation, verify with:
+        rem db diff
     """
-    click.echo(f"Discovering models in {models}")
+    from ...registry import get_model_registry
+
+    # Import downstream model modules to trigger @rem.register_model decorators
+    imported_modules = _import_model_modules()
+    if imported_modules:
+        click.echo(f"Imported model modules: {', '.join(imported_modules)}")
+
+    registry = get_model_registry()
+    models = registry.get_models(include_core=True)
+    click.echo(f"Generating schema from {len(models)} registered models")
 
-    # Use settings.sql_dir if not provided
-    actual_output_dir = output_dir or Path(settings.sql_dir)
+    # Default to package migrations directory
+    actual_output_dir = output_dir or get_package_migrations_dir()
     generator = SchemaGenerator(output_dir=actual_output_dir)
 
-    # Generate schema
+    # Generate schema from registry
     try:
-        schema_sql = asyncio.run(generator.generate_from_directory(models, output_file=output.name))
+        schema_sql = asyncio.run(generator.generate_from_registry(output_file=output.name))
 
         click.echo(f"✓ Schema generated: {len(generator.schemas)} tables")
         click.echo(f"✓ Written to: {actual_output_dir / output.name}")
 
-        # Generate background indexes
+        # Generate background indexes in parent sql dir
         background_indexes = generator.generate_background_indexes()
         if background_indexes:
-            bg_file = actual_output_dir / "background_indexes.sql"
+            bg_file = get_package_sql_dir() / "background_indexes.sql"
             bg_file.write_text(background_indexes)
             click.echo(f"✓ Background indexes: {bg_file}")
 
@@ -94,48 +143,46 @@ def generate(models: Path, output: Path, output_dir: Path | None):
 
 
 @click.command()
-@click.option(
-    "--models",
-    "-m",
-    required=True,
-    type=click.Path(exists=True, path_type=Path),
-    help="Directory containing Pydantic models",
-)
-def validate(models: Path):
+def validate():
     """
-    Validate Pydantic models for schema generation.
+    Validate registered Pydantic models for schema generation.
 
     Checks:
-    - Models can be loaded
+    - Models can be loaded from registry
     - Models have suitable entity_key fields
     - Fields with embeddings are properly configured
+
+    Set MODELS__IMPORT_MODULES to include custom models from downstream apps.
     """
-    click.echo(f"Validating models in {models}")
+    from ...registry import get_model_registry
 
-    generator = SchemaGenerator()
-    discovered = generator.discover_models(models)
+    # Import downstream model modules to trigger @rem.register_model decorators
+    imported_modules = _import_model_modules()
+    if imported_modules:
+        click.echo(f"Imported model modules: {', '.join(imported_modules)}")
 
-    if not discovered:
-        click.echo("✗ No models found", err=True)
-        raise click.Abort()
+    registry = get_model_registry()
+    models = registry.get_models(include_core=True)
+
+    click.echo(f"Validating {len(models)} registered models")
 
-    click.echo(f"✓ Discovered {len(discovered)} models")
+    if not models:
+        click.echo("✗ No models found in registry", err=True)
+        raise click.Abort()
 
+    generator = SchemaGenerator()
     errors: list[str] = []
     warnings: list[str] = []
 
-    for model_name, model in discovered.items():
-        table_name = generator.infer_table_name(model)
-        entity_key = generator.infer_entity_key_field(model)
+    for model_name, ext in models.items():
+        model = ext.model
+        table_name = ext.table_name or generator.infer_table_name(model)
+        entity_key = ext.entity_key_field or generator.infer_entity_key_field(model)
 
         # Check for entity_key
         if entity_key == "id":
             warnings.append(f"{model_name}: No natural key field, using 'id'")
 
-        # Check for embeddable fields
-        # TODO: Implement should_embed_field check
-        embeddable: list[str] = []  # Placeholder - needs implementation
-
         click.echo(f"  {model_name} -> {table_name} (key: {entity_key})")
 
     if warnings:
@@ -158,7 +205,7 @@ def validate(models: Path):
     "-o",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Output file for background indexes (default: {settings.sql_dir}/background_indexes.sql)",
+    help="Output file for background indexes (default: package sql/background_indexes.sql)",
 )
 def indexes(output: Path):
     """

rem/cli/main.py

@@ -22,17 +22,30 @@ except Exception:
     __version__ = "unknown"
 
 
+def _configure_logger(level: str):
+    """Configure loguru with custom level icons."""
+    logger.remove()
+
+    # Configure level icons - only warnings and errors get visual indicators
+    logger.level("DEBUG", icon=" ")
+    logger.level("INFO", icon=" ")
+    logger.level("WARNING", icon="🟠")
+    logger.level("ERROR", icon="🔴")
+    logger.level("CRITICAL", icon="🔴")
+
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="<green>{time:HH:mm:ss}</green> | {level.icon} <level>{level: <8}</level> | <level>{message}</level>",
+    )
+
+
 @click.group()
 @click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging")
 @click.version_option(version=__version__, prog_name="rem")
 def cli(verbose: bool):
     """REM - Resources Entities Moments system CLI."""
-    if verbose:
-        logger.remove()
-        logger.add(sys.stderr, level="DEBUG")
-    else:
-        logger.remove()
-        logger.add(sys.stderr, level="INFO")
+    _configure_logger("DEBUG" if verbose else "INFO")
 
 
 @cli.group()
@@ -65,6 +78,12 @@ def dreaming():
     pass
 
 
+@cli.group()
+def cluster():
+    """Kubernetes cluster deployment and management."""
+    pass
+
+
 # Register commands
 from .commands.schema import register_commands as register_schema_commands
 from .commands.db import register_commands as register_db_commands
@@ -75,16 +94,20 @@ from .commands.experiments import experiments as experiments_group
 from .commands.configure import register_command as register_configure_command
 from .commands.serve import register_command as register_serve_command
 from .commands.mcp import register_command as register_mcp_command
+from .commands.scaffold import scaffold as scaffold_command
+from .commands.cluster import register_commands as register_cluster_commands
 
 register_schema_commands(schema)
 register_db_commands(db)
 register_process_commands(process)
 register_dreaming_commands(dreaming)
+register_cluster_commands(cluster)
 register_ask_command(cli)
 register_configure_command(cli)
 register_serve_command(cli)
 register_mcp_command(cli)
 cli.add_command(experiments_group)
+cli.add_command(scaffold_command)
 
 
 def main():

rem/config.py

@@ -15,7 +15,7 @@ File Format (~/.rem/config.yaml):
       pool_max_size: 20
 
     llm:
-      default_model: anthropic:claude-sonnet-4-5-20250929
+      default_model: openai:gpt-4.1
       openai_api_key: sk-...
       anthropic_api_key: sk-ant-...
 
@@ -95,9 +95,16 @@ def load_config() -> dict[str, Any]:
     """
     Load configuration from ~/.rem/config.yaml.
 
+    Set REM_SKIP_CONFIG=1 to skip loading the config file (useful when using .env files).
+
     Returns:
-        Configuration dictionary (empty if file doesn't exist)
+        Configuration dictionary (empty if file doesn't exist or skipped)
     """
+    # Allow skipping config file via environment variable
+    if os.environ.get("REM_SKIP_CONFIG", "").lower() in ("1", "true", "yes"):
+        logger.debug("Skipping config file (REM_SKIP_CONFIG is set)")
+        return {}
+
     config_path = get_config_path()
 
     if not config_path.exists():
@@ -216,7 +223,7 @@ def get_default_config() -> dict[str, Any]:
             "pool_max_size": 20,
         },
         "llm": {
-            "default_model": "anthropic:claude-sonnet-4-5-20250929",
+            "default_model": "openai:gpt-4.1",
             "default_temperature": 0.5,
             # API keys will be prompted for in wizard
             # "openai_api_key": "",

rem/models/core/core_model.py

@@ -52,7 +52,13 @@ class CoreModel(BaseModel):
         default=None, description="Tenant identifier for multi-tenancy isolation"
     )
     user_id: Optional[str] = Field(
-        default=None, description="Owner user identifier (tenant-scoped)"
+        default=None,
+        description=(
+            "Owner user identifier (tenant-scoped). This is a VARCHAR(256), not a UUID, "
+            "to allow flexibility for external identity providers. Typically generated as "
+            "a hash of the user's email address. In future, other strong unique claims "
+            "(e.g., OAuth sub, verified phone) could also be used for generation."
+        ),
     )
     graph_edges: list[dict] = Field(
         default_factory=list,

rem/models/core/experiment.py

@@ -318,6 +318,15 @@ class ExperimentConfig(BaseModel):
         )
     )
 
+    task: str = Field(
+        default="general",
+        description=(
+            "Task name for organizing experiments by purpose.\n"
+            "Used with agent name to form directory: {agent}/{task}/\n"
+            "Examples: 'risk-assessment', 'classification', 'general'"
+        )
+    )
+
     description: str = Field(
         description="Human-readable description of experiment purpose and goals"
     )
@@ -410,6 +419,24 @@ class ExperimentConfig(BaseModel):
 
         return v
 
+    @field_validator("task")
+    @classmethod
+    def validate_task(cls, v: str) -> str:
+        """Validate task name follows conventions."""
+        if not v:
+            return "general"  # Default value
+
+        if not v.islower():
+            raise ValueError("Task name must be lowercase")
+
+        if " " in v:
+            raise ValueError("Task name cannot contain spaces (use hyphens)")
+
+        if not all(c.isalnum() or c == "-" for c in v):
+            raise ValueError("Task name can only contain lowercase letters, numbers, and hyphens")
+
+        return v
+
     @field_validator("tags")
     @classmethod
     def validate_tags(cls, v: list[str]) -> list[str]:
@@ -420,6 +447,15 @@ class ExperimentConfig(BaseModel):
         """Get the experiment directory path."""
         return Path(base_path) / self.name
 
+    def get_agent_task_dir(self, base_path: str = ".experiments") -> Path:
+        """
+        Get the experiment directory path organized by agent/task.
+
+        Returns: Path like .experiments/{agent}/{task}/
+        This is the recommended structure for S3 export compatibility.
+        """
+        return Path(base_path) / self.agent_schema_ref.name / self.task
+
     def get_config_path(self, base_path: str = ".experiments") -> Path:
         """Get the path to experiment.yaml file."""
         return self.get_experiment_dir(base_path) / "experiment.yaml"
@@ -428,6 +464,22 @@ class ExperimentConfig(BaseModel):
         """Get the path to README.md file."""
         return self.get_experiment_dir(base_path) / "README.md"
 
+    def get_evaluator_filename(self) -> str:
+        """
+        Get the evaluator filename with task prefix.
+
+        Returns: {agent_name}-{task}.yaml (e.g., siggy-risk-assessment.yaml)
+        """
+        return f"{self.agent_schema_ref.name}-{self.task}.yaml"
+
+    def get_s3_export_path(self, bucket: str, version: str = "v0") -> str:
+        """
+        Get the S3 path for exporting this experiment.
+
+        Returns: s3://{bucket}/{version}/datasets/calibration/experiments/{agent}/{task}/
+        """
+        return f"s3://{bucket}/{version}/datasets/calibration/experiments/{self.agent_schema_ref.name}/{self.task}"
+
     def to_yaml(self) -> str:
         """Export configuration as YAML string."""
         import yaml
@@ -483,6 +535,7 @@ class ExperimentConfig(BaseModel):
 ## Configuration
 
 **Status**: `{self.status.value}`
+**Task**: `{self.task}`
 **Tags**: {', '.join(f'`{tag}`' for tag in self.tags) if self.tags else 'None'}
 
 ## Agent Schema
@@ -494,6 +547,7 @@ class ExperimentConfig(BaseModel):
 ## Evaluator Schema
 
 - **Name**: `{self.evaluator_schema_ref.name}`
+- **File**: `{self.get_evaluator_filename()}`
 - **Type**: `{self.evaluator_schema_ref.type}`
 
 ## Datasets

rem/models/core/rem_query.py

@@ -112,7 +112,7 @@ class SearchParameters(BaseModel):
     table_name: str = Field(..., description="Table to search (resources, moments, etc.)")
     limit: int = Field(default=10, gt=0, description="Maximum results")
     min_similarity: float = Field(
-        default=0.7, ge=0.0, le=1.0, description="Minimum similarity score"
+        default=0.3, ge=0.0, le=1.0, description="Minimum similarity score (0.3 recommended for general queries)"
     )
 
 
@@ -198,7 +198,10 @@ class RemQuery(BaseModel):
         | SQLParameters
         | TraverseParameters
     ) = Field(..., description="Query parameters")
-    user_id: str = Field(..., description="User identifier for isolation")
+    user_id: Optional[str] = Field(
+        default=None,
+        description="User identifier (UUID5 hash of email). None = anonymous (shared/public data only)"
+    )
 
 
 class TraverseStage(BaseModel):

rem/models/entities/__init__.py

@@ -5,6 +5,9 @@ Core entity types for the REM system:
 - Resources: Base content units (documents, conversations, artifacts)
 - ImageResources: Image-specific resources with CLIP embeddings
 - Messages: Communication content
+- Sessions: Conversation sessions (normal or evaluation mode)
+- SharedSessions: Session sharing between users for collaboration
+- Feedback: User feedback on messages/sessions with trace integration
 - Users: User entities
 - Files: File metadata and tracking
 - Moments: Temporal narratives (meetings, coding sessions, conversations)
@@ -19,6 +22,8 @@ All entities inherit from CoreModel and support:
 - Natural language labels for conversational queries
 """
 
+from .domain_resource import DomainResource
+from .feedback import Feedback, FeedbackCategory
 from .file import File
 from .image_resource import ImageResource
 from .message import Message
@@ -27,12 +32,28 @@ from .ontology import Ontology
 from .ontology_config import OntologyConfig
 from .resource import Resource
 from .schema import Schema
+from .session import Session, SessionMode
+from .shared_session import (
+    SharedSession,
+    SharedSessionCreate,
+    SharedWithMeResponse,
+    SharedWithMeSummary,
+)
 from .user import User, UserTier
 
 __all__ = [
     "Resource",
+    "DomainResource",
     "ImageResource",
     "Message",
+    "Session",
+    "SessionMode",
+    "SharedSession",
+    "SharedSessionCreate",
+    "SharedWithMeResponse",
+    "SharedWithMeSummary",
+    "Feedback",
+    "FeedbackCategory",
     "User",
     "UserTier",
     "File",