remdb 0.3.103__py3-none-any.whl → 0.3.118__py3-none-any.whl

rem/cli/commands/schema.py

@@ -8,6 +8,7 @@ Usage:
 """
 
 import asyncio
+import importlib
 from pathlib import Path
 
 import click
@@ -15,16 +16,35 @@ from loguru import logger
 
 from ...settings import settings
 from ...services.postgres.schema_generator import SchemaGenerator
+from ...utils.sql_paths import get_package_sql_dir, get_package_migrations_dir
+
+
+def _import_model_modules() -> list[str]:
+    """
+    Import modules specified in MODELS__IMPORT_MODULES setting.
+
+    This ensures downstream models decorated with @rem.register_model
+    are registered before schema generation.
+
+    Returns:
+        List of successfully imported module names
+    """
+    imported = []
+    for module_name in settings.models.module_list:
+        try:
+            importlib.import_module(module_name)
+            imported.append(module_name)
+            logger.debug(f"Imported model module: {module_name}")
+        except ImportError as e:
+            logger.warning(f"Failed to import model module '{module_name}': {e}")
+            click.echo(
+                click.style(f"  ⚠ Could not import '{module_name}': {e}", fg="yellow"),
+                err=True,
+            )
+    return imported
 
 
 @click.command()
-@click.option(
-    "--models",
-    "-m",
-    required=True,
-    type=click.Path(exists=True, path_type=Path),
-    help="Directory containing Pydantic models",
-)
 @click.option(
     "--output",
     "-o",
@@ -36,13 +56,13 @@ from ...services.postgres.schema_generator import SchemaGenerator
     "--output-dir",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Base output directory (default: {settings.sql_dir}/migrations)",
+    help="Base output directory (default: package sql/migrations)",
 )
-def generate(models: Path, output: Path, output_dir: Path | None):
+def generate(output: Path, output_dir: Path | None):
     """
-    Generate database schema from Pydantic models.
+    Generate database schema from registered Pydantic models.
 
-    Scans the specified directory for Pydantic models and generates:
+    Uses the model registry (core models + user-registered models) to generate:
     - CREATE TABLE statements
     - Embeddings tables (embeddings_<table>)
     - KV_STORE triggers for cache maintenance
@@ -51,24 +71,53 @@ def generate(models: Path, output: Path, output_dir: Path | None):
     Output is written to src/rem/sql/migrations/002_install_models.sql by default.
 
     Example:
-        rem db schema generate --models src/rem/models/entities
+        rem db schema generate
+
+    To register custom models in downstream apps:
+
+    1. Create models with @rem.register_model decorator:
+
+        # models/__init__.py
+        import rem
+        from rem.models.core import CoreModel
+
+        @rem.register_model
+        class MyEntity(CoreModel):
+            name: str
+
+    2. Set MODELS__IMPORT_MODULES in your .env:
+
+        MODELS__IMPORT_MODULES=models
+
+    3. Run schema generation:
+
+        rem db schema generate
 
     This creates:
     - src/rem/sql/migrations/002_install_models.sql - Entity tables and triggers
     - src/rem/sql/background_indexes.sql - HNSW indexes (apply after data load)
 
-    After generation, apply with:
-        rem db migrate
+    After generation, verify with:
+        rem db diff
     """
-    click.echo(f"Discovering models in {models}")
+    from ...registry import get_model_registry
+
+    # Import downstream model modules to trigger @rem.register_model decorators
+    imported_modules = _import_model_modules()
+    if imported_modules:
+        click.echo(f"Imported model modules: {', '.join(imported_modules)}")
+
+    registry = get_model_registry()
+    models = registry.get_models(include_core=True)
+    click.echo(f"Generating schema from {len(models)} registered models")
 
-    # Default to migrations directory
-    actual_output_dir = output_dir or Path(settings.sql_dir) / "migrations"
+    # Default to package migrations directory
+    actual_output_dir = output_dir or get_package_migrations_dir()
     generator = SchemaGenerator(output_dir=actual_output_dir)
 
-    # Generate schema
+    # Generate schema from registry
     try:
-        schema_sql = asyncio.run(generator.generate_from_directory(models, output_file=output.name))
+        schema_sql = asyncio.run(generator.generate_from_registry(output_file=output.name))
 
         click.echo(f"✓ Schema generated: {len(generator.schemas)} tables")
         click.echo(f"✓ Written to: {actual_output_dir / output.name}")
@@ -76,7 +125,7 @@ def generate(models: Path, output: Path, output_dir: Path | None):
         # Generate background indexes in parent sql dir
         background_indexes = generator.generate_background_indexes()
         if background_indexes:
-            bg_file = Path(settings.sql_dir) / "background_indexes.sql"
+            bg_file = get_package_sql_dir() / "background_indexes.sql"
             bg_file.write_text(background_indexes)
             click.echo(f"✓ Background indexes: {bg_file}")
 
@@ -94,48 +143,46 @@ def generate(models: Path, output: Path, output_dir: Path | None):
 
 
 @click.command()
-@click.option(
-    "--models",
-    "-m",
-    required=True,
-    type=click.Path(exists=True, path_type=Path),
-    help="Directory containing Pydantic models",
-)
-def validate(models: Path):
+def validate():
     """
-    Validate Pydantic models for schema generation.
+    Validate registered Pydantic models for schema generation.
 
     Checks:
-    - Models can be loaded
+    - Models can be loaded from registry
     - Models have suitable entity_key fields
     - Fields with embeddings are properly configured
+
+    Set MODELS__IMPORT_MODULES to include custom models from downstream apps.
     """
-    click.echo(f"Validating models in {models}")
+    from ...registry import get_model_registry
 
-    generator = SchemaGenerator()
-    discovered = generator.discover_models(models)
+    # Import downstream model modules to trigger @rem.register_model decorators
+    imported_modules = _import_model_modules()
+    if imported_modules:
+        click.echo(f"Imported model modules: {', '.join(imported_modules)}")
 
-    if not discovered:
-        click.echo("✗ No models found", err=True)
-        raise click.Abort()
+    registry = get_model_registry()
+    models = registry.get_models(include_core=True)
+
+    click.echo(f"Validating {len(models)} registered models")
 
-    click.echo(f"✓ Discovered {len(discovered)} models")
+    if not models:
+        click.echo("✗ No models found in registry", err=True)
+        raise click.Abort()
 
+    generator = SchemaGenerator()
     errors: list[str] = []
     warnings: list[str] = []
 
-    for model_name, model in discovered.items():
-        table_name = generator.infer_table_name(model)
-        entity_key = generator.infer_entity_key_field(model)
+    for model_name, ext in models.items():
+        model = ext.model
+        table_name = ext.table_name or generator.infer_table_name(model)
+        entity_key = ext.entity_key_field or generator.infer_entity_key_field(model)
 
         # Check for entity_key
         if entity_key == "id":
             warnings.append(f"{model_name}: No natural key field, using 'id'")
 
-        # Check for embeddable fields
-        # TODO: Implement should_embed_field check
-        embeddable: list[str] = []  # Placeholder - needs implementation
-
         click.echo(f"  {model_name} -> {table_name} (key: {entity_key})")
 
     if warnings:
@@ -158,7 +205,7 @@ def validate(models: Path):
     "-o",
     type=click.Path(path_type=Path),
     default=None,
-    help=f"Output file for background indexes (default: {settings.sql_dir}/background_indexes.sql)",
+    help="Output file for background indexes (default: package sql/background_indexes.sql)",
 )
 def indexes(output: Path):
     """

rem/cli/main.py

@@ -22,17 +22,30 @@ except Exception:
     __version__ = "unknown"
 
 
+def _configure_logger(level: str):
+    """Configure loguru with custom level icons."""
+    logger.remove()
+
+    # Configure level icons - only warnings and errors get visual indicators
+    logger.level("DEBUG", icon=" ")
+    logger.level("INFO", icon=" ")
+    logger.level("WARNING", icon="🟠")
+    logger.level("ERROR", icon="🔴")
+    logger.level("CRITICAL", icon="🔴")
+
+    logger.add(
+        sys.stderr,
+        level=level,
+        format="<green>{time:HH:mm:ss}</green> | {level.icon} <level>{level: <8}</level> | <level>{message}</level>",
+    )
+
+
 @click.group()
 @click.option("--verbose", "-v", is_flag=True, help="Enable verbose logging")
 @click.version_option(version=__version__, prog_name="rem")
 def cli(verbose: bool):
     """REM - Resources Entities Moments system CLI."""
-    if verbose:
-        logger.remove()
-        logger.add(sys.stderr, level="DEBUG")
-    else:
-        logger.remove()
-        logger.add(sys.stderr, level="INFO")
+    _configure_logger("DEBUG" if verbose else "INFO")
 
 
 @cli.group()
@@ -65,6 +78,12 @@ def dreaming():
     pass
 
 
+@cli.group()
+def cluster():
+    """Kubernetes cluster deployment and management."""
+    pass
+
+
 # Register commands
 from .commands.schema import register_commands as register_schema_commands
 from .commands.db import register_commands as register_db_commands
@@ -76,11 +95,13 @@ from .commands.configure import register_command as register_configure_command
 from .commands.serve import register_command as register_serve_command
 from .commands.mcp import register_command as register_mcp_command
 from .commands.scaffold import scaffold as scaffold_command
+from .commands.cluster import register_commands as register_cluster_commands
 
 register_schema_commands(schema)
 register_db_commands(db)
 register_process_commands(process)
 register_dreaming_commands(dreaming)
+register_cluster_commands(cluster)
 register_ask_command(cli)
 register_configure_command(cli)
 register_serve_command(cli)

rem/models/core/rem_query.py

@@ -112,7 +112,7 @@ class SearchParameters(BaseModel):
     table_name: str = Field(..., description="Table to search (resources, moments, etc.)")
     limit: int = Field(default=10, gt=0, description="Maximum results")
     min_similarity: float = Field(
-        default=0.7, ge=0.0, le=1.0, description="Minimum similarity score"
+        default=0.3, ge=0.0, le=1.0, description="Minimum similarity score (0.3 recommended for general queries)"
     )
 
 
@@ -198,7 +198,10 @@ class RemQuery(BaseModel):
         | SQLParameters
         | TraverseParameters
     ) = Field(..., description="Query parameters")
-    user_id: str = Field(..., description="User identifier for isolation")
+    user_id: Optional[str] = Field(
+        default=None,
+        description="User identifier (UUID5 hash of email). None = anonymous (shared/public data only)"
+    )
 
 
 class TraverseStage(BaseModel):

rem/models/entities/shared_session.py

@@ -111,28 +111,20 @@ To permanently delete, an admin can run:
 
 from datetime import datetime
 from typing import Optional
-from uuid import UUID
 
 from pydantic import BaseModel, Field
 
-from ...utils.date_utils import utc_now
+from ..core import CoreModel
 
 
-class SharedSession(BaseModel):
+class SharedSession(CoreModel):
     """
     Session sharing record between users.
 
     Links a session (identified by session_id from Message records) to a
     recipient user, enabling collaborative access to conversation history.
-
-    This is NOT a CoreModel - it's a lightweight linking table without
-    graph edges, metadata, or embeddings.
     """
 
-    id: Optional[UUID] = Field(
-        default=None,
-        description="Unique identifier (auto-generated)",
-    )
     session_id: str = Field(
         ...,
         description="The session being shared (matches Message.session_id)",
@@ -145,24 +137,6 @@ class SharedSession(BaseModel):
         ...,
         description="User ID of the recipient (who can now view the session)",
     )
-    tenant_id: str = Field(
-        default="default",
-        description="Tenant identifier for multi-tenancy isolation",
-    )
-    created_at: datetime = Field(
-        default_factory=utc_now,
-        description="When the share was created",
-    )
-    updated_at: datetime = Field(
-        default_factory=utc_now,
-        description="Last modification timestamp",
-    )
-    deleted_at: Optional[datetime] = Field(
-        default=None,
-        description="Soft delete timestamp (null = active share)",
-    )
-
-    model_config = {"from_attributes": True}
 
 
 class SharedSessionCreate(BaseModel):

rem/registry.py

@@ -123,6 +123,7 @@ class ModelRegistry:
             return
 
         from .models.entities import (
+            Feedback,
             File,
             ImageResource,
             Message,
@@ -131,19 +132,24 @@ class ModelRegistry:
             OntologyConfig,
             Resource,
             Schema,
+            Session,
+            SharedSession,
             User,
         )
 
         core_models = [
-            Resource,
+            Feedback,
+            File,
             ImageResource,
             Message,
-            User,
-            File,
             Moment,
-            Schema,
             Ontology,
             OntologyConfig,
+            Resource,
+            Schema,
+            Session,
+            SharedSession,
+            User,
         ]
 
         for model in core_models:

rem/services/content/service.py

@@ -370,11 +370,32 @@ class ContentService:
         file_size = len(file_content)
         logger.info(f"Read {file_size} bytes from {file_uri} (source: {source_type})")
 
-        # Step 2: Write to internal storage (user-scoped)
+        # Step 1.5: Early schema detection for YAML/JSON files
+        # Skip File entity creation for schemas (agents/evaluators)
+        file_suffix = Path(file_name).suffix.lower()
+        if file_suffix in ['.yaml', '.yml', '.json']:
+            import yaml
+            import json
+            try:
+                content_text = file_content.decode('utf-8') if isinstance(file_content, bytes) else file_content
+                data = yaml.safe_load(content_text) if file_suffix in ['.yaml', '.yml'] else json.loads(content_text)
+                if isinstance(data, dict):
+                    json_schema_extra = data.get('json_schema_extra', {})
+                    kind = json_schema_extra.get('kind', '')
+                    if kind in ['agent', 'evaluator']:
+                        # Route directly to schema processing, skip File entity
+                        logger.info(f"Detected {kind} schema: {file_name}, routing to _process_schema")
+                        result = self.process_uri(file_uri)
+                        return await self._process_schema(result, file_uri, user_id)
+            except Exception as e:
+                logger.debug(f"Early schema detection failed for {file_name}: {e}")
+                # Fall through to standard file processing
+
+        # Step 2: Write to internal storage (public or user-scoped)
         file_id = str(uuid4())
         storage_uri, internal_key, content_type, _ = await fs_service.write_to_internal_storage(
             content=file_content,
-            tenant_id=user_id,  # Using user_id for storage scoping
+            tenant_id=user_id or "public",  # Storage path: public/ or user_id/
             file_name=file_name,
             file_id=file_id,
         )
@@ -383,7 +404,7 @@ class ContentService:
         # Step 3: Create File entity
         file_entity = File(
             id=file_id,
-            tenant_id=user_id,  # Set tenant_id to user_id (application scoped to user)
+            tenant_id=user_id,  # None = public/shared
             user_id=user_id,
             name=file_name,
             uri=storage_uri,
@@ -538,7 +559,7 @@ class ContentService:
             size_bytes=result["metadata"].get("size"),
             mime_type=result["metadata"].get("content_type"),
             processing_status="completed",
-            tenant_id=user_id or "default",  # Required field
+            tenant_id=user_id,  # None = public/shared
             user_id=user_id,
         )
 
@@ -571,7 +592,7 @@ class ContentService:
                 ordinal=i,
                 content=chunk,
                 category="document",
-                tenant_id=user_id or "default",  # Required field
+                tenant_id=user_id,  # None = public/shared
                 user_id=user_id,
             )
             for i, chunk in enumerate(chunks)
@@ -645,9 +666,10 @@ class ContentService:
         # IMPORTANT: category field distinguishes agents from evaluators
         # - kind=agent → category="agent" (AI agents with tools/resources)
         # - kind=evaluator → category="evaluator" (LLM-as-a-Judge evaluators)
+        # Schemas (agents/evaluators) default to system tenant for shared access
         schema_entity = Schema(
-            tenant_id=user_id or "default",
-            user_id=user_id,
+            tenant_id="system",
+            user_id=None,
             name=name,
             spec=schema_data,
             category=kind,  # Maps kind → category for database filtering
@@ -717,7 +739,7 @@ class ContentService:
             processor = EngramProcessor(postgres)
             result = await processor.process_engram(
                 data=data,
-                tenant_id=user_id or "default",
+                tenant_id=user_id,  # None = public/shared
                 user_id=user_id,
             )
             logger.info(f"✅ Engram processed: {result.get('resource_id')} with {len(result.get('moment_ids', []))} moments")

rem/services/embeddings/api.py

@@ -45,7 +45,7 @@ def generate_embedding(
             return [0.0] * DEFAULT_EMBEDDING_DIMS
 
         try:
-            logger.info(f"Generating OpenAI embedding for text using {model}")
+            logger.debug(f"Generating OpenAI embedding for text using {model}")
 
             response = requests.post(
                 "https://api.openai.com/v1/embeddings",
@@ -60,7 +60,7 @@ def generate_embedding(
 
             data = response.json()
             embedding = data["data"][0]["embedding"]
-            logger.info(f"Successfully generated embedding (dimension: {len(embedding)})")
+            logger.debug(f"Successfully generated embedding (dimension: {len(embedding)})")
             return cast(list[float], embedding)
 
         except Exception as e:
@@ -97,7 +97,7 @@ async def generate_embedding_async(
             return [0.0] * DEFAULT_EMBEDDING_DIMS
 
         try:
-            logger.info(f"Generating OpenAI embedding for text using {model}")
+            logger.debug(f"Generating OpenAI embedding for text using {model}")
 
             async with httpx.AsyncClient() as client:
                 response = await client.post(
@@ -113,7 +113,7 @@ async def generate_embedding_async(
 
                 data = response.json()
                 embedding = data["data"][0]["embedding"]
-                logger.info(
+                logger.debug(
                     f"Successfully generated embedding (dimension: {len(embedding)})"
                 )
                 return cast(list[float], embedding)

rem/services/embeddings/worker.py

@@ -69,7 +69,7 @@ def get_global_embedding_worker(postgres_service: Any = None) -> "EmbeddingWorke
         if postgres_service is None:
             raise RuntimeError("Must provide postgres_service on first call to get_global_embedding_worker")
         _global_worker = EmbeddingWorker(postgres_service=postgres_service)
-        logger.info("Created global EmbeddingWorker singleton")
+        logger.debug("Created global EmbeddingWorker singleton")
 
     return _global_worker
 
@@ -117,7 +117,7 @@ class EmbeddingWorker:
                 "No OpenAI API key provided - embeddings will use zero vectors"
             )
 
-        logger.info(
+        logger.debug(
             f"Initialized EmbeddingWorker: {num_workers} workers, "
             f"batch_size={batch_size}, timeout={batch_timeout}s"
         )
@@ -125,17 +125,17 @@ class EmbeddingWorker:
     async def start(self) -> None:
         """Start worker pool."""
         if self.running:
-            logger.warning("EmbeddingWorker already running")
+            logger.debug("EmbeddingWorker already running")
             return
 
         self.running = True
-        logger.info(f"Starting {self.num_workers} embedding workers")
+        logger.debug(f"Starting {self.num_workers} embedding workers")
 
         for i in range(self.num_workers):
             worker = asyncio.create_task(self._worker_loop(i))
             self.workers.append(worker)
 
-        logger.info("EmbeddingWorker started")
+        logger.debug("EmbeddingWorker started")
 
     async def stop(self) -> None:
         """Stop worker pool gracefully - processes remaining queue before stopping."""
@@ -143,7 +143,7 @@ class EmbeddingWorker:
             return
 
         queue_size = self.task_queue.qsize()
-        logger.info(f"Stopping EmbeddingWorker (processing {queue_size} queued tasks first)")
+        logger.debug(f"Stopping EmbeddingWorker (processing {queue_size} queued tasks first)")
 
         # Wait for queue to drain (with timeout)
         max_wait = 30  # 30 seconds max
@@ -171,7 +171,7 @@ class EmbeddingWorker:
         await asyncio.gather(*self.workers, return_exceptions=True)
 
         self.workers.clear()
-        logger.info("EmbeddingWorker stopped")
+        logger.debug("EmbeddingWorker stopped")
 
     async def queue_task(self, task: EmbeddingTask) -> None:
         """
@@ -195,7 +195,7 @@ class EmbeddingWorker:
         Args:
             worker_id: Unique worker identifier
         """
-        logger.info(f"Worker {worker_id} started")
+        logger.debug(f"Worker {worker_id} started")
 
         while self.running:
             try:
@@ -205,7 +205,7 @@ class EmbeddingWorker:
                 if not batch:
                     continue
 
-                logger.info(f"Worker {worker_id} processing batch of {len(batch)} tasks")
+                logger.debug(f"Worker {worker_id} processing batch of {len(batch)} tasks")
 
                 # Generate embeddings for batch
                 await self._process_batch(batch)
@@ -213,14 +213,14 @@ class EmbeddingWorker:
                 logger.debug(f"Worker {worker_id} completed batch")
 
             except asyncio.CancelledError:
-                logger.info(f"Worker {worker_id} cancelled")
+                logger.debug(f"Worker {worker_id} cancelled")
                 break
             except Exception as e:
                 logger.error(f"Worker {worker_id} error: {e}", exc_info=True)
                 # Continue processing (don't crash worker on error)
                 await asyncio.sleep(1)
 
-        logger.info(f"Worker {worker_id} stopped")
+        logger.debug(f"Worker {worker_id} stopped")
 
     async def _collect_batch(self) -> list[EmbeddingTask]:
         """
@@ -284,10 +284,10 @@ class EmbeddingWorker:
             )
 
             # Upsert to database
-            logger.info(f"Upserting {len(embeddings)} embeddings to database...")
+            logger.debug(f"Upserting {len(embeddings)} embeddings to database...")
             await self._upsert_embeddings(batch, embeddings)
 
-            logger.info(
+            logger.debug(
                 f"Successfully generated and stored {len(embeddings)} embeddings "
                 f"(provider={provider}, model={model})"
             )
@@ -315,7 +315,7 @@ class EmbeddingWorker:
         """
         if provider == "openai" and self.openai_api_key:
             try:
-                logger.info(
+                logger.debug(
                     f"Generating OpenAI embeddings for {len(texts)} texts using {model}"
                 )
 
@@ -336,7 +336,7 @@ class EmbeddingWorker:
                     data = response.json()
                     embeddings = [item["embedding"] for item in data["data"]]
 
-                    logger.info(
+                    logger.debug(
                         f"Successfully generated {len(embeddings)} embeddings from OpenAI"
                     )
                     return embeddings
@@ -409,7 +409,7 @@ class EmbeddingWorker:
                     ),
                 )
 
-                logger.info(
+                logger.debug(
                     f"Upserted embedding: {task.table_name}.{task.entity_id}.{task.field_name}"
                 )