remdb 0.3.114__py3-none-any.whl → 0.3.172__py3-none-any.whl

rem/services/postgres/diff_service.py

@@ -22,6 +22,7 @@ from alembic.runtime.migration import MigrationContext
 from alembic.script import ScriptDirectory
 from loguru import logger
 from sqlalchemy import create_engine, text
+from sqlalchemy.dialects import postgresql
 
 from ...settings import settings
 from .pydantic_to_sqlalchemy import get_target_metadata
@@ -49,6 +50,7 @@ class SchemaDiff:
     summary: list[str] = field(default_factory=list)
     sql: str = ""
     upgrade_ops: Optional[ops.UpgradeOps] = None
+    filtered_count: int = 0  # Number of operations filtered out by strategy
 
     @property
     def change_count(self) -> int:
@@ -61,17 +63,24 @@ class DiffService:
     Service for comparing Pydantic models against database schema.
 
     Uses Alembic's autogenerate machinery without creating revision files.
+
+    Strategies:
+        additive: Only ADD operations (columns, tables, indexes). No drops. Safe for production.
+        full: All operations including DROPs. Use with caution.
+        safe: Additive + safe column type changes (widenings like VARCHAR(50) -> VARCHAR(256)).
     """
 
-    def __init__(self, models_dir: Optional[Path] = None):
+    def __init__(self, models_dir: Optional[Path] = None, strategy: str = "additive"):
         """
         Initialize diff service.
 
         Args:
             models_dir: Directory containing Pydantic models.
                        If None, uses default rem/models/entities location.
+            strategy: Migration strategy - 'additive' (default), 'full', or 'safe'
         """
         self.models_dir = models_dir
+        self.strategy = strategy
         self._metadata = None
 
     def get_connection_url(self) -> str:
@@ -130,6 +139,7 @@ class DiffService:
         metadata = self.get_target_metadata()
 
         summary = []
+        filtered_count = 0
 
         with engine.connect() as conn:
             # Create migration context for comparison
@@ -148,9 +158,13 @@ class DiffService:
             migration_script = produce_migrations(context, metadata)
             upgrade_ops = migration_script.upgrade_ops
 
-            # Process detected operations
+            # Filter operations based on strategy
             if upgrade_ops and upgrade_ops.ops:
-                for op in upgrade_ops.ops:
+                filtered_ops, filtered_count = self._filter_operations(upgrade_ops.ops)
+                upgrade_ops.ops = filtered_ops
+
+                # Process filtered operations
+                for op in filtered_ops:
                     summary.extend(self._describe_operation(op))
 
         has_changes = len(summary) > 0
@@ -165,8 +179,100 @@ class DiffService:
             summary=summary,
             sql=sql,
             upgrade_ops=upgrade_ops,
+            filtered_count=filtered_count,
         )
 
+    def _filter_operations(self, operations: list) -> tuple[list, int]:
+        """
+        Filter operations based on migration strategy.
+
+        Args:
+            operations: List of Alembic operations
+
+        Returns:
+            Tuple of (filtered_operations, count_of_filtered_out)
+        """
+        if self.strategy == "full":
+            # Full strategy: include everything
+            return operations, 0
+
+        filtered = []
+        filtered_count = 0
+
+        for op in operations:
+            if isinstance(op, ops.ModifyTableOps):
+                # Filter sub-operations within table
+                sub_filtered, sub_count = self._filter_operations(op.ops)
+                filtered_count += sub_count
+                if sub_filtered:
+                    op.ops = sub_filtered
+                    filtered.append(op)
+            elif self._is_allowed_operation(op):
+                filtered.append(op)
+            else:
+                filtered_count += 1
+
+        return filtered, filtered_count
+
+    def _is_allowed_operation(self, op: ops.MigrateOperation) -> bool:
+        """
+        Check if an operation is allowed by the current strategy.
+
+        Args:
+            op: Alembic operation
+
+        Returns:
+            True if operation is allowed, False if it should be filtered out
+        """
+        # Additive operations (allowed in all strategies)
+        if isinstance(op, (ops.CreateTableOp, ops.AddColumnOp, ops.CreateIndexOp, ops.CreateForeignKeyOp)):
+            return True
+
+        # Destructive operations (only allowed in 'full' strategy)
+        if isinstance(op, (ops.DropTableOp, ops.DropColumnOp, ops.DropIndexOp, ops.DropConstraintOp)):
+            return self.strategy == "full"
+
+        # Alter operations
+        if isinstance(op, ops.AlterColumnOp):
+            if self.strategy == "full":
+                return True
+            if self.strategy == "safe":
+                # Allow safe type changes (widenings)
+                return self._is_safe_type_change(op)
+            # additive: no alter operations
+            return False
+
+        # Unknown operations: allow in full, deny otherwise
+        return self.strategy == "full"
+
+    def _is_safe_type_change(self, op: ops.AlterColumnOp) -> bool:
+        """
+        Check if a column type change is safe (widening, not narrowing).
+
+        Safe changes:
+        - VARCHAR(n) -> VARCHAR(m) where m > n
+        - INTEGER -> BIGINT
+        - Adding nullable (NOT NULL -> NULL)
+
+        Args:
+            op: AlterColumnOp to check
+
+        Returns:
+            True if the change is safe
+        """
+        # Allowing nullable is always safe
+        if op.modify_nullable is True:
+            return True
+
+        # Type changes: only allow VARCHAR widenings for now
+        if op.modify_type is not None:
+            new_type = str(op.modify_type).upper()
+            # VARCHAR widenings are generally safe
+            if "VARCHAR" in new_type:
+                return True  # Assume widening; could add length comparison
+
+        return False
+
     def _describe_operation(self, op: ops.MigrateOperation, prefix: str = "") -> list[str]:
         """Convert Alembic operation to human-readable description."""
         descriptions = []
@@ -367,6 +473,18 @@ class DiffService:
 
         return "\n".join(lines) + "\n"
 
+    def _compile_type(self, col_type) -> str:
+        """Compile SQLAlchemy type to PostgreSQL DDL string.
+
+        SQLAlchemy types like ARRAY(Text) need dialect-specific compilation
+        to render correctly (e.g., "TEXT[]" instead of just "ARRAY").
+        """
+        try:
+            return col_type.compile(dialect=postgresql.dialect())
+        except Exception:
+            # Fallback to string representation if compilation fails
+            return str(col_type)
+
     def _op_to_sql(self, op: ops.MigrateOperation) -> list[str]:
         """Convert operation to SQL statements."""
         lines = []
@@ -376,7 +494,8 @@ class DiffService:
             for col in op.columns:
                 if hasattr(col, 'name') and hasattr(col, 'type'):
                     nullable = "" if getattr(col, 'nullable', True) else " NOT NULL"
-                    cols.append(f"    {col.name} {col.type}{nullable}")
+                    type_str = self._compile_type(col.type)
+                    cols.append(f"    {col.name} {type_str}{nullable}")
             col_str = ",\n".join(cols)
             lines.append(f"CREATE TABLE IF NOT EXISTS {op.table_name} (\n{col_str}\n);")
 
@@ -386,14 +505,16 @@ class DiffService:
         elif isinstance(op, ops.AddColumnOp):
             col = op.column
             nullable = "" if getattr(col, 'nullable', True) else " NOT NULL"
-            lines.append(f"ALTER TABLE {op.table_name} ADD COLUMN IF NOT EXISTS {col.name} {col.type}{nullable};")
+            type_str = self._compile_type(col.type)
+            lines.append(f"ALTER TABLE {op.table_name} ADD COLUMN IF NOT EXISTS {col.name} {type_str}{nullable};")
 
         elif isinstance(op, ops.DropColumnOp):
             lines.append(f"ALTER TABLE {op.table_name} DROP COLUMN IF EXISTS {op.column_name};")
 
         elif isinstance(op, ops.AlterColumnOp):
             if op.modify_type is not None:
-                lines.append(f"ALTER TABLE {op.table_name} ALTER COLUMN {op.column_name} TYPE {op.modify_type};")
+                type_str = self._compile_type(op.modify_type)
+                lines.append(f"ALTER TABLE {op.table_name} ALTER COLUMN {op.column_name} TYPE {type_str};")
             if op.modify_nullable is not None:
                 if op.modify_nullable:
                     lines.append(f"ALTER TABLE {op.table_name} ALTER COLUMN {op.column_name} DROP NOT NULL;")

rem/services/postgres/pydantic_to_sqlalchemy.py

@@ -494,12 +494,13 @@ def _build_embeddings_table(base_table_name: str, metadata: MetaData) -> Table:
     ]
 
     # Create table with unique constraint
-    # Note: constraint name matches PostgreSQL's auto-generated naming convention
+    # Truncate constraint name to fit PostgreSQL's 63-char identifier limit
+    constraint_name = f"uq_{base_table_name[:30]}_emb_entity_field_prov"
     table = Table(
         embeddings_table_name,
         metadata,
         *columns,
-        UniqueConstraint("entity_id", "field_name", "provider", name=f"{embeddings_table_name}_entity_id_field_name_provider_key"),
+        UniqueConstraint("entity_id", "field_name", "provider", name=constraint_name),
     )
 
     # Add indexes (matching register_type output)
@@ -509,22 +510,53 @@ def _build_embeddings_table(base_table_name: str, metadata: MetaData) -> Table:
     return table
 
 
-def get_target_metadata() -> MetaData:
+def _import_model_modules() -> list[str]:
     """
-    Get SQLAlchemy metadata for Alembic autogenerate.
+    Import modules specified in MODELS__IMPORT_MODULES setting.
 
-    This is the main entry point used by alembic/env.py.
+    This ensures downstream models decorated with @rem.register_model
+    are registered before schema generation.
 
     Returns:
-        SQLAlchemy MetaData object representing current Pydantic models
+        List of successfully imported module names
     """
-    import rem
+    import importlib
+    from ...settings import settings
+
+    imported = []
+    for module_name in settings.models.module_list:
+        try:
+            importlib.import_module(module_name)
+            imported.append(module_name)
+            logger.debug(f"Imported model module: {module_name}")
+        except ImportError as e:
+            logger.warning(f"Failed to import model module '{module_name}': {e}")
+    return imported
 
-    package_root = Path(rem.__file__).parent.parent.parent
-    models_dir = package_root / "src" / "rem" / "models" / "entities"
 
-    if not models_dir.exists():
-        logger.error(f"Models directory not found: {models_dir}")
-        return MetaData()
+def get_target_metadata() -> MetaData:
+    """
+    Get SQLAlchemy metadata for Alembic autogenerate.
 
-    return build_sqlalchemy_metadata_from_pydantic(models_dir)
+    This is the main entry point used by alembic/env.py and rem db diff.
+
+    Uses the model registry as the source of truth, which includes:
+    - Core REM models (Resource, Message, User, etc.)
+    - User-registered models via @rem.register_model decorator
+
+    Before building metadata, imports model modules from settings to ensure
+    downstream models are registered. This supports:
+    - Auto-detection of ./models directory (convention)
+    - MODELS__IMPORT_MODULES env var (explicit configuration)
+
+    Returns:
+        SQLAlchemy MetaData object representing all registered Pydantic models
+    """
+    # Import model modules first (auto-detects ./models or uses MODELS__IMPORT_MODULES)
+    imported = _import_model_modules()
+    if imported:
+        logger.info(f"Imported model modules: {imported}")
+
+    # build_sqlalchemy_metadata_from_pydantic uses the registry internally,
+    # so no directory path is needed (the parameter is kept for backwards compat)
+    return build_sqlalchemy_metadata_from_pydantic()

rem/services/postgres/repository.py

@@ -141,13 +141,13 @@ class Repository(Generic[T]):
         # Return single item or list to match input type
         return records_list[0] if is_single else records_list
 
-    async def get_by_id(self, record_id: str, tenant_id: str) -> T | None:
+    async def get_by_id(self, record_id: str, tenant_id: str | None = None) -> T | None:
         """
         Get a single record by ID.
 
         Args:
             record_id: Record identifier
-            tenant_id: Tenant identifier for multi-tenancy isolation
+            tenant_id: Optional tenant identifier (deprecated, not used for filtering)
 
         Returns:
             Model instance or None if not found
@@ -164,13 +164,14 @@ class Repository(Generic[T]):
         if not self.db.pool:
             raise RuntimeError("Failed to establish database connection")
 
+        # Note: tenant_id filtering removed - use user_id for access control instead
         query = f"""
             SELECT * FROM {self.table_name}
-            WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
+            WHERE id = $1 AND deleted_at IS NULL
         """
 
         async with self.db.pool.acquire() as conn:
-            row = await conn.fetchrow(query, record_id, tenant_id)
+            row = await conn.fetchrow(query, record_id)
 
         if not row:
             return None

rem/services/postgres/schema_generator.py

@@ -12,6 +12,7 @@ Output includes:
 - KV_STORE triggers
 - Indexes (foreground and background)
 - Migrations
+- Schema table entries (for agent-like table access)
 
 Usage:
     from rem.services.postgres.schema_generator import SchemaGenerator
@@ -30,14 +31,192 @@ Usage:
 
 import importlib.util
 import inspect
+import json
+import uuid
 from pathlib import Path
-from typing import Type
+from typing import Any, Type
 
 from loguru import logger
 from pydantic import BaseModel
 
 from ...settings import settings
-from .register_type import register_type
+from ...utils.sql_paths import get_package_sql_dir
+from .register_type import register_type, should_embed_field
+
+# Namespace UUID for generating deterministic UUIDs from model names
+# Using UUID5 with this namespace ensures same model always gets same UUID
+REM_SCHEMA_NAMESPACE = uuid.UUID("6ba7b810-9dad-11d1-80b4-00c04fd430c8")  # DNS namespace
+
+
+def generate_model_uuid(fully_qualified_name: str) -> uuid.UUID:
+    """
+    Generate deterministic UUID from fully qualified model name.
+
+    Uses UUID5 (SHA-1 hash) with REM namespace for reproducibility.
+    Same fully qualified name always produces same UUID.
+
+    Args:
+        fully_qualified_name: Full module path, e.g., "rem.models.entities.Resource"
+
+    Returns:
+        Deterministic UUID for this model
+    """
+    return uuid.uuid5(REM_SCHEMA_NAMESPACE, fully_qualified_name)
+
+
+def extract_model_schema_metadata(
+    model: Type[BaseModel],
+    table_name: str,
+    entity_key_field: str,
+    include_search_tool: bool = True,
+) -> dict[str, Any]:
+    """
+    Extract schema metadata from a Pydantic model for schemas table.
+
+    Args:
+        model: Pydantic model class
+        table_name: Database table name
+        entity_key_field: Field used as entity key in kv_store
+        include_search_tool: If True, add search_rem tool for querying this table
+
+    Returns:
+        Dict with schema metadata ready for schemas table insert
+    """
+    # Get fully qualified name
+    fqn = f"{model.__module__}.{model.__name__}"
+
+    # Generate deterministic UUID
+    schema_id = generate_model_uuid(fqn)
+
+    # Get JSON schema from Pydantic
+    json_schema = model.model_json_schema()
+
+    # Find embedding fields
+    embedding_fields = []
+    for field_name, field_info in model.model_fields.items():
+        if should_embed_field(field_name, field_info):
+            embedding_fields.append(field_name)
+
+    # Build description with search capability note
+    base_description = model.__doc__ or f"Schema for {model.__name__}"
+    search_note = (
+        f"\n\nThis agent can search the `{table_name}` table using the `search_rem` tool. "
+        f"Use REM query syntax: LOOKUP for exact match, FUZZY for typo-tolerant search, "
+        f"SEARCH for semantic similarity, or SQL for complex queries."
+    ) if include_search_tool else ""
+
+    # Build spec with table metadata and tools
+    # Note: default_search_table is used by create_agent to append a description
+    # suffix to the search_rem tool when loading it dynamically
+    has_embeddings = bool(embedding_fields)
+
+    spec = {
+        "type": "object",
+        "description": base_description + search_note,
+        "properties": json_schema.get("properties", {}),
+        "required": json_schema.get("required", []),
+        "json_schema_extra": {
+            "table_name": table_name,
+            "entity_key_field": entity_key_field,
+            "embedding_fields": embedding_fields,
+            "fully_qualified_name": fqn,
+            "tools": ["search_rem"] if include_search_tool else [],
+            "default_search_table": table_name,
+            "has_embeddings": has_embeddings,
+        },
+    }
+
+    # Build content (documentation)
+    content = f"""# {model.__name__}
+
+{base_description}
+
+## Overview
+
+The `{model.__name__}` entity is stored in the `{table_name}` table. Each record is uniquely
+identified by its `{entity_key_field}` field for lookups and graph traversal.
+
+## Search Capabilities
+
+This schema includes the `search_rem` tool which supports:
+- **LOOKUP**: O(1) exact match by {entity_key_field} (e.g., `LOOKUP "entity-name"`)
+- **FUZZY**: Typo-tolerant search (e.g., `FUZZY "partial" THRESHOLD 0.3`)
+- **SEARCH**: Semantic vector search on {', '.join(embedding_fields) if embedding_fields else 'content'} (e.g., `SEARCH "concept" FROM {table_name} LIMIT 10`)
+- **SQL**: Complex queries (e.g., `SELECT * FROM {table_name} WHERE ...`)
+
+## Table Info
+
+| Property | Value |
+|----------|-------|
+| Table | `{table_name}` |
+| Entity Key | `{entity_key_field}` |
+| Embedding Fields | {', '.join(f'`{f}`' for f in embedding_fields) if embedding_fields else 'None'} |
+| Tools | {', '.join(['`search_rem`'] if include_search_tool else ['None'])} |
+
+## Fields
+
+"""
+    for field_name, field_info in model.model_fields.items():
+        field_type = str(field_info.annotation) if field_info.annotation else "Any"
+        field_desc = field_info.description or ""
+        required = "Required" if field_info.is_required() else "Optional"
+        content += f"### `{field_name}`\n"
+        content += f"- **Type**: `{field_type}`\n"
+        content += f"- **{required}**\n"
+        if field_desc:
+            content += f"- {field_desc}\n"
+        content += "\n"
+
+    return {
+        "id": str(schema_id),
+        "name": model.__name__,
+        "table_name": table_name,
+        "entity_key_field": entity_key_field,
+        "embedding_fields": embedding_fields,
+        "fqn": fqn,
+        "spec": spec,
+        "content": content,
+        "category": "entity",
+    }
+
+
+def generate_schema_upsert_sql(schema_metadata: dict[str, Any]) -> str:
+    """
+    Generate SQL UPSERT statement for schemas table.
+
+    Uses ON CONFLICT DO UPDATE for idempotency.
+
+    Args:
+        schema_metadata: Dict from extract_model_schema_metadata()
+
+    Returns:
+        SQL INSERT ... ON CONFLICT statement
+    """
+    # Escape single quotes in content and spec
+    content_escaped = schema_metadata["content"].replace("'", "''")
+    spec_json = json.dumps(schema_metadata["spec"]).replace("'", "''")
+
+    sql = f"""
+-- Schema entry for {schema_metadata['name']} ({schema_metadata['table_name']})
+INSERT INTO schemas (id, tenant_id, name, content, spec, category, metadata)
+VALUES (
+    '{schema_metadata['id']}'::uuid,
+    'system',
+    '{schema_metadata['name']}',
+    '{content_escaped}',
+    '{spec_json}'::jsonb,
+    'entity',
+    '{{"table_name": "{schema_metadata['table_name']}", "entity_key_field": "{schema_metadata['entity_key_field']}", "embedding_fields": {json.dumps(schema_metadata['embedding_fields'])}, "fqn": "{schema_metadata['fqn']}"}}'::jsonb
+)
+ON CONFLICT (id) DO UPDATE SET
+    name = EXCLUDED.name,
+    content = EXCLUDED.content,
+    spec = EXCLUDED.spec,
+    category = EXCLUDED.category,
+    metadata = EXCLUDED.metadata,
+    updated_at = CURRENT_TIMESTAMP;
+"""
+    return sql.strip()
 
 
 class SchemaGenerator:
@@ -56,9 +235,9 @@ class SchemaGenerator:
         Initialize schema generator.
 
         Args:
-            output_dir: Optional directory for output files (defaults to settings.sql_dir)
+            output_dir: Optional directory for output files (defaults to package sql dir)
         """
-        self.output_dir = output_dir or Path(settings.sql_dir)
+        self.output_dir = output_dir or get_package_sql_dir()
         self.schemas: dict[str, dict] = {}
 
     def discover_models(self, directory: str | Path) -> dict[str, Type[BaseModel]]:
@@ -234,6 +413,14 @@ class SchemaGenerator:
             create_kv_trigger=True,
         )
 
+        # Extract schema metadata for schemas table entry
+        schema_metadata = extract_model_schema_metadata(
+            model=model,
+            table_name=table_name,
+            entity_key_field=entity_key_field,
+        )
+        schema["schema_metadata"] = schema_metadata
+
         self.schemas[table_name] = schema
         return schema
 
@@ -343,6 +530,7 @@ class SchemaGenerator:
             "-- 2. Embeddings tables (embeddings_<table>)",
             "-- 3. KV_STORE triggers for cache maintenance",
             "-- 4. Indexes (foreground only, background indexes separate)",
+            "-- 5. Schema table entries (for agent-like table access)",
             "",
             "-- ============================================================================",
             "-- PREREQUISITES CHECK",
@@ -388,6 +576,19 @@ class SchemaGenerator:
                 sql_parts.append(schema["sql"]["kv_trigger"])
                 sql_parts.append("")
 
+        # Add schema table entries (every entity table is also an "agent")
+        sql_parts.append("-- ============================================================================")
+        sql_parts.append("-- SCHEMA TABLE ENTRIES")
+        sql_parts.append("-- Every entity table gets a schemas entry for agent-like access")
+        sql_parts.append("-- ============================================================================")
+        sql_parts.append("")
+
+        for table_name, schema in self.schemas.items():
+            if "schema_metadata" in schema:
+                schema_upsert = generate_schema_upsert_sql(schema["schema_metadata"])
+                sql_parts.append(schema_upsert)
+                sql_parts.append("")
+
         # Add migration record
         sql_parts.append("-- ============================================================================")
         sql_parts.append("-- RECORD MIGRATION")