remdb 0.3.180__py3-none-any.whl → 0.3.258__py3-none-any.whl

rem/utils/schema_loader.py

@@ -84,6 +84,7 @@ Schema Caching Status:
 """
 
 import importlib.resources
+import time
 from pathlib import Path
 from typing import Any, cast
 
@@ -104,10 +105,32 @@ SCHEMA_SEARCH_PATHS = [
 # In-memory cache for filesystem schemas (no TTL - immutable)
 _fs_schema_cache: dict[str, dict[str, Any]] = {}
 
-# Future: Database schema cache (with TTL - mutable)
-# Will be used when loading schemas from database (SchemaRepository)
-# _db_schema_cache: dict[tuple[str, str], tuple[dict[str, Any], float]] = {}
-# _db_schema_ttl: int = 300  # 5 minutes in seconds
+# Database schema cache (with TTL - mutable, supports hot-reload)
+# Cache key: (schema_name, user_id or "public") → (schema_dict, timestamp)
+_db_schema_cache: dict[tuple[str, str], tuple[dict[str, Any], float]] = {}
+_db_schema_ttl: int = 300  # 5 minutes in seconds
+
+
+def _get_cached_db_schema(schema_name: str, user_id: str | None) -> dict[str, Any] | None:
+    """Get schema from DB cache if exists and not expired."""
+    cache_key = (schema_name.lower(), user_id or "public")
+    if cache_key in _db_schema_cache:
+        schema, timestamp = _db_schema_cache[cache_key]
+        if time.time() - timestamp < _db_schema_ttl:
+            logger.debug(f"Schema cache hit: {schema_name} (age: {time.time() - timestamp:.0f}s)")
+            return schema
+        else:
+            # Expired, remove from cache
+            del _db_schema_cache[cache_key]
+            logger.debug(f"Schema cache expired: {schema_name}")
+    return None
+
+
+def _cache_db_schema(schema_name: str, user_id: str | None, schema: dict[str, Any]) -> None:
+    """Add schema to DB cache with current timestamp."""
+    cache_key = (schema_name.lower(), user_id or "public")
+    _db_schema_cache[cache_key] = (schema, time.time())
+    logger.debug(f"Schema cached: {schema_name} (TTL: {_db_schema_ttl}s)")
 
 
 def _load_schema_from_database(schema_name: str, user_id: str) -> dict[str, Any] | None:
@@ -147,15 +170,25 @@ def _load_schema_from_database(schema_name: str, user_id: str) -> dict[str, Any]
             try:
                 await db.connect()
 
-                query = """
-                    SELECT spec FROM schemas
-                    WHERE LOWER(name) = LOWER($1)
-                    AND (user_id = $2 OR user_id = 'system' OR user_id IS NULL)
-                    LIMIT 1
-                """
-                logger.debug(f"Executing schema lookup: name={schema_name}, user_id={user_id}")
-
-                row = await db.fetchrow(query, schema_name, user_id)
+                # Query for public schemas (user_id IS NULL) and optionally user-specific
+                if user_id:
+                    query = """
+                        SELECT spec FROM schemas
+                        WHERE LOWER(name) = LOWER($1)
+                        AND (user_id = $2 OR user_id = 'system' OR user_id IS NULL)
+                        LIMIT 1
+                    """
+                    row = await db.fetchrow(query, schema_name, user_id)
+                else:
+                    # No user_id - only search public schemas
+                    query = """
+                        SELECT spec FROM schemas
+                        WHERE LOWER(name) = LOWER($1)
+                        AND (user_id = 'system' OR user_id IS NULL)
+                        LIMIT 1
+                    """
+                    row = await db.fetchrow(query, schema_name)
+                logger.debug(f"Executing schema lookup: name={schema_name}, user_id={user_id or 'public'}")
 
                 if row:
                     spec = row.get("spec")
@@ -193,17 +226,25 @@ def _load_schema_from_database(schema_name: str, user_id: str) -> dict[str, Any]
         try:
             await db.connect()
 
-            # Query schemas table directly by name
-            # Note: Schema name lookup is case-insensitive for user convenience
-            query = """
-                SELECT spec FROM schemas
-                WHERE LOWER(name) = LOWER($1)
-                AND (user_id = $2 OR user_id = 'system')
-                LIMIT 1
-            """
-            logger.debug(f"Executing schema lookup: name={schema_name}, user_id={user_id}")
-
-            row = await db.fetchrow(query, schema_name, user_id)
+            # Query for public schemas (user_id IS NULL) and optionally user-specific
+            if user_id:
+                query = """
+                    SELECT spec FROM schemas
+                    WHERE LOWER(name) = LOWER($1)
+                    AND (user_id = $2 OR user_id = 'system' OR user_id IS NULL)
+                    LIMIT 1
+                """
+                row = await db.fetchrow(query, schema_name, user_id)
+            else:
+                # No user_id - only search public schemas
+                query = """
+                    SELECT spec FROM schemas
+                    WHERE LOWER(name) = LOWER($1)
+                    AND (user_id = 'system' OR user_id IS NULL)
+                    LIMIT 1
+                """
+                row = await db.fetchrow(query, schema_name)
+            logger.debug(f"Executing schema lookup: name={schema_name}, user_id={user_id or 'public'}")
 
             if row:
                 spec = row.get("spec")
@@ -231,74 +272,66 @@ def load_agent_schema(
     enable_db_fallback: bool = True,
 ) -> dict[str, Any]:
     """
-    Load agent schema from YAML file with unified search logic and caching.
+    Load agent schema with database-first priority for hot-reloading support.
 
     Schema names are case-invariant - "Rem", "rem", "REM" all resolve to the same schema.
 
-    Filesystem schemas are cached indefinitely (immutable, versioned with code).
-    Database schemas (future) will be cached with TTL for invalidation.
+    **IMPORTANT**: Database is checked FIRST (before filesystem) to enable hot-reloading
+    of schema updates without redeploying the application. This allows operators to
+    update schemas via `rem process ingest` and have changes take effect immediately.
 
     Handles path resolution automatically:
-    - "rem" → searches schemas/agents/rem.yaml (top-level)
-    - "moment-builder" → searches schemas/agents/core/moment-builder.yaml
-    - "contract-analyzer" → searches schemas/agents/examples/contract-analyzer.yaml
-    - "core/moment-builder" → searches schemas/agents/core/moment-builder.yaml
-    - "/absolute/path.yaml" → loads directly
-    - "relative/path.yaml" → loads relative to cwd
+    - "rem" → searches database, then schemas/agents/rem.yaml
+    - "moment-builder" → searches database, then schemas/agents/core/moment-builder.yaml
+    - "/absolute/path.yaml" → loads directly from filesystem (exact paths skip database)
+    - "relative/path.yaml" → loads relative to cwd (exact paths skip database)
 
     Search Order:
-    1. Check cache (if use_cache=True and schema found in FS cache)
-    2. Exact path if it exists (absolute or relative)
-    3. Custom paths from rem.register_schema_path() and SCHEMA__PATHS env var
-    4. Package resources: schemas/agents/{name}.yaml (top-level)
-    5. Package resources: schemas/agents/core/{name}.yaml
-    6. Package resources: schemas/agents/examples/{name}.yaml
-    7. Package resources: schemas/evaluators/{name}.yaml
-    8. Package resources: schemas/{name}.yaml
-    9. Database LOOKUP: schemas table (if enable_db_fallback=True and user_id provided)
+    1. Exact path if it exists (absolute or relative) - skips database
+    2. Database LOOKUP: schemas table (if enable_db_fallback=True) - PREFERRED for hot-reload
+    3. Check cache (if use_cache=True and schema found in FS cache)
+    4. Custom paths from rem.register_schema_path() and SCHEMA__PATHS env var
+    5. Package resources: schemas/agents/{name}.yaml (top-level)
+    6. Package resources: schemas/agents/core/{name}.yaml
+    7. Package resources: schemas/agents/examples/{name}.yaml
+    8. Package resources: schemas/evaluators/{name}.yaml
+    9. Package resources: schemas/{name}.yaml
 
     Args:
         schema_name_or_path: Schema name or file path (case-invariant for names)
             Examples: "rem-query-agent", "Contract-Analyzer", "./my-schema.yaml"
         use_cache: If True, uses in-memory cache for filesystem schemas
-        user_id: User ID for database schema lookup (required for DB fallback)
-        enable_db_fallback: If True, falls back to database LOOKUP when file not found
+        user_id: User ID for database schema lookup
+        enable_db_fallback: If True, checks database FIRST for schema (default: True)
 
     Returns:
         Agent schema as dictionary
 
     Raises:
-        FileNotFoundError: If schema not found in any search location (filesystem + database)
+        FileNotFoundError: If schema not found in any search location (database + filesystem)
         yaml.YAMLError: If schema file is invalid YAML
 
     Examples:
-        >>> # Load by short name (cached after first load) - case invariant
-        >>> schema = load_agent_schema("Contract-Analyzer")  # same as "contract-analyzer"
+        >>> # Load by short name - checks database first for hot-reload support
+        >>> schema = load_agent_schema("Contract-Analyzer")  # case invariant
         >>>
-        >>> # Load from custom path (not cached - custom paths may change)
+        >>> # Load from custom path (skips database - exact paths always use filesystem)
         >>> schema = load_agent_schema("./my-agent.yaml")
         >>>
-        >>> # Load evaluator schema (cached)
+        >>> # Load evaluator schema
         >>> schema = load_agent_schema("rem-lookup-correctness")
-        >>>
-        >>> # Load custom user schema from database (case invariant)
-        >>> schema = load_agent_schema("My-Agent", user_id="user-123")  # same as "my-agent"
     """
     # Normalize the name for cache key (lowercase for case-invariant lookups)
     cache_key = str(schema_name_or_path).replace('agents/', '').replace('schemas/', '').replace('evaluators/', '').replace('core/', '').replace('examples/', '').lower()
     if cache_key.endswith('.yaml') or cache_key.endswith('.yml'):
         cache_key = cache_key.rsplit('.', 1)[0]
 
-    # Check cache first (only for package resources, not custom paths)
     path = Path(schema_name_or_path)
-    is_custom_path = path.exists() or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
-
-    if use_cache and not is_custom_path and cache_key in _fs_schema_cache:
-        logger.debug(f"Loading schema from cache: {cache_key}")
-        return _fs_schema_cache[cache_key]
+    is_custom_path = (path.exists() and path.is_file()) or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
 
-    # 1. Try exact path first (absolute or relative to cwd)
-    if path.exists():
+    # 1. Try exact path first (absolute or relative to cwd) - must be a file, not directory
+    # Exact paths skip database lookup (explicit file reference)
+    if path.exists() and path.is_file():
         logger.debug(f"Loading schema from exact path: {path}")
         with open(path, "r") as f:
             schema = yaml.safe_load(f)
@@ -306,10 +339,28 @@ def load_agent_schema(
         # Don't cache custom paths (they may change)
         return cast(dict[str, Any], schema)
 
-    # 2. Normalize name for package resource search (lowercase)
+    # 2. Normalize name for lookups (lowercase)
     base_name = cache_key
 
-    # 3. Try custom schema paths (from registry + SCHEMA__PATHS env var + auto-detected)
+    # 3. Try database FIRST (if enabled) - enables hot-reload without redeploy
+    # Database schemas are NOT cached to ensure hot-reload works immediately
+    if enable_db_fallback and not is_custom_path:
+        try:
+            logger.debug(f"Checking database for schema: {base_name} (user_id={user_id or 'public'})")
+            db_schema = _load_schema_from_database(base_name, user_id)
+            if db_schema:
+                logger.info(f"✅ Loaded schema from database: {base_name}")
+                return db_schema
+        except Exception as e:
+            logger.debug(f"Database schema lookup failed: {e}")
+            # Fall through to filesystem search
+
+    # 4. Check filesystem cache (only for package resources, not custom paths)
+    if use_cache and not is_custom_path and cache_key in _fs_schema_cache:
+        logger.debug(f"Loading schema from cache: {cache_key}")
+        return _fs_schema_cache[cache_key]
+
+    # 5. Try custom schema paths (from registry + SCHEMA__PATHS env var + auto-detected)
     from ..registry import get_schema_paths
 
     custom_paths = get_schema_paths()
@@ -340,7 +391,7 @@ def load_agent_schema(
                 # Don't cache custom paths (they may change during development)
                 return cast(dict[str, Any], schema)
 
-    # 4. Try package resources with standard search paths
+    # 6. Try package resources with standard search paths
     for search_pattern in SCHEMA_SEARCH_PATHS:
         search_path = search_pattern.format(name=base_name)
 
@@ -365,19 +416,7 @@ def load_agent_schema(
             logger.debug(f"Could not load from {search_path}: {e}")
             continue
 
-    # 5. Try database LOOKUP fallback (if enabled and user_id provided)
-    if enable_db_fallback and user_id:
-        try:
-            logger.debug(f"Attempting database LOOKUP for schema: {base_name} (user_id={user_id})")
-            db_schema = _load_schema_from_database(base_name, user_id)
-            if db_schema:
-                logger.info(f"✅ Loaded schema from database: {base_name} (user_id={user_id})")
-                return db_schema
-        except Exception as e:
-            logger.debug(f"Database schema lookup failed: {e}")
-            # Fall through to error below
-
-    # 6. Schema not found in any location
+    # 7. Schema not found in any location
     searched_paths = [pattern.format(name=base_name) for pattern in SCHEMA_SEARCH_PATHS]
 
     custom_paths_note = ""
@@ -387,9 +426,9 @@ def load_agent_schema(
     db_search_note = ""
     if enable_db_fallback:
         if user_id:
-            db_search_note = f"\n  - Database: LOOKUP '{base_name}' FROM schemas WHERE user_id='{user_id}' (no match)"
+            db_search_note = f"\n  - Database: LOOKUP '{base_name}' FROM schemas WHERE user_id IN ('{user_id}', 'system', NULL) (no match)"
         else:
-            db_search_note = "\n  - Database: (skipped - no user_id provided)"
+            db_search_note = f"\n  - Database: LOOKUP '{base_name}' FROM schemas WHERE user_id IN ('system', NULL) (no match)"
 
     raise FileNotFoundError(
         f"Schema not found: {schema_name_or_path}\n"
@@ -405,18 +444,21 @@ async def load_agent_schema_async(
     schema_name_or_path: str,
     user_id: str | None = None,
     db=None,
+    enable_db_fallback: bool = True,
 ) -> dict[str, Any]:
     """
-    Async version of load_agent_schema for use in async contexts.
+    Async version of load_agent_schema with database-first priority.
 
     Schema names are case-invariant - "MyAgent", "myagent", "MYAGENT" all resolve to the same schema.
 
-    This version accepts an existing database connection to avoid creating new connections.
+    **IMPORTANT**: Database is checked FIRST (before filesystem) to enable hot-reloading
+    of schema updates without redeploying the application.
 
     Args:
         schema_name_or_path: Schema name or file path (case-invariant for names)
         user_id: User ID for database schema lookup
         db: Optional existing PostgresService connection (if None, will create one)
+        enable_db_fallback: If True, checks database FIRST for schema (default: True)
 
     Returns:
         Agent schema as dictionary
@@ -424,7 +466,6 @@ async def load_agent_schema_async(
     Raises:
         FileNotFoundError: If schema not found
     """
-    # First try filesystem search (sync operations are fine)
     path = Path(schema_name_or_path)
 
     # Normalize the name for cache key (lowercase for case-invariant lookups)
@@ -432,15 +473,10 @@ async def load_agent_schema_async(
     if cache_key.endswith('.yaml') or cache_key.endswith('.yml'):
         cache_key = cache_key.rsplit('.', 1)[0]
 
-    is_custom_path = path.exists() or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
-
-    # Check cache
-    if not is_custom_path and cache_key in _fs_schema_cache:
-        logger.debug(f"Loading schema from cache: {cache_key}")
-        return _fs_schema_cache[cache_key]
+    is_custom_path = (path.exists() and path.is_file()) or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
 
-    # Try exact path
-    if path.exists():
+    # 1. Try exact path first (skips database - explicit file reference)
+    if path.exists() and path.is_file():
         logger.debug(f"Loading schema from exact path: {path}")
         with open(path, "r") as f:
             schema = yaml.safe_load(f)
@@ -448,7 +484,60 @@ async def load_agent_schema_async(
 
     base_name = cache_key
 
-    # Try custom schema paths (from registry + SCHEMA__PATHS env var + auto-detected)
+    # 2. Try database FIRST (if enabled) - enables hot-reload without redeploy
+    if enable_db_fallback and not is_custom_path:
+        # Check DB schema cache first (TTL-based)
+        cached_schema = _get_cached_db_schema(base_name, user_id)
+        if cached_schema is not None:
+            logger.info(f"✅ Loaded schema from cache: {base_name}")
+            return cached_schema
+
+        # Cache miss - query database
+        from rem.services.postgres import get_postgres_service
+
+        should_disconnect = False
+        if db is None:
+            db = get_postgres_service()
+            if db:
+                await db.connect()
+                should_disconnect = True
+
+        if db:
+            try:
+                if user_id:
+                    query = """
+                        SELECT spec FROM schemas
+                        WHERE LOWER(name) = LOWER($1)
+                        AND (user_id = $2 OR user_id = 'system' OR user_id IS NULL)
+                        LIMIT 1
+                    """
+                    row = await db.fetchrow(query, base_name, user_id)
+                else:
+                    # No user_id - only search public schemas
+                    query = """
+                        SELECT spec FROM schemas
+                        WHERE LOWER(name) = LOWER($1)
+                        AND (user_id = 'system' OR user_id IS NULL)
+                        LIMIT 1
+                    """
+                    row = await db.fetchrow(query, base_name)
+                if row:
+                    spec = row.get("spec")
+                    if spec and isinstance(spec, dict):
+                        # Cache the schema for future requests
+                        _cache_db_schema(base_name, user_id, spec)
+                        logger.info(f"✅ Loaded schema from database: {base_name}")
+                        return spec
+            finally:
+                if should_disconnect:
+                    await db.disconnect()
+
+    # 3. Check filesystem cache
+    if not is_custom_path and cache_key in _fs_schema_cache:
+        logger.debug(f"Loading schema from cache: {cache_key}")
+        return _fs_schema_cache[cache_key]
+
+    # 4. Try custom schema paths (from registry + SCHEMA__PATHS env var + auto-detected)
     from ..registry import get_schema_paths
     custom_paths = get_schema_paths()
 
@@ -470,7 +559,7 @@ async def load_agent_schema_async(
                     schema = yaml.safe_load(f)
                 return cast(dict[str, Any], schema)
 
-    # Try package resources
+    # 5. Try package resources
     for search_pattern in SCHEMA_SEARCH_PATHS:
         search_path = search_pattern.format(name=base_name)
         try:
@@ -484,35 +573,6 @@ async def load_agent_schema_async(
         except Exception:
             continue
 
-    # Try database lookup
-    if user_id:
-        from rem.services.postgres import get_postgres_service
-
-        should_disconnect = False
-        if db is None:
-            db = get_postgres_service()
-            if db:
-                await db.connect()
-                should_disconnect = True
-
-        if db:
-            try:
-                query = """
-                    SELECT spec FROM schemas
-                    WHERE LOWER(name) = LOWER($1)
-                    AND (user_id = $2 OR user_id = 'system' OR user_id IS NULL)
-                    LIMIT 1
-                """
-                row = await db.fetchrow(query, base_name, user_id)
-                if row:
-                    spec = row.get("spec")
-                    if spec and isinstance(spec, dict):
-                        logger.info(f"✅ Loaded schema from database: {base_name}")
-                        return spec
-            finally:
-                if should_disconnect:
-                    await db.disconnect()
-
     # Not found
     raise FileNotFoundError(f"Schema not found: {schema_name_or_path}")
 

{remdb-0.3.180.dist-info → remdb-0.3.258.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: remdb
-Version: 0.3.180
+Version: 0.3.258
 Summary: Resources Entities Moments - Bio-inspired memory system for agentic AI workloads
 Project-URL: Homepage, https://github.com/Percolation-Labs/reminiscent
 Project-URL: Documentation, https://github.com/Percolation-Labs/reminiscent/blob/main/README.md
@@ -28,7 +28,7 @@ Requires-Dist: gitpython>=3.1.45
 Requires-Dist: hypercorn>=0.17.0
 Requires-Dist: itsdangerous>=2.0.0
 Requires-Dist: json-schema-to-pydantic>=0.2.0
-Requires-Dist: kreuzberg<4.0.0,>=3.21.0
+Requires-Dist: kreuzberg>=4.0.5
 Requires-Dist: loguru>=0.7.0
 Requires-Dist: openinference-instrumentation-pydantic-ai>=0.1.0
 Requires-Dist: opentelemetry-api>=1.28.0
@@ -39,7 +39,7 @@ Requires-Dist: opentelemetry-instrumentation-fastapi>=0.49b0
 Requires-Dist: opentelemetry-instrumentation>=0.49b0
 Requires-Dist: opentelemetry-sdk>=1.28.0
 Requires-Dist: psycopg[binary]>=3.2.0
-Requires-Dist: pydantic-ai>=0.0.14
+Requires-Dist: pydantic-ai>=1.0.0
 Requires-Dist: pydantic-settings>=2.6.0
 Requires-Dist: pydantic>=2.10.0
 Requires-Dist: pydub>=0.25.0
@@ -1300,15 +1300,16 @@ FuzzyQuery ::= FUZZY <text:string> [THRESHOLD <t:float>] [LIMIT <n:int>]
   available   : Stage 1+
   example     : FUZZY "sara" THRESHOLD 0.5 LIMIT 10
 
-SearchQuery ::= SEARCH <text:string> [TABLE <table:string>] [WHERE <clause:string>] [LIMIT <n:int>]
+SearchQuery ::= SEARCH <text:string> [IN|TABLE <table:string>] [WHERE <clause:string>] [LIMIT <n:int>]
   text        : Semantic query text
-  table       : Target table (default: "resources")
+  table       : Target table (default: "resources"). Use IN or TABLE keyword.
   clause      : Optional PostgreSQL WHERE clause for hybrid filtering (combines vector + structured)
   limit       : Max results (default: 10)
   performance : Indexed (pgvector)
   available   : Stage 3+
   examples    :
-    - SEARCH "database migration" TABLE resources LIMIT 10
+    - SEARCH "database migration" IN resources LIMIT 10
+    - SEARCH "parcel delivery" IN ontologies
     - SEARCH "team discussion" TABLE moments WHERE "moment_type='meeting'" LIMIT 5
     - SEARCH "project updates" WHERE "created_at >= '2024-01-01'" LIMIT 20
     - SEARCH "AI research" WHERE "tags @> ARRAY['machine-learning']" LIMIT 10