remdb 0.3.163__py3-none-any.whl → 0.3.200__py3-none-any.whl

rem/sql/migrations/004_cache_system.sql

@@ -1,16 +1,15 @@
 -- REM Cache System
--- Description: Self-healing cache for UNLOGGED tables (kv_store)
+-- Description: Cache management helpers for UNLOGGED tables (kv_store)
 -- Version: 1.0.0
 -- Date: 2025-11-29
 --
 -- This migration adds:
 --   1. cache_system_state table for debouncing and API secret storage
 --   2. maybe_trigger_kv_rebuild() function for async rebuild triggering
---   3. Updated rem_lookup/fuzzy/traverse with self-healing on empty cache
+--   3. Helper functions for cache management
 --
--- Self-Healing Flow:
---   Query returns 0 results → Check if kv_store empty → Trigger async rebuild
---   Priority: pg_net (if available) → dblink (always available)
+-- NOTE: Core functions (rem_lookup, rem_fuzzy, rem_traverse) are defined in 001_install.sql
+-- This file only provides cache-specific infrastructure.
 
 -- ============================================================================
 -- REQUIRED EXTENSION
@@ -194,271 +193,6 @@ $$ LANGUAGE plpgsql;
 COMMENT ON FUNCTION maybe_trigger_kv_rebuild IS
 'Async trigger for kv_store rebuild. Uses pg_net (API) or dblink (SQL). Includes debouncing.';
 
--- ============================================================================
--- UPDATED: rem_lookup with self-healing
--- ============================================================================
-
-CREATE OR REPLACE FUNCTION rem_lookup(
-    p_entity_key VARCHAR(255),
-    p_tenant_id VARCHAR(100),
-    p_user_id VARCHAR(100)
-)
-RETURNS TABLE(
-    entity_type VARCHAR(100),
-    data JSONB
-) AS $$
-DECLARE
-    entity_table VARCHAR(100);
-    query_sql TEXT;
-    effective_user_id VARCHAR(100);
-    v_result_count INTEGER := 0;
-BEGIN
-    effective_user_id := COALESCE(p_user_id, p_tenant_id);
-
-    -- First lookup in KV store to get entity_type (table name)
-    SELECT kv.entity_type INTO entity_table
-    FROM kv_store kv
-    WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
-    AND kv.entity_key = p_entity_key
-    LIMIT 1;
-
-    -- If not found, check if cache is empty and maybe trigger rebuild
-    IF entity_table IS NULL THEN
-        -- SELF-HEALING: Check if this is because cache is empty
-        IF rem_kv_store_empty(effective_user_id) THEN
-            PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_lookup');
-        END IF;
-        RETURN;
-    END IF;
-
-    -- Fetch raw record from underlying table as JSONB
-    query_sql := format('
-        SELECT
-            %L::VARCHAR(100) AS entity_type,
-            row_to_json(t)::jsonb AS data
-        FROM %I t
-        WHERE (t.user_id = $1 OR t.user_id IS NULL)
-        AND t.name = $2
-        AND t.deleted_at IS NULL
-    ', entity_table, entity_table);
-
-    RETURN QUERY EXECUTE query_sql USING effective_user_id, p_entity_key;
-END;
-$$ LANGUAGE plpgsql STABLE;
-
--- ============================================================================
--- UPDATED: rem_fuzzy with self-healing
--- ============================================================================
-
-CREATE OR REPLACE FUNCTION rem_fuzzy(
-    p_query TEXT,
-    p_tenant_id VARCHAR(100),
-    p_threshold REAL DEFAULT 0.3,
-    p_limit INTEGER DEFAULT 10,
-    p_user_id VARCHAR(100) DEFAULT NULL
-)
-RETURNS TABLE(
-    entity_type VARCHAR(100),
-    similarity_score REAL,
-    data JSONB
-) AS $$
-DECLARE
-    kv_matches RECORD;
-    entities_by_table JSONB := '{}'::jsonb;
-    table_keys JSONB;
-    effective_user_id VARCHAR(100);
-    v_found_any BOOLEAN := FALSE;
-BEGIN
-    effective_user_id := COALESCE(p_user_id, p_tenant_id);
-
-    -- Find matching keys in KV store
-    FOR kv_matches IN
-        SELECT
-            kv.entity_key,
-            kv.entity_type,
-            similarity(kv.entity_key, p_query) AS sim_score
-        FROM kv_store kv
-        WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
-        AND kv.entity_key % p_query
-        AND similarity(kv.entity_key, p_query) >= p_threshold
-        ORDER BY sim_score DESC
-        LIMIT p_limit
-    LOOP
-        v_found_any := TRUE;
-        -- Build JSONB mapping {table: [keys]}
-        IF entities_by_table ? kv_matches.entity_type THEN
-            table_keys := entities_by_table->kv_matches.entity_type;
-            entities_by_table := jsonb_set(
-                entities_by_table,
-                ARRAY[kv_matches.entity_type],
-                table_keys || jsonb_build_array(kv_matches.entity_key)
-            );
-        ELSE
-            entities_by_table := jsonb_set(
-                entities_by_table,
-                ARRAY[kv_matches.entity_type],
-                jsonb_build_array(kv_matches.entity_key)
-            );
-        END IF;
-    END LOOP;
-
-    -- SELF-HEALING: If no matches and cache is empty, trigger rebuild
-    IF NOT v_found_any AND rem_kv_store_empty(effective_user_id) THEN
-        PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_fuzzy');
-    END IF;
-
-    -- Fetch full records
-    RETURN QUERY
-    SELECT
-        f.entity_type::VARCHAR(100),
-        similarity(f.entity_key, p_query) AS similarity_score,
-        f.entity_record AS data
-    FROM rem_fetch(entities_by_table, effective_user_id) f
-    ORDER BY similarity_score DESC;
-END;
-$$ LANGUAGE plpgsql STABLE;
-
--- ============================================================================
--- UPDATED: rem_traverse with self-healing
--- ============================================================================
-
-CREATE OR REPLACE FUNCTION rem_traverse(
-    p_entity_key VARCHAR(255),
-    p_tenant_id VARCHAR(100),
-    p_user_id VARCHAR(100),
-    p_max_depth INTEGER DEFAULT 1,
-    p_rel_type VARCHAR(100) DEFAULT NULL,
-    p_keys_only BOOLEAN DEFAULT FALSE
-)
-RETURNS TABLE(
-    depth INTEGER,
-    entity_key VARCHAR(255),
-    entity_type VARCHAR(100),
-    entity_id UUID,
-    rel_type VARCHAR(100),
-    rel_weight REAL,
-    path TEXT[],
-    entity_record JSONB
-) AS $$
-DECLARE
-    graph_keys RECORD;
-    entities_by_table JSONB := '{}'::jsonb;
-    table_keys JSONB;
-    effective_user_id VARCHAR(100);
-    v_found_start BOOLEAN := FALSE;
-BEGIN
-    effective_user_id := COALESCE(p_user_id, p_tenant_id);
-
-    -- Check if start entity exists in kv_store
-    SELECT TRUE INTO v_found_start
-    FROM kv_store kv
-    WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
-    AND kv.entity_key = p_entity_key
-    LIMIT 1;
-
-    -- SELF-HEALING: If start not found and cache is empty, trigger rebuild
-    IF NOT COALESCE(v_found_start, FALSE) THEN
-        IF rem_kv_store_empty(effective_user_id) THEN
-            PERFORM maybe_trigger_kv_rebuild(effective_user_id, 'rem_traverse');
-        END IF;
-        RETURN;
-    END IF;
-
-    -- Original traverse logic
-    FOR graph_keys IN
-        WITH RECURSIVE graph_traversal AS (
-            SELECT
-                0 AS depth,
-                kv.entity_key,
-                kv.entity_type,
-                kv.entity_id,
-                NULL::VARCHAR(100) AS rel_type,
-                NULL::REAL AS rel_weight,
-                ARRAY[kv.entity_key]::TEXT[] AS path
-            FROM kv_store kv
-            WHERE (kv.user_id = effective_user_id OR kv.user_id IS NULL)
-            AND kv.entity_key = p_entity_key
-
-            UNION ALL
-
-            SELECT
-                gt.depth + 1,
-                target_kv.entity_key,
-                target_kv.entity_type,
-                target_kv.entity_id,
-                (edge->>'rel_type')::VARCHAR(100) AS rel_type,
-                COALESCE((edge->>'weight')::REAL, 1.0) AS rel_weight,
-                gt.path || target_kv.entity_key AS path
-            FROM graph_traversal gt
-            JOIN kv_store source_kv ON source_kv.entity_key = gt.entity_key
-                AND (source_kv.user_id = effective_user_id OR source_kv.user_id IS NULL)
-            CROSS JOIN LATERAL jsonb_array_elements(COALESCE(source_kv.graph_edges, '[]'::jsonb)) AS edge
-            JOIN kv_store target_kv ON target_kv.entity_key = (edge->>'dst')::VARCHAR(255)
-                AND (target_kv.user_id = effective_user_id OR target_kv.user_id IS NULL)
-            WHERE gt.depth < p_max_depth
-            AND (p_rel_type IS NULL OR (edge->>'rel_type')::VARCHAR(100) = p_rel_type)
-            AND NOT (target_kv.entity_key = ANY(gt.path))
-        )
-        SELECT DISTINCT ON (gt.entity_key)
-            gt.depth,
-            gt.entity_key,
-            gt.entity_type,
-            gt.entity_id,
-            gt.rel_type,
-            gt.rel_weight,
-            gt.path
-        FROM graph_traversal gt
-        WHERE gt.depth > 0
-        ORDER BY gt.entity_key, gt.depth
-    LOOP
-        IF p_keys_only THEN
-            depth := graph_keys.depth;
-            entity_key := graph_keys.entity_key;
-            entity_type := graph_keys.entity_type;
-            entity_id := graph_keys.entity_id;
-            rel_type := graph_keys.rel_type;
-            rel_weight := graph_keys.rel_weight;
-            path := graph_keys.path;
-            entity_record := NULL;
-            RETURN NEXT;
-        ELSE
-            IF entities_by_table ? graph_keys.entity_type THEN
-                table_keys := entities_by_table->graph_keys.entity_type;
-                entities_by_table := jsonb_set(
-                    entities_by_table,
-                    ARRAY[graph_keys.entity_type],
-                    table_keys || jsonb_build_array(graph_keys.entity_key)
-                );
-            ELSE
-                entities_by_table := jsonb_set(
-                    entities_by_table,
-                    ARRAY[graph_keys.entity_type],
-                    jsonb_build_array(graph_keys.entity_key)
-                );
-            END IF;
-        END IF;
-    END LOOP;
-
-    IF NOT p_keys_only THEN
-        RETURN QUERY
-        SELECT
-            g.depth,
-            g.entity_key,
-            g.entity_type,
-            g.entity_id,
-            g.rel_type,
-            g.rel_weight,
-            g.path,
-            f.entity_record
-        FROM (
-            SELECT * FROM rem_traverse(p_entity_key, p_tenant_id, effective_user_id, p_max_depth, p_rel_type, TRUE)
-        ) g
-        LEFT JOIN rem_fetch(entities_by_table, effective_user_id) f
-            ON g.entity_key = f.entity_key;
-    END IF;
-END;
-$$ LANGUAGE plpgsql STABLE;
-
 -- ============================================================================
 -- HELPER: Get API secret for validation
 -- ============================================================================
@@ -527,9 +261,9 @@ BEGIN
     RAISE NOTICE '';
     RAISE NOTICE 'Functions:';
     RAISE NOTICE '  maybe_trigger_kv_rebuild() - Async rebuild trigger';
-    RAISE NOTICE '  rem_lookup() - Updated with self-healing';
-    RAISE NOTICE '  rem_fuzzy() - Updated with self-healing';
-    RAISE NOTICE '  rem_traverse() - Updated with self-healing';
+    RAISE NOTICE '  rem_kv_store_empty() - Check if cache is empty';
+    RAISE NOTICE '  rem_get_cache_api_secret() - Get API secret';
+    RAISE NOTICE '  rem_record_cache_rebuild() - Record rebuild completion';
     RAISE NOTICE '';
     RAISE NOTICE 'Async Methods Available:';
     IF v_has_pgnet THEN
@@ -542,7 +276,5 @@ BEGIN
     ELSE
         RAISE NOTICE '  [ ] dblink - Not installed';
     END IF;
-    RAISE NOTICE '';
-    RAISE NOTICE 'Self-Healing: Queries will auto-trigger rebuild on empty cache';
     RAISE NOTICE '============================================================';
 END $$;

rem/utils/model_helpers.py

@@ -19,6 +19,9 @@ Table Name Inference:
 
 Model Resolution:
 - model_from_arbitrary_casing: Resolve model class from flexible input casing
+
+Data Validation:
+- validate_data_for_model: Validate row data against a Pydantic model with clear error reporting
 """
 
 import re
@@ -389,3 +392,101 @@ def model_from_arbitrary_casing(
         f"Unknown model: '{name}' (normalized: '{normalized}'). "
         f"Available models: {', '.join(available)}"
     )
+
+
+class ValidationResult:
+    """Result of validating data against a Pydantic model."""
+
+    def __init__(
+        self,
+        valid: bool,
+        instance: BaseModel | None = None,
+        errors: list[str] | None = None,
+        missing_required: set[str] | None = None,
+        extra_fields: set[str] | None = None,
+        required_fields: set[str] | None = None,
+        optional_fields: set[str] | None = None,
+    ):
+        self.valid = valid
+        self.instance = instance
+        self.errors = errors or []
+        self.missing_required = missing_required or set()
+        self.extra_fields = extra_fields or set()
+        self.required_fields = required_fields or set()
+        self.optional_fields = optional_fields or set()
+
+    def log_errors(self, row_label: str = "Row") -> None:
+        """Log validation errors using loguru."""
+        if self.valid:
+            return
+
+        logger.error(f"{row_label}: Validation failed")
+        if self.missing_required:
+            logger.error(f"  Missing required: {self.missing_required}")
+        if self.extra_fields:
+            logger.warning(f"  Unknown fields (ignored): {self.extra_fields}")
+        for err in self.errors:
+            logger.error(f"  - {err}")
+        logger.info(f"  Required: {self.required_fields or '(none)'}")
+        logger.info(f"  Optional: {self.optional_fields}")
+
+
+def validate_data_for_model(
+    model: Type[BaseModel],
+    data: dict[str, Any],
+) -> ValidationResult:
+    """
+    Validate a data dict against a Pydantic model with detailed error reporting.
+
+    Args:
+        model: Pydantic model class to validate against
+        data: Dictionary of field values
+
+    Returns:
+        ValidationResult with validation status and detailed field info
+
+    Example:
+        >>> from rem.models.entities import Resource
+        >>> result = validate_data_for_model(Resource, {"name": "test", "content": "hello"})
+        >>> result.valid
+        True
+        >>> result = validate_data_for_model(Resource, {"unknown_field": "value"})
+        >>> result.valid
+        True  # Resource has no required fields
+        >>> result.extra_fields
+        {'unknown_field'}
+    """
+    from pydantic import ValidationError
+
+    model_fields = set(model.model_fields.keys())
+    required = {k for k, v in model.model_fields.items() if v.is_required()}
+    optional = model_fields - required
+    data_fields = set(data.keys())
+
+    missing_required = required - data_fields
+    extra_fields = data_fields - model_fields
+
+    try:
+        instance = model(**data)
+        return ValidationResult(
+            valid=True,
+            instance=instance,
+            required_fields=required,
+            optional_fields=optional,
+            extra_fields=extra_fields,
+        )
+    except ValidationError as e:
+        errors = []
+        for err in e.errors():
+            field = ".".join(str(p) for p in err["loc"])
+            if field not in missing_required:  # Don't double-report missing
+                errors.append(f"{field}: {err['msg']}")
+
+        return ValidationResult(
+            valid=False,
+            errors=errors,
+            missing_required=missing_required,
+            extra_fields=extra_fields,
+            required_fields=required,
+            optional_fields=optional,
+        )

rem/utils/schema_loader.py

@@ -132,13 +132,51 @@ def _load_schema_from_database(schema_name: str, user_id: str) -> dict[str, Any]
     # Check if we're already in an async context
     try:
         loop = asyncio.get_running_loop()
-        # We're in an async context - can't use asyncio.run()
-        # This shouldn't happen in normal usage since load_agent_schema is called from sync contexts
-        logger.warning(
-            "Database schema lookup called from async context. "
-            "This may cause issues. Consider using async version of load_agent_schema."
-        )
-        return None
+        # We're in an async context - use thread executor to run async code
+        import concurrent.futures
+
+        async def _async_lookup():
+            """Async helper to query database."""
+            from rem.services.postgres import get_postgres_service
+
+            db = get_postgres_service()
+            if not db:
+                logger.debug("PostgreSQL service not available for schema lookup")
+                return None
+
+            try:
+                await db.connect()
+
+                query = """
+                    SELECT spec FROM schemas
+                    WHERE LOWER(name) = LOWER($1)
+                    AND (user_id = $2 OR user_id = 'system' OR user_id IS NULL)
+                    LIMIT 1
+                """
+                logger.debug(f"Executing schema lookup: name={schema_name}, user_id={user_id}")
+
+                row = await db.fetchrow(query, schema_name, user_id)
+
+                if row:
+                    spec = row.get("spec")
+                    if spec and isinstance(spec, dict):
+                        logger.debug(f"Found schema in database: {schema_name}")
+                        return spec
+
+                logger.debug(f"Schema not found in database: {schema_name}")
+                return None
+
+            except Exception as e:
+                logger.debug(f"Database schema lookup error: {e}")
+                return None
+            finally:
+                await db.disconnect()
+
+        # Run in thread pool to avoid blocking the event loop
+        with concurrent.futures.ThreadPoolExecutor() as pool:
+            future = pool.submit(asyncio.run, _async_lookup())
+            return future.result(timeout=10)
+
     except RuntimeError:
         # Not in async context - safe to use asyncio.run()
         pass
@@ -253,14 +291,14 @@ def load_agent_schema(
 
     # Check cache first (only for package resources, not custom paths)
     path = Path(schema_name_or_path)
-    is_custom_path = path.exists() or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
+    is_custom_path = (path.exists() and path.is_file()) or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
 
     if use_cache and not is_custom_path and cache_key in _fs_schema_cache:
         logger.debug(f"Loading schema from cache: {cache_key}")
         return _fs_schema_cache[cache_key]
 
-    # 1. Try exact path first (absolute or relative to cwd)
-    if path.exists():
+    # 1. Try exact path first (absolute or relative to cwd) - must be a file, not directory
+    if path.exists() and path.is_file():
         logger.debug(f"Loading schema from exact path: {path}")
         with open(path, "r") as f:
             schema = yaml.safe_load(f)
@@ -394,15 +432,15 @@ async def load_agent_schema_async(
     if cache_key.endswith('.yaml') or cache_key.endswith('.yml'):
         cache_key = cache_key.rsplit('.', 1)[0]
 
-    is_custom_path = path.exists() or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
+    is_custom_path = (path.exists() and path.is_file()) or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
 
     # Check cache
     if not is_custom_path and cache_key in _fs_schema_cache:
         logger.debug(f"Loading schema from cache: {cache_key}")
         return _fs_schema_cache[cache_key]
 
-    # Try exact path
-    if path.exists():
+    # Try exact path (must be a file, not directory)
+    if path.exists() and path.is_file():
         logger.debug(f"Loading schema from exact path: {path}")
         with open(path, "r") as f:
             schema = yaml.safe_load(f)

{remdb-0.3.163.dist-info → remdb-0.3.200.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: remdb
-Version: 0.3.163
+Version: 0.3.200
 Summary: Resources Entities Moments - Bio-inspired memory system for agentic AI workloads
 Project-URL: Homepage, https://github.com/Percolation-Labs/reminiscent
 Project-URL: Documentation, https://github.com/Percolation-Labs/reminiscent/blob/main/README.md