remdb 0.3.0__py3-none-any.whl → 0.3.127__py3-none-any.whl

rem/utils/model_helpers.py

@@ -16,8 +16,12 @@ Embedding Field Detection:
 Table Name Inference:
 1. model_config.json_schema_extra.table_name
 2. CamelCase → snake_case + pluralization
+
+Model Resolution:
+- model_from_arbitrary_casing: Resolve model class from flexible input casing
 """
 
+import re
 from typing import Any, Type
 
 from loguru import logger
@@ -94,7 +98,9 @@ def get_table_name(model: Type[BaseModel]) -> str:
         if isinstance(model_config, dict):
             json_extra = model_config.get("json_schema_extra", {})
             if isinstance(json_extra, dict) and "table_name" in json_extra:
-                return json_extra["table_name"]
+                table_name = json_extra["table_name"]
+                if isinstance(table_name, str):
+                    return table_name
 
     # Infer from class name
     name = model.__name__
@@ -234,3 +240,152 @@ def get_model_metadata(model: Type[BaseModel]) -> dict[str, Any]:
         "entity_key_field": get_entity_key_field(model),
         "embeddable_fields": get_embeddable_fields(model),
     }
+
+
+def normalize_to_title_case(name: str) -> str:
+    """
+    Normalize arbitrary casing to TitleCase (PascalCase).
+
+    Handles various input formats:
+    - kebab-case: domain-resource → DomainResource
+    - snake_case: domain_resource → DomainResource
+    - lowercase: domainresource → Domainresource (single word)
+    - TitleCase: DomainResource → DomainResource (passthrough)
+    - Mixed: Domain-Resource, DOMAIN_RESOURCE → DomainResource
+
+    Args:
+        name: Input name in any casing format
+
+    Returns:
+        TitleCase (PascalCase) version of the name
+
+    Example:
+        >>> normalize_to_title_case("domain-resource")
+        'DomainResource'
+        >>> normalize_to_title_case("domain_resources")
+        'DomainResources'
+        >>> normalize_to_title_case("DomainResource")
+        'DomainResource'
+    """
+    # If already TitleCase (starts with uppercase, has no delimiters, and has
+    # at least one lowercase letter), return as-is
+    if (
+        name
+        and name[0].isupper()
+        and '-' not in name
+        and '_' not in name
+        and any(c.islower() for c in name)
+    ):
+        return name
+
+    # Split on common delimiters (hyphen, underscore)
+    parts = re.split(r'[-_]', name)
+
+    # Capitalize first letter of each part, lowercase the rest
+    normalized_parts = [part.capitalize() for part in parts if part]
+
+    return "".join(normalized_parts)
+
+
+def model_from_arbitrary_casing(
+    name: str,
+    registry: dict[str, Type[BaseModel]] | None = None,
+) -> Type[BaseModel]:
+    """
+    Resolve a model class from arbitrary casing input.
+
+    REM entity models use strict TitleCase (PascalCase) naming. This function
+    allows flexible input formats while maintaining consistency:
+
+    Input formats supported:
+    - kebab-case: domain-resource, domain-resources
+    - snake_case: domain_resource, domain_resources
+    - lowercase: resource, domainresource
+    - TitleCase: Resource, DomainResource
+
+    Args:
+        name: Model name in any supported casing format
+        registry: Optional dict mapping TitleCase names to model classes.
+                  If not provided, uses rem.models.entities module.
+
+    Returns:
+        The resolved Pydantic model class
+
+    Raises:
+        ValueError: If no model matches the normalized name
+
+    Example:
+        >>> model = model_from_arbitrary_casing("domain-resources")
+        >>> model.__name__
+        'DomainResource'
+        >>> model = model_from_arbitrary_casing("Resource")
+        >>> model.__name__
+        'Resource'
+    """
+    # Build default registry from entities module if not provided
+    if registry is None:
+        from rem.models.entities import (
+            DomainResource,
+            Feedback,
+            File,
+            ImageResource,
+            Message,
+            Moment,
+            Ontology,
+            OntologyConfig,
+            Resource,
+            Schema,
+            Session,
+            User,
+        )
+
+        registry = {
+            "Resource": Resource,
+            "Resources": Resource,  # Plural alias
+            "DomainResource": DomainResource,
+            "DomainResources": DomainResource,  # Plural alias
+            "ImageResource": ImageResource,
+            "ImageResources": ImageResource,
+            "File": File,
+            "Files": File,
+            "Message": Message,
+            "Messages": Message,
+            "Moment": Moment,
+            "Moments": Moment,
+            "Session": Session,
+            "Sessions": Session,
+            "Feedback": Feedback,
+            "User": User,
+            "Users": User,
+            "Schema": Schema,
+            "Schemas": Schema,
+            "Ontology": Ontology,
+            "Ontologies": Ontology,
+            "OntologyConfig": OntologyConfig,
+            "OntologyConfigs": OntologyConfig,
+        }
+
+    # Normalize input to TitleCase
+    normalized = normalize_to_title_case(name)
+
+    # Look up in registry
+    if normalized in registry:
+        logger.debug(f"Resolved model '{name}' → {registry[normalized].__name__}")
+        return registry[normalized]
+
+    # Try without trailing 's' (singular form)
+    if normalized.endswith("s") and normalized[:-1] in registry:
+        logger.debug(f"Resolved model '{name}' → {registry[normalized[:-1]].__name__} (singular)")
+        return registry[normalized[:-1]]
+
+    # Try with trailing 's' (plural form)
+    plural = normalized + "s"
+    if plural in registry:
+        logger.debug(f"Resolved model '{name}' → {registry[plural].__name__} (plural)")
+        return registry[plural]
+
+    available = sorted(set(m.__name__ for m in registry.values()))
+    raise ValueError(
+        f"Unknown model: '{name}' (normalized: '{normalized}'). "
+        f"Available models: {', '.join(available)}"
+    )

rem/utils/schema_loader.py

@@ -9,7 +9,7 @@ Design Pattern:
 - Support short names: "contract-analyzer" → "schemas/agents/contract-analyzer.yaml"
 - Support relative/absolute paths
 - Consistent error messages and logging
-i
+
 Usage:
     # From API
     schema = load_agent_schema("rem")
@@ -20,6 +20,26 @@ Usage:
     # From agent factory
     schema = load_agent_schema("contract-analyzer")
 
+TODO: Git FS Integration
+    The schema loader currently uses importlib.resources for package schemas
+    and direct filesystem access for custom paths. The FS abstraction layer
+    (rem.services.fs.FS) could be used to abstract storage backends:
+
+    - Local filesystem (current)
+    - Git repositories (GitService)
+    - S3 (via FS provider)
+
+    This would enable loading schemas from versioned Git repos or S3 buckets
+    without changing the API. The FS provider pattern already exists and just
+    needs integration testing with the schema loader.
+
+    Example future usage:
+        # Load from Git at specific version
+        schema = load_agent_schema("git://rem/schemas/agents/rem.yaml?ref=v1.0.0")
+
+        # Load from S3
+        schema = load_agent_schema("s3://rem-schemas/agents/cv-parser.yaml")
+
 Schema Caching Status:
 
     ✅ IMPLEMENTED: Filesystem Schema Caching (2025-11-22)
@@ -71,13 +91,14 @@ import yaml
 from loguru import logger
 
 
-# Standard search paths for agent schemas (in priority order)
+# Standard search paths for agent/evaluator schemas (in priority order)
 SCHEMA_SEARCH_PATHS = [
     "schemas/agents/{name}.yaml",          # Top-level agents (e.g., rem.yaml)
     "schemas/agents/core/{name}.yaml",     # Core system agents
     "schemas/agents/examples/{name}.yaml", # Example agents
-    "schemas/evaluators/{name}.yaml",
-    "schemas/{name}.yaml",
+    "schemas/evaluators/{name}.yaml",      # Nested evaluators (e.g., hello-world/default)
+    "schemas/evaluators/rem/{name}.yaml",  # REM evaluators (e.g., lookup-correctness)
+    "schemas/{name}.yaml",                 # Generic schemas
 ]
 
 # In-memory cache for filesystem schemas (no TTL - immutable)
@@ -125,7 +146,6 @@ def _load_schema_from_database(schema_name: str, user_id: str) -> dict[str, Any]
     async def _async_lookup():
         """Async helper to query database."""
         from rem.services.postgres import get_postgres_service
-        from rem.models.entities import Schema
 
         db = get_postgres_service()
         if not db:
@@ -135,19 +155,20 @@ def _load_schema_from_database(schema_name: str, user_id: str) -> dict[str, Any]
         try:
             await db.connect()
 
-            # Use REM LOOKUP query to find schema
-            query = f"LOOKUP '{schema_name}' FROM schemas"
-            logger.debug(f"Executing: {query} (user_id={user_id})")
+            # Query schemas table directly by name
+            # Note: Schema name lookup is case-insensitive for user convenience
+            query = """
+                SELECT spec FROM schemas
+                WHERE LOWER(name) = LOWER($1)
+                AND (user_id = $2 OR user_id = 'system')
+                LIMIT 1
+            """
+            logger.debug(f"Executing schema lookup: name={schema_name}, user_id={user_id}")
 
-            result = await db.execute_rem_query(
-                query=query,
-                user_id=user_id,
-            )
+            row = await db.fetchrow(query, schema_name, user_id)
 
-            if result and isinstance(result, dict):
-                # LOOKUP returns single entity or None
-                # Extract spec field (JSON Schema)
-                spec = result.get("spec")
+            if row:
+                spec = row.get("spec")
                 if spec and isinstance(spec, dict):
                     logger.debug(f"Found schema in database: {schema_name}")
                     return spec
@@ -174,6 +195,8 @@ def load_agent_schema(
     """
     Load agent schema from YAML file with unified search logic and caching.
 
+    Schema names are case-invariant - "Rem", "rem", "REM" all resolve to the same schema.
+
     Filesystem schemas are cached indefinitely (immutable, versioned with code).
     Database schemas (future) will be cached with TTL for invalidation.
 
@@ -188,16 +211,17 @@ def load_agent_schema(
     Search Order:
     1. Check cache (if use_cache=True and schema found in FS cache)
     2. Exact path if it exists (absolute or relative)
-    3. Package resources: schemas/agents/{name}.yaml (top-level)
-    4. Package resources: schemas/agents/core/{name}.yaml
-    5. Package resources: schemas/agents/examples/{name}.yaml
-    6. Package resources: schemas/evaluators/{name}.yaml
-    7. Package resources: schemas/{name}.yaml
-    8. Database LOOKUP: schemas table (if enable_db_fallback=True and user_id provided)
+    3. Custom paths from rem.register_schema_path() and SCHEMA__PATHS env var
+    4. Package resources: schemas/agents/{name}.yaml (top-level)
+    5. Package resources: schemas/agents/core/{name}.yaml
+    6. Package resources: schemas/agents/examples/{name}.yaml
+    7. Package resources: schemas/evaluators/{name}.yaml
+    8. Package resources: schemas/{name}.yaml
+    9. Database LOOKUP: schemas table (if enable_db_fallback=True and user_id provided)
 
     Args:
-        schema_name_or_path: Schema name or file path
-            Examples: "rem-query-agent", "contract-analyzer", "./my-schema.yaml"
+        schema_name_or_path: Schema name or file path (case-invariant for names)
+            Examples: "rem-query-agent", "Contract-Analyzer", "./my-schema.yaml"
         use_cache: If True, uses in-memory cache for filesystem schemas
         user_id: User ID for database schema lookup (required for DB fallback)
         enable_db_fallback: If True, falls back to database LOOKUP when file not found
@@ -210,8 +234,8 @@ def load_agent_schema(
         yaml.YAMLError: If schema file is invalid YAML
 
     Examples:
-        >>> # Load by short name (cached after first load)
-        >>> schema = load_agent_schema("contract-analyzer")
+        >>> # Load by short name (cached after first load) - case invariant
+        >>> schema = load_agent_schema("Contract-Analyzer")  # same as "contract-analyzer"
         >>>
         >>> # Load from custom path (not cached - custom paths may change)
         >>> schema = load_agent_schema("./my-agent.yaml")
@@ -219,11 +243,11 @@ def load_agent_schema(
         >>> # Load evaluator schema (cached)
         >>> schema = load_agent_schema("rem-lookup-correctness")
         >>>
-        >>> # Load custom user schema from database
-        >>> schema = load_agent_schema("my-custom-agent", user_id="user-123")
+        >>> # Load custom user schema from database (case invariant)
+        >>> schema = load_agent_schema("My-Agent", user_id="user-123")  # same as "my-agent"
     """
-    # Normalize the name for cache key
-    cache_key = str(schema_name_or_path).replace('agents/', '').replace('schemas/', '').replace('evaluators/', '').replace('core/', '').replace('examples/', '')
+    # Normalize the name for cache key (lowercase for case-invariant lookups)
+    cache_key = str(schema_name_or_path).replace('agents/', '').replace('schemas/', '').replace('evaluators/', '').replace('core/', '').replace('examples/', '').lower()
     if cache_key.endswith('.yaml') or cache_key.endswith('.yml'):
         cache_key = cache_key.rsplit('.', 1)[0]
 
@@ -244,10 +268,41 @@ def load_agent_schema(
         # Don't cache custom paths (they may change)
         return cast(dict[str, Any], schema)
 
-    # 2. Normalize name for package resource search
+    # 2. Normalize name for package resource search (lowercase)
     base_name = cache_key
 
-    # 3. Try package resources with standard search paths
+    # 3. Try custom schema paths (from registry + SCHEMA__PATHS env var + auto-detected)
+    from ..registry import get_schema_paths
+
+    custom_paths = get_schema_paths()
+
+    # Auto-detect local folders if they exist (convention over configuration)
+    auto_detect_folders = ["./agents", "./schemas", "./evaluators"]
+    for auto_folder in auto_detect_folders:
+        auto_path = Path(auto_folder)
+        if auto_path.exists() and auto_path.is_dir():
+            resolved = str(auto_path.resolve())
+            if resolved not in custom_paths:
+                custom_paths.insert(0, resolved)
+                logger.debug(f"Auto-detected schema directory: {auto_folder}")
+    for custom_dir in custom_paths:
+        # Try various patterns within each custom directory
+        for pattern in [
+            f"{base_name}.yaml",
+            f"{base_name}.yml",
+            f"agents/{base_name}.yaml",
+            f"evaluators/{base_name}.yaml",
+        ]:
+            custom_path = Path(custom_dir) / pattern
+            if custom_path.exists():
+                logger.debug(f"Loading schema from custom path: {custom_path}")
+                with open(custom_path, "r") as f:
+                    schema = yaml.safe_load(f)
+                logger.debug(f"Loaded schema with keys: {list(schema.keys())}")
+                # Don't cache custom paths (they may change during development)
+                return cast(dict[str, Any], schema)
+
+    # 4. Try package resources with standard search paths
     for search_pattern in SCHEMA_SEARCH_PATHS:
         search_path = search_pattern.format(name=base_name)
 
@@ -272,7 +327,7 @@ def load_agent_schema(
             logger.debug(f"Could not load from {search_path}: {e}")
             continue
 
-    # 4. Try database LOOKUP fallback (if enabled and user_id provided)
+    # 5. Try database LOOKUP fallback (if enabled and user_id provided)
     if enable_db_fallback and user_id:
         try:
             logger.debug(f"Attempting database LOOKUP for schema: {base_name} (user_id={user_id})")
@@ -284,8 +339,13 @@ def load_agent_schema(
             logger.debug(f"Database schema lookup failed: {e}")
             # Fall through to error below
 
-    # 5. Schema not found in any location
+    # 6. Schema not found in any location
     searched_paths = [pattern.format(name=base_name) for pattern in SCHEMA_SEARCH_PATHS]
+
+    custom_paths_note = ""
+    if custom_paths:
+        custom_paths_note = f"\n  - Custom paths: {', '.join(custom_paths)}"
+
     db_search_note = ""
     if enable_db_fallback:
         if user_id:
@@ -296,12 +356,129 @@ def load_agent_schema(
     raise FileNotFoundError(
         f"Schema not found: {schema_name_or_path}\n"
         f"Searched locations:\n"
-        f"  - Exact path: {path}\n"
+        f"  - Exact path: {path}"
+        f"{custom_paths_note}\n"
         f"  - Package resources: {', '.join(searched_paths)}"
         f"{db_search_note}"
     )
 
 
+async def load_agent_schema_async(
+    schema_name_or_path: str,
+    user_id: str | None = None,
+    db=None,
+) -> dict[str, Any]:
+    """
+    Async version of load_agent_schema for use in async contexts.
+
+    Schema names are case-invariant - "MyAgent", "myagent", "MYAGENT" all resolve to the same schema.
+
+    This version accepts an existing database connection to avoid creating new connections.
+
+    Args:
+        schema_name_or_path: Schema name or file path (case-invariant for names)
+        user_id: User ID for database schema lookup
+        db: Optional existing PostgresService connection (if None, will create one)
+
+    Returns:
+        Agent schema as dictionary
+
+    Raises:
+        FileNotFoundError: If schema not found
+    """
+    # First try filesystem search (sync operations are fine)
+    path = Path(schema_name_or_path)
+
+    # Normalize the name for cache key (lowercase for case-invariant lookups)
+    cache_key = str(schema_name_or_path).replace('agents/', '').replace('schemas/', '').replace('evaluators/', '').replace('core/', '').replace('examples/', '').lower()
+    if cache_key.endswith('.yaml') or cache_key.endswith('.yml'):
+        cache_key = cache_key.rsplit('.', 1)[0]
+
+    is_custom_path = path.exists() or '/' in str(schema_name_or_path) or '\\' in str(schema_name_or_path)
+
+    # Check cache
+    if not is_custom_path and cache_key in _fs_schema_cache:
+        logger.debug(f"Loading schema from cache: {cache_key}")
+        return _fs_schema_cache[cache_key]
+
+    # Try exact path
+    if path.exists():
+        logger.debug(f"Loading schema from exact path: {path}")
+        with open(path, "r") as f:
+            schema = yaml.safe_load(f)
+        return cast(dict[str, Any], schema)
+
+    base_name = cache_key
+
+    # Try custom schema paths (from registry + SCHEMA__PATHS env var + auto-detected)
+    from ..registry import get_schema_paths
+    custom_paths = get_schema_paths()
+
+    # Auto-detect local folders if they exist (convention over configuration)
+    auto_detect_folders = ["./agents", "./schemas", "./evaluators"]
+    for auto_folder in auto_detect_folders:
+        auto_path = Path(auto_folder)
+        if auto_path.exists() and auto_path.is_dir():
+            resolved = str(auto_path.resolve())
+            if resolved not in custom_paths:
+                custom_paths.insert(0, resolved)
+                logger.debug(f"Auto-detected schema directory: {auto_folder}")
+
+    for custom_dir in custom_paths:
+        for pattern in [f"{base_name}.yaml", f"{base_name}.yml", f"agents/{base_name}.yaml"]:
+            custom_path = Path(custom_dir) / pattern
+            if custom_path.exists():
+                with open(custom_path, "r") as f:
+                    schema = yaml.safe_load(f)
+                return cast(dict[str, Any], schema)
+
+    # Try package resources
+    for search_pattern in SCHEMA_SEARCH_PATHS:
+        search_path = search_pattern.format(name=base_name)
+        try:
+            schema_ref = importlib.resources.files("rem") / search_path
+            schema_path = Path(str(schema_ref))
+            if schema_path.exists():
+                with open(schema_path, "r") as f:
+                    schema = yaml.safe_load(f)
+                _fs_schema_cache[cache_key] = schema
+                return cast(dict[str, Any], schema)
+        except Exception:
+            continue
+
+    # Try database lookup
+    if user_id:
+        from rem.services.postgres import get_postgres_service
+
+        should_disconnect = False
+        if db is None:
+            db = get_postgres_service()
+            if db:
+                await db.connect()
+                should_disconnect = True
+
+        if db:
+            try:
+                query = """
+                    SELECT spec FROM schemas
+                    WHERE LOWER(name) = LOWER($1)
+                    AND (user_id = $2 OR user_id = 'system' OR user_id IS NULL)
+                    LIMIT 1
+                """
+                row = await db.fetchrow(query, base_name, user_id)
+                if row:
+                    spec = row.get("spec")
+                    if spec and isinstance(spec, dict):
+                        logger.info(f"✅ Loaded schema from database: {base_name}")
+                        return spec
+            finally:
+                if should_disconnect:
+                    await db.disconnect()
+
+    # Not found
+    raise FileNotFoundError(f"Schema not found: {schema_name_or_path}")
+
+
 def validate_agent_schema(schema: dict[str, Any]) -> bool:
     """
     Validate agent schema structure.
@@ -334,3 +511,73 @@ def validate_agent_schema(schema: dict[str, Any]) -> bool:
 
     logger.debug("Schema validation passed")
     return True
+
+
+def get_evaluator_schema_path(evaluator_name: str) -> Path | None:
+    """
+    Find the file path to an evaluator schema.
+
+    Searches standard locations for the evaluator schema YAML file:
+    - ./evaluators/{name}.yaml (local project)
+    - Custom schema paths from registry
+    - Package resources: schemas/evaluators/{name}.yaml
+
+    Args:
+        evaluator_name: Name of the evaluator (e.g., "mental-health-classifier")
+
+    Returns:
+        Path to the evaluator schema file, or None if not found
+
+    Example:
+        >>> path = get_evaluator_schema_path("mental-health-classifier")
+        >>> if path:
+        ...     print(f"Found evaluator at: {path}")
+    """
+    from ..registry import get_schema_paths
+
+    base_name = evaluator_name.lower().replace('.yaml', '').replace('.yml', '')
+
+    # 1. Try custom schema paths (from registry + auto-detected)
+    custom_paths = get_schema_paths()
+
+    # Auto-detect local folders
+    auto_detect_folders = ["./evaluators", "./schemas", "./agents"]
+    for auto_folder in auto_detect_folders:
+        auto_path = Path(auto_folder)
+        if auto_path.exists() and auto_path.is_dir():
+            resolved = str(auto_path.resolve())
+            if resolved not in custom_paths:
+                custom_paths.insert(0, resolved)
+
+    for custom_dir in custom_paths:
+        # Try various patterns within each custom directory
+        for pattern in [
+            f"{base_name}.yaml",
+            f"{base_name}.yml",
+            f"evaluators/{base_name}.yaml",
+        ]:
+            custom_path = Path(custom_dir) / pattern
+            if custom_path.exists():
+                logger.debug(f"Found evaluator schema: {custom_path}")
+                return custom_path
+
+    # 2. Try package resources
+    evaluator_search_paths = [
+        f"schemas/evaluators/{base_name}.yaml",
+        f"schemas/evaluators/rem/{base_name}.yaml",
+    ]
+
+    for search_path in evaluator_search_paths:
+        try:
+            schema_ref = importlib.resources.files("rem") / search_path
+            schema_path = Path(str(schema_ref))
+
+            if schema_path.exists():
+                logger.debug(f"Found evaluator schema in package: {schema_path}")
+                return schema_path
+        except Exception as e:
+            logger.debug(f"Could not check {search_path}: {e}")
+            continue
+
+    logger.warning(f"Evaluator schema not found: {evaluator_name}")
+    return None