remdb 0.3.114__py3-none-any.whl → 0.3.172__py3-none-any.whl

rem/utils/README.md

@@ -4,6 +4,7 @@
 
 1. [SQL Types](#sql-types-sql_typespy) - Pydantic to PostgreSQL type mapping
 2. [Embeddings](#embeddings-embeddingspy) - Vector embeddings generation
+3. [Files](#files-filespy) - File utilities and DataFrame I/O
 
 ## SQL Types (`sql_types.py`)
 
@@ -581,3 +582,47 @@ This will demonstrate:
 - `sql_types.py` - Use `embedding_provider` in json_schema_extra for TEXT fields
 - OpenAI Embeddings API: https://platform.openai.com/docs/api-reference/embeddings
 - pgvector Documentation: https://github.com/pgvector/pgvector
+
+---
+
+## Files (`files.py`)
+
+File utilities including temporary file handling and DataFrame I/O with automatic format detection.
+
+### DataFrame I/O
+
+Read and write DataFrames with format auto-detected from file extension:
+
+```python
+from rem.utils.files import read_dataframe, write_dataframe
+
+# Read - format inferred from extension
+df = read_dataframe("data.csv")
+df = read_dataframe("data.parquet")
+df = read_dataframe("data.xlsx")
+
+# Read from bytes (e.g., from S3)
+df = read_dataframe(content_bytes, filename="data.csv")
+
+# Write - format inferred from extension
+write_dataframe(df, "output.parquet")
+```
+
+**Supported formats**: `.csv`, `.tsv`, `.parquet`, `.json`, `.jsonl`, `.avro`, `.xlsx`, `.xls`, `.ods`, `.ipc`, `.arrow`, `.feather`
+
+Note: Some formats require optional dependencies (e.g., `fastexcel` for Excel).
+
+### Temporary File Utilities
+
+```python
+from rem.utils.files import temp_file_from_bytes, temp_directory
+
+# Create temp file from bytes, auto-cleanup
+with temp_file_from_bytes(pdf_bytes, suffix=".pdf") as tmp_path:
+    result = process_pdf(tmp_path)
+
+# Create temp directory, auto-cleanup
+with temp_directory() as tmp_dir:
+    # Work with files in tmp_dir
+    pass
+```

rem/utils/__init__.py

@@ -5,6 +5,7 @@ Utility functions and helpers for the REM system:
 - sql_types: Pydantic to PostgreSQL type mapping
 - embeddings: Vector embeddings generation using requests library
 - user_id: Deterministic UUID generation from email addresses
+- sql_paths: SQL file path resolution for packages and user migrations
 """
 
 from .embeddings import (
@@ -24,6 +25,15 @@ from .user_id import (
     is_valid_uuid,
     user_id_to_uuid,
 )
+from .sql_paths import (
+    USER_SQL_DIR_CONVENTION,
+    get_package_sql_dir,
+    get_package_migrations_dir,
+    get_user_sql_dir,
+    list_package_migrations,
+    list_user_migrations,
+    list_all_migrations,
+)
 
 __all__ = [
     # SQL Types
@@ -40,4 +50,12 @@ __all__ = [
     "email_to_user_id",
     "user_id_to_uuid",
     "is_valid_uuid",
+    # SQL Paths
+    "USER_SQL_DIR_CONVENTION",
+    "get_package_sql_dir",
+    "get_package_migrations_dir",
+    "get_user_sql_dir",
+    "list_package_migrations",
+    "list_user_migrations",
+    "list_all_migrations",
 ]

rem/utils/files.py

@@ -3,13 +3,18 @@ File utilities for consistent file handling throughout REM.
 
 Provides context managers and helpers for temporary file operations,
 ensuring proper cleanup and consistent patterns.
+
+Also provides DataFrame I/O utilities using Polars with automatic
+format detection based on file extension.
 """
 
 import tempfile
 from contextlib import contextmanager
+from io import BytesIO
 from pathlib import Path
-from typing import Generator, Optional
+from typing import Generator, Optional, Union
 
+import polars as pl
 from loguru import logger
 
 
@@ -165,3 +170,154 @@ def safe_delete(path: Path) -> bool:
     except Exception as e:
         logger.warning(f"Failed to delete {path}: {e}")
         return False
+
+
+# Extension to Polars reader mapping
+_EXTENSION_READERS = {
+    ".csv": pl.read_csv,
+    ".tsv": lambda p, **kw: pl.read_csv(p, separator="\t", **kw),
+    ".parquet": pl.read_parquet,
+    ".pq": pl.read_parquet,
+    ".json": pl.read_json,
+    ".jsonl": pl.read_ndjson,
+    ".ndjson": pl.read_ndjson,
+    ".avro": pl.read_avro,
+    ".xlsx": pl.read_excel,
+    ".xls": pl.read_excel,
+    ".ods": pl.read_ods,
+    ".ipc": pl.read_ipc,
+    ".arrow": pl.read_ipc,
+    ".feather": pl.read_ipc,
+}
+
+# Extension to Polars writer mapping
+_EXTENSION_WRITERS = {
+    ".csv": "write_csv",
+    ".tsv": "write_csv",  # with separator="\t"
+    ".parquet": "write_parquet",
+    ".pq": "write_parquet",
+    ".json": "write_json",
+    ".jsonl": "write_ndjson",
+    ".ndjson": "write_ndjson",
+    ".avro": "write_avro",
+    ".xlsx": "write_excel",
+    ".ipc": "write_ipc",
+    ".arrow": "write_ipc",
+    ".feather": "write_ipc",
+}
+
+
+def read_dataframe(
+    source: Union[str, Path, bytes],
+    filename: Optional[str] = None,
+    **kwargs,
+) -> pl.DataFrame:
+    """
+    Read a DataFrame from a file, inferring format from extension.
+
+    Supports all Polars-compatible formats:
+    - CSV (.csv), TSV (.tsv)
+    - Parquet (.parquet, .pq)
+    - JSON (.json), JSONL/NDJSON (.jsonl, .ndjson)
+    - Avro (.avro)
+    - Excel (.xlsx, .xls)
+    - OpenDocument (.ods)
+    - Arrow IPC (.ipc, .arrow, .feather)
+
+    Args:
+        source: File path (str/Path) or bytes content
+        filename: Required when source is bytes, to determine format
+        **kwargs: Additional arguments passed to the Polars reader
+
+    Returns:
+        Polars DataFrame
+
+    Raises:
+        ValueError: If format cannot be determined or is unsupported
+
+    Examples:
+        >>> df = read_dataframe("data.csv")
+        >>> df = read_dataframe("data.parquet")
+        >>> df = read_dataframe(csv_bytes, filename="data.csv")
+    """
+    # Determine the file extension
+    if isinstance(source, bytes):
+        if not filename:
+            raise ValueError("filename is required when source is bytes")
+        ext = Path(filename).suffix.lower()
+        # For bytes, we need to wrap in BytesIO
+        file_like = BytesIO(source)
+    else:
+        path = Path(source)
+        ext = path.suffix.lower()
+        file_like = path
+
+    # Get the appropriate reader
+    reader = _EXTENSION_READERS.get(ext)
+    if reader is None:
+        supported = ", ".join(sorted(_EXTENSION_READERS.keys()))
+        raise ValueError(
+            f"Unsupported file format: {ext}. "
+            f"Supported formats: {supported}"
+        )
+
+    try:
+        return reader(file_like, **kwargs)
+    except Exception as e:
+        logger.error(f"Failed to read DataFrame from {ext} format: {e}")
+        raise
+
+
+def write_dataframe(
+    df: pl.DataFrame,
+    dest: Union[str, Path],
+    **kwargs,
+) -> None:
+    """
+    Write a DataFrame to a file, inferring format from extension.
+
+    Supports most Polars-writable formats:
+    - CSV (.csv), TSV (.tsv)
+    - Parquet (.parquet, .pq)
+    - JSON (.json), JSONL/NDJSON (.jsonl, .ndjson)
+    - Avro (.avro)
+    - Excel (.xlsx)
+    - Arrow IPC (.ipc, .arrow, .feather)
+
+    Args:
+        df: Polars DataFrame to write
+        dest: Destination file path
+        **kwargs: Additional arguments passed to the Polars writer
+
+    Raises:
+        ValueError: If format cannot be determined or is unsupported
+
+    Examples:
+        >>> write_dataframe(df, "output.csv")
+        >>> write_dataframe(df, "output.parquet")
+        >>> write_dataframe(df, "output.jsonl")
+    """
+    path = Path(dest)
+    ext = path.suffix.lower()
+
+    writer_method = _EXTENSION_WRITERS.get(ext)
+    if writer_method is None:
+        supported = ", ".join(sorted(_EXTENSION_WRITERS.keys()))
+        raise ValueError(
+            f"Unsupported file format for writing: {ext}. "
+            f"Supported formats: {supported}"
+        )
+
+    # Ensure parent directory exists
+    ensure_parent_exists(path)
+
+    # Handle TSV special case
+    if ext == ".tsv":
+        kwargs.setdefault("separator", "\t")
+
+    try:
+        writer = getattr(df, writer_method)
+        writer(path, **kwargs)
+    except Exception as e:
+        logger.error(f"Failed to write DataFrame to {ext} format: {e}")
+        raise

rem/utils/schema_loader.py

@@ -132,13 +132,51 @@ def _load_schema_from_database(schema_name: str, user_id: str) -> dict[str, Any]
     # Check if we're already in an async context
     try:
         loop = asyncio.get_running_loop()
-        # We're in an async context - can't use asyncio.run()
-        # This shouldn't happen in normal usage since load_agent_schema is called from sync contexts
-        logger.warning(
-            "Database schema lookup called from async context. "
-            "This may cause issues. Consider using async version of load_agent_schema."
-        )
-        return None
+        # We're in an async context - use thread executor to run async code
+        import concurrent.futures
+
+        async def _async_lookup():
+            """Async helper to query database."""
+            from rem.services.postgres import get_postgres_service
+
+            db = get_postgres_service()
+            if not db:
+                logger.debug("PostgreSQL service not available for schema lookup")
+                return None
+
+            try:
+                await db.connect()
+
+                query = """
+                    SELECT spec FROM schemas
+                    WHERE LOWER(name) = LOWER($1)
+                    AND (user_id = $2 OR user_id = 'system' OR user_id IS NULL)
+                    LIMIT 1
+                """
+                logger.debug(f"Executing schema lookup: name={schema_name}, user_id={user_id}")
+
+                row = await db.fetchrow(query, schema_name, user_id)
+
+                if row:
+                    spec = row.get("spec")
+                    if spec and isinstance(spec, dict):
+                        logger.debug(f"Found schema in database: {schema_name}")
+                        return spec
+
+                logger.debug(f"Schema not found in database: {schema_name}")
+                return None
+
+            except Exception as e:
+                logger.debug(f"Database schema lookup error: {e}")
+                return None
+            finally:
+                await db.disconnect()
+
+        # Run in thread pool to avoid blocking the event loop
+        with concurrent.futures.ThreadPoolExecutor() as pool:
+            future = pool.submit(asyncio.run, _async_lookup())
+            return future.result(timeout=10)
+
     except RuntimeError:
         # Not in async context - safe to use asyncio.run()
         pass
@@ -195,7 +233,7 @@ def load_agent_schema(
     """
     Load agent schema from YAML file with unified search logic and caching.
 
-    Schema names are case-invariant - "Siggy", "siggy", "SIGGY" all resolve to the same schema.
+    Schema names are case-invariant - "Rem", "rem", "REM" all resolve to the same schema.
 
     Filesystem schemas are cached indefinitely (immutable, versioned with code).
     Database schemas (future) will be cached with TTL for invalidation.
@@ -271,10 +309,20 @@ def load_agent_schema(
     # 2. Normalize name for package resource search (lowercase)
     base_name = cache_key
 
-    # 3. Try custom schema paths (from registry + SCHEMA__PATHS env var)
+    # 3. Try custom schema paths (from registry + SCHEMA__PATHS env var + auto-detected)
     from ..registry import get_schema_paths
 
     custom_paths = get_schema_paths()
+
+    # Auto-detect local folders if they exist (convention over configuration)
+    auto_detect_folders = ["./agents", "./schemas", "./evaluators"]
+    for auto_folder in auto_detect_folders:
+        auto_path = Path(auto_folder)
+        if auto_path.exists() and auto_path.is_dir():
+            resolved = str(auto_path.resolve())
+            if resolved not in custom_paths:
+                custom_paths.insert(0, resolved)
+                logger.debug(f"Auto-detected schema directory: {auto_folder}")
     for custom_dir in custom_paths:
         # Try various patterns within each custom directory
         for pattern in [
@@ -400,9 +448,20 @@ async def load_agent_schema_async(
 
     base_name = cache_key
 
-    # Try custom schema paths
+    # Try custom schema paths (from registry + SCHEMA__PATHS env var + auto-detected)
     from ..registry import get_schema_paths
     custom_paths = get_schema_paths()
+
+    # Auto-detect local folders if they exist (convention over configuration)
+    auto_detect_folders = ["./agents", "./schemas", "./evaluators"]
+    for auto_folder in auto_detect_folders:
+        auto_path = Path(auto_folder)
+        if auto_path.exists() and auto_path.is_dir():
+            resolved = str(auto_path.resolve())
+            if resolved not in custom_paths:
+                custom_paths.insert(0, resolved)
+                logger.debug(f"Auto-detected schema directory: {auto_folder}")
+
     for custom_dir in custom_paths:
         for pattern in [f"{base_name}.yaml", f"{base_name}.yml", f"agents/{base_name}.yaml"]:
             custom_path = Path(custom_dir) / pattern
@@ -490,3 +549,73 @@ def validate_agent_schema(schema: dict[str, Any]) -> bool:
 
     logger.debug("Schema validation passed")
     return True
+
+
+def get_evaluator_schema_path(evaluator_name: str) -> Path | None:
+    """
+    Find the file path to an evaluator schema.
+
+    Searches standard locations for the evaluator schema YAML file:
+    - ./evaluators/{name}.yaml (local project)
+    - Custom schema paths from registry
+    - Package resources: schemas/evaluators/{name}.yaml
+
+    Args:
+        evaluator_name: Name of the evaluator (e.g., "mental-health-classifier")
+
+    Returns:
+        Path to the evaluator schema file, or None if not found
+
+    Example:
+        >>> path = get_evaluator_schema_path("mental-health-classifier")
+        >>> if path:
+        ...     print(f"Found evaluator at: {path}")
+    """
+    from ..registry import get_schema_paths
+
+    base_name = evaluator_name.lower().replace('.yaml', '').replace('.yml', '')
+
+    # 1. Try custom schema paths (from registry + auto-detected)
+    custom_paths = get_schema_paths()
+
+    # Auto-detect local folders
+    auto_detect_folders = ["./evaluators", "./schemas", "./agents"]
+    for auto_folder in auto_detect_folders:
+        auto_path = Path(auto_folder)
+        if auto_path.exists() and auto_path.is_dir():
+            resolved = str(auto_path.resolve())
+            if resolved not in custom_paths:
+                custom_paths.insert(0, resolved)
+
+    for custom_dir in custom_paths:
+        # Try various patterns within each custom directory
+        for pattern in [
+            f"{base_name}.yaml",
+            f"{base_name}.yml",
+            f"evaluators/{base_name}.yaml",
+        ]:
+            custom_path = Path(custom_dir) / pattern
+            if custom_path.exists():
+                logger.debug(f"Found evaluator schema: {custom_path}")
+                return custom_path
+
+    # 2. Try package resources
+    evaluator_search_paths = [
+        f"schemas/evaluators/{base_name}.yaml",
+        f"schemas/evaluators/rem/{base_name}.yaml",
+    ]
+
+    for search_path in evaluator_search_paths:
+        try:
+            schema_ref = importlib.resources.files("rem") / search_path
+            schema_path = Path(str(schema_ref))
+
+            if schema_path.exists():
+                logger.debug(f"Found evaluator schema in package: {schema_path}")
+                return schema_path
+        except Exception as e:
+            logger.debug(f"Could not check {search_path}: {e}")
+            continue
+
+    logger.warning(f"Evaluator schema not found: {evaluator_name}")
+    return None

rem/utils/sql_paths.py

@@ -0,0 +1,146 @@
+"""Utilities for resolving SQL file paths.
+
+Handles package SQL directory resolution and user migrations.
+
+Convention for user migrations:
+    Place custom SQL files in `./sql/migrations/` relative to your project root.
+    Files should be numbered (e.g., `100_custom_table.sql`) to control execution order.
+    Package migrations (001-099) run first, then user migrations (100+).
+"""
+
+from pathlib import Path
+from typing import List, Optional
+import importlib.resources
+
+# Convention: Default location for user-maintained migrations
+USER_SQL_DIR_CONVENTION = "sql"
+
+
+def get_package_sql_dir() -> Path:
+    """Get the SQL directory from the installed rem package.
+
+    Returns:
+        Path to the package's sql directory
+
+    Raises:
+        FileNotFoundError: If the SQL directory cannot be found
+    """
+    try:
+        # Use importlib.resources for Python 3.9+
+        sql_ref = importlib.resources.files("rem") / "sql"
+        package_sql = Path(str(sql_ref))
+        if package_sql.exists():
+            return package_sql
+    except (AttributeError, TypeError):
+        pass
+
+    # Fallback: use __file__ to find package location
+    try:
+        import rem
+        package_sql = Path(rem.__file__).parent / "sql"
+        if package_sql.exists():
+            return package_sql
+    except (ImportError, AttributeError):
+        pass
+
+    # Development fallback: check relative to cwd
+    dev_sql = Path("src/rem/sql")
+    if dev_sql.exists():
+        return dev_sql
+
+    raise FileNotFoundError(
+        "Could not locate rem SQL directory. "
+        "Ensure remdb is properly installed or run from the source directory."
+    )
+
+
+def get_package_migrations_dir() -> Path:
+    """Get the migrations directory from the installed rem package.
+
+    Returns:
+        Path to the package's migrations directory
+    """
+    return get_package_sql_dir() / "migrations"
+
+
+def get_user_sql_dir() -> Optional[Path]:
+    """Get the conventional user SQL directory if it exists.
+
+    Looks for `./sql/` relative to the current working directory.
+    This follows the convention for user-maintained migrations.
+
+    Returns:
+        Path to user sql directory if it exists, None otherwise
+    """
+    user_sql = Path.cwd() / USER_SQL_DIR_CONVENTION
+    if user_sql.exists() and user_sql.is_dir():
+        return user_sql
+    return None
+
+
+def list_package_migrations() -> List[Path]:
+    """List all migration files in the package.
+
+    Returns:
+        Sorted list of migration file paths
+    """
+    try:
+        migrations_dir = get_package_migrations_dir()
+        if migrations_dir.exists():
+            return sorted(
+                f for f in migrations_dir.glob("*.sql")
+                if f.name[0].isdigit()  # Only numbered migrations
+            )
+    except FileNotFoundError:
+        pass
+
+    return []
+
+
+def list_user_migrations() -> List[Path]:
+    """List all migration files in the user's sql/migrations directory.
+
+    Returns:
+        Sorted list of user migration file paths
+    """
+    user_sql = get_user_sql_dir()
+    if user_sql:
+        migrations_dir = user_sql / "migrations"
+        if migrations_dir.exists():
+            return sorted(
+                f for f in migrations_dir.glob("*.sql")
+                if f.name[0].isdigit()  # Only numbered migrations
+            )
+    return []
+
+
+def list_all_migrations() -> List[Path]:
+    """List all migration files from package and user directories.
+
+    Collects migrations from:
+    1. Package migrations directory
+    2. User directory (./sql/migrations/) if it exists
+
+    Files are sorted by name, so use numbered prefixes to control order:
+    - 001-099: Reserved for package migrations
+    - 100+: Recommended for user migrations
+
+    Returns:
+        Sorted list of all migration file paths (by filename)
+    """
+    all_migrations = []
+    seen_names = set()
+
+    # Package migrations first
+    for f in list_package_migrations():
+        if f.name not in seen_names:
+            all_migrations.append(f)
+            seen_names.add(f.name)
+
+    # User migrations second
+    for f in list_user_migrations():
+        if f.name not in seen_names:
+            all_migrations.append(f)
+            seen_names.add(f.name)
+
+    return sorted(all_migrations, key=lambda p: p.name)

rem/utils/vision.py

@@ -106,7 +106,7 @@ class ImageAnalyzer:
             elif provider == VisionProvider.GEMINI:
                 model = "gemini-2.0-flash-exp"
             elif provider == VisionProvider.OPENAI:
-                model = "gpt-4o"
+                model = "gpt-4.1"
 
         self.model = model
         self.base_url = base_url

rem/workers/__init__.py

@@ -1,5 +1,7 @@
 """Background workers for processing tasks."""
 
+from .db_listener import DBListener
 from .sqs_file_processor import SQSFileProcessor
+from .unlogged_maintainer import UnloggedMaintainer
 
-__all__ = ["SQSFileProcessor"]
+__all__ = ["DBListener", "SQSFileProcessor", "UnloggedMaintainer"]