remdb 0.2.6__py3-none-any.whl → 0.3.118__py3-none-any.whl

rem/utils/sql_paths.py

@@ -0,0 +1,146 @@
+"""Utilities for resolving SQL file paths.
+
+Handles package SQL directory resolution and user migrations.
+
+Convention for user migrations:
+    Place custom SQL files in `./sql/migrations/` relative to your project root.
+    Files should be numbered (e.g., `100_custom_table.sql`) to control execution order.
+    Package migrations (001-099) run first, then user migrations (100+).
+"""
+
+from pathlib import Path
+from typing import List, Optional
+import importlib.resources
+
+# Convention: Default location for user-maintained migrations
+USER_SQL_DIR_CONVENTION = "sql"
+
+
+def get_package_sql_dir() -> Path:
+    """Get the SQL directory from the installed rem package.
+
+    Returns:
+        Path to the package's sql directory
+
+    Raises:
+        FileNotFoundError: If the SQL directory cannot be found
+    """
+    try:
+        # Use importlib.resources for Python 3.9+
+        sql_ref = importlib.resources.files("rem") / "sql"
+        package_sql = Path(str(sql_ref))
+        if package_sql.exists():
+            return package_sql
+    except (AttributeError, TypeError):
+        pass
+
+    # Fallback: use __file__ to find package location
+    try:
+        import rem
+        package_sql = Path(rem.__file__).parent / "sql"
+        if package_sql.exists():
+            return package_sql
+    except (ImportError, AttributeError):
+        pass
+
+    # Development fallback: check relative to cwd
+    dev_sql = Path("src/rem/sql")
+    if dev_sql.exists():
+        return dev_sql
+
+    raise FileNotFoundError(
+        "Could not locate rem SQL directory. "
+        "Ensure remdb is properly installed or run from the source directory."
+    )
+
+
+def get_package_migrations_dir() -> Path:
+    """Get the migrations directory from the installed rem package.
+
+    Returns:
+        Path to the package's migrations directory
+    """
+    return get_package_sql_dir() / "migrations"
+
+
+def get_user_sql_dir() -> Optional[Path]:
+    """Get the conventional user SQL directory if it exists.
+
+    Looks for `./sql/` relative to the current working directory.
+    This follows the convention for user-maintained migrations.
+
+    Returns:
+        Path to user sql directory if it exists, None otherwise
+    """
+    user_sql = Path.cwd() / USER_SQL_DIR_CONVENTION
+    if user_sql.exists() and user_sql.is_dir():
+        return user_sql
+    return None
+
+
+def list_package_migrations() -> List[Path]:
+    """List all migration files in the package.
+
+    Returns:
+        Sorted list of migration file paths
+    """
+    try:
+        migrations_dir = get_package_migrations_dir()
+        if migrations_dir.exists():
+            return sorted(
+                f for f in migrations_dir.glob("*.sql")
+                if f.name[0].isdigit()  # Only numbered migrations
+            )
+    except FileNotFoundError:
+        pass
+
+    return []
+
+
+def list_user_migrations() -> List[Path]:
+    """List all migration files in the user's sql/migrations directory.
+
+    Returns:
+        Sorted list of user migration file paths
+    """
+    user_sql = get_user_sql_dir()
+    if user_sql:
+        migrations_dir = user_sql / "migrations"
+        if migrations_dir.exists():
+            return sorted(
+                f for f in migrations_dir.glob("*.sql")
+                if f.name[0].isdigit()  # Only numbered migrations
+            )
+    return []
+
+
+def list_all_migrations() -> List[Path]:
+    """List all migration files from package and user directories.
+
+    Collects migrations from:
+    1. Package migrations directory
+    2. User directory (./sql/migrations/) if it exists
+
+    Files are sorted by name, so use numbered prefixes to control order:
+    - 001-099: Reserved for package migrations
+    - 100+: Recommended for user migrations
+
+    Returns:
+        Sorted list of all migration file paths (by filename)
+    """
+    all_migrations = []
+    seen_names = set()
+
+    # Package migrations first
+    for f in list_package_migrations():
+        if f.name not in seen_names:
+            all_migrations.append(f)
+            seen_names.add(f.name)
+
+    # User migrations second
+    for f in list_user_migrations():
+        if f.name not in seen_names:
+            all_migrations.append(f)
+            seen_names.add(f.name)
+
+    return sorted(all_migrations, key=lambda p: p.name)

rem/utils/sql_types.py

@@ -16,6 +16,7 @@ Best Practices:
 - UUID for identifiers in Union types
 """
 
+import types
 from datetime import date, datetime, time
 from typing import Any, Union, get_args, get_origin
 from uuid import UUID
@@ -78,8 +79,9 @@ def get_sql_type(field_info: FieldInfo, field_name: str) -> str:
         return "TEXT"
 
     # Handle Union types (including Optional[T] which is Union[T, None])
+    # Also handles Python 3.10+ `X | None` syntax which uses types.UnionType
     origin = get_origin(annotation)
-    if origin is Union:
+    if origin is Union or isinstance(annotation, types.UnionType):
         args = get_args(annotation)
         # Filter out NoneType
         non_none_args = [arg for arg in args if arg is not type(None)]

rem/utils/vision.py

@@ -11,7 +11,6 @@ markdown descriptions of images.
 """
 
 import base64
-import os
 from enum import Enum
 from pathlib import Path
 from typing import Optional
@@ -19,6 +18,9 @@ from typing import Optional
 import requests
 from loguru import logger
 
+from rem.utils.constants import HTTP_TIMEOUT_LONG, VISION_MAX_TOKENS
+from rem.utils.mime_types import EXTENSION_TO_MIME
+
 
 class VisionProvider(str, Enum):
     """Supported vision providers."""
@@ -141,14 +143,7 @@ class ImageAnalyzer:
 
         # Detect media type
         suffix = image_path.suffix.lower()
-        media_type_map = {
-            ".png": "image/png",
-            ".jpg": "image/jpeg",
-            ".jpeg": "image/jpeg",
-            ".gif": "image/gif",
-            ".webp": "image/webp",
-        }
-        media_type = media_type_map.get(suffix, "image/png")
+        media_type = EXTENSION_TO_MIME.get(suffix, "image/png")
 
         logger.info(f"Analyzing {image_path.name} with {self.provider.value} ({self.model})")
 
@@ -190,7 +185,7 @@ class ImageAnalyzer:
 
         body = {
             "model": self.model,
-            "max_tokens": 2048,
+            "max_tokens": VISION_MAX_TOKENS,
             "messages": [
                 {
                     "role": "user",
@@ -216,7 +211,7 @@ class ImageAnalyzer:
             "https://api.anthropic.com/v1/messages",
             headers=headers,
             json=body,
-            timeout=60.0,
+            timeout=HTTP_TIMEOUT_LONG,
         )
 
         if response.status_code != 200:
@@ -261,7 +256,7 @@ class ImageAnalyzer:
             url,
             params=params,
             json=body,
-            timeout=60.0,
+            timeout=HTTP_TIMEOUT_LONG,
         )
 
         if response.status_code != 200:
@@ -311,14 +306,14 @@ class ImageAnalyzer:
                     ],
                 }
             ],
-            "max_tokens": 2048,
+            "max_tokens": VISION_MAX_TOKENS,
         }
 
         response = requests.post(
             url,
             headers=headers,
             json=body,
-            timeout=60.0,
+            timeout=HTTP_TIMEOUT_LONG,
         )
 
         if response.status_code != 200:

rem/workers/README.md

@@ -207,7 +207,7 @@ Reads recent activity to generate comprehensive user profiles.
 
 **CLI:**
 ```bash
-rem-dreaming user-model --tenant-id=tenant-123
+rem-dreaming user-model 
 ```
 
 **Frequency:** Daily (runs as part of full workflow)
@@ -235,13 +235,13 @@ Extracts temporal narratives from resources.
 **CLI:**
 ```bash
 # Process last 24 hours
-rem-dreaming moments --tenant-id=tenant-123
+rem-dreaming moments 
 
 # Custom lookback
-rem-dreaming moments --tenant-id=tenant-123 --lookback-hours=48
+rem-dreaming moments  --lookback-hours=48
 
 # Limit resources processed
-rem-dreaming moments --tenant-id=tenant-123 --limit=100
+rem-dreaming moments  --limit=100
 ```
 
 **Frequency:** Daily or on-demand
@@ -283,13 +283,13 @@ Builds semantic relationships between resources.
 **CLI:**
 ```bash
 # Semantic mode (fast, cheap)
-rem-dreaming affinity --tenant-id=tenant-123
+rem-dreaming affinity 
 
 # LLM mode (intelligent, expensive)
-rem-dreaming affinity --tenant-id=tenant-123 --use-llm --limit=100
+rem-dreaming affinity  --use-llm --limit=100
 
 # Custom lookback
-rem-dreaming affinity --tenant-id=tenant-123 --lookback-hours=168
+rem-dreaming affinity  --lookback-hours=168
 ```
 
 **Frequency:**
@@ -308,13 +308,13 @@ Runs all operations in sequence.
 **CLI:**
 ```bash
 # Single tenant
-rem-dreaming full --tenant-id=tenant-123
+rem-dreaming full 
 
 # All active tenants (daily cron)
 rem-dreaming full --all-tenants
 
 # Use LLM affinity mode
-rem-dreaming full --tenant-id=tenant-123 --use-llm-affinity
+rem-dreaming full  --use-llm-affinity
 ```
 
 **Frequency:** Daily at 3 AM UTC
@@ -455,16 +455,16 @@ export REM_API_URL=http://localhost:8000
 export OPENAI_API_KEY=sk-...
 
 # Run user model update
-python -m rem.cli.dreaming user-model --tenant-id=tenant-test
+python -m rem.cli.dreaming user-model 
 
 # Run moment construction
-python -m rem.cli.dreaming moments --tenant-id=tenant-test --lookback-hours=24
+python -m rem.cli.dreaming moments  --lookback-hours=24
 
 # Run affinity (semantic mode)
-python -m rem.cli.dreaming affinity --tenant-id=tenant-test
+python -m rem.cli.dreaming affinity 
 
 # Run full workflow
-python -m rem.cli.dreaming full --tenant-id=tenant-test
+python -m rem.cli.dreaming full 
 ```
 
 ### Testing with Docker
@@ -478,7 +478,7 @@ docker run --rm \
   -e REM_API_URL=http://host.docker.internal:8000 \
   -e OPENAI_API_KEY=$OPENAI_API_KEY \
   rem-stack:latest \
-  python -m rem.cli.dreaming full --tenant-id=tenant-test
+  python -m rem.cli.dreaming full 
 ```
 
 ## Architecture Decisions

rem/workers/__init__.py

@@ -1,5 +1,6 @@
 """Background workers for processing tasks."""
 
 from .sqs_file_processor import SQSFileProcessor
+from .unlogged_maintainer import UnloggedMaintainer
 
-__all__ = ["SQSFileProcessor"]
+__all__ = ["SQSFileProcessor", "UnloggedMaintainer"]

rem/workers/db_maintainer.py

@@ -0,0 +1,74 @@
+"""
+Database Maintainer Worker.
+
+Handles background maintenance tasks for PostgreSQL:
+1. Cleaning up expired rate limit counters (UNLOGGED table).
+2. Refreshing materialized views (if any).
+3. Vacuuming specific tables (if needed).
+
+Usage:
+    python -m rem.workers.db_maintainer
+    
+    # Or via docker-compose:
+    # command: python -m rem.workers.db_maintainer
+"""
+
+import asyncio
+import signal
+from loguru import logger
+
+from ..services.postgres.service import PostgresService
+from ..services.rate_limit import RateLimitService
+
+class DatabaseMaintainer:
+    def __init__(self):
+        self.running = False
+        self.db = PostgresService()
+        self.rate_limiter = RateLimitService(self.db)
+
+    async def start(self):
+        """Start maintenance loop."""
+        self.running = True
+        logger.info("Starting Database Maintainer Worker")
+        
+        await self.db.connect()
+        
+        try:
+            while self.running:
+                await self._run_maintenance_cycle()
+                # Sleep for 5 minutes
+                await asyncio.sleep(300)
+        finally:
+            await self.db.disconnect()
+
+    async def _run_maintenance_cycle(self):
+        """Execute maintenance tasks."""
+        logger.debug("Running maintenance cycle...")
+        
+        try:
+            # 1. Cleanup Rate Limits
+            await self.rate_limiter.cleanup_expired()
+            
+            # 2. (Future) Refresh Views
+            # await self.db.execute("REFRESH MATERIALIZED VIEW ...")
+            
+        except Exception as e:
+            logger.error(f"Maintenance cycle failed: {e}")
+
+    def stop(self):
+        """Stop worker gracefully."""
+        self.running = False
+        logger.info("Stopping Database Maintainer Worker...")
+
+async def main():
+    worker = DatabaseMaintainer()
+    
+    # Handle signals
+    loop = asyncio.get_running_loop()
+    for sig in (signal.SIGTERM, signal.SIGINT):
+        loop.add_signal_handler(sig, worker.stop)
+        
+    await worker.start()
+
+if __name__ == "__main__":
+    asyncio.run(main())