remdb 0.3.7__py3-none-any.whl → 0.3.133__py3-none-any.whl

rem/settings.py

@@ -15,7 +15,7 @@ Example .env file:
     API__LOG_LEVEL=info
 
     # LLM
-    LLM__DEFAULT_MODEL=anthropic:claude-sonnet-4-5-20250929
+    LLM__DEFAULT_MODEL=openai:gpt-4.1
     LLM__DEFAULT_TEMPERATURE=0.5
     LLM__MAX_RETRIES=10
     LLM__OPENAI_API_KEY=sk-...
@@ -33,14 +33,15 @@ Example .env file:
     AUTH__OIDC_CLIENT_ID=your-client-id
     AUTH__SESSION_SECRET=your-secret-key
 
-    # OpenTelemetry (disabled by default)
+    # OpenTelemetry (disabled by default - enable via env var when collector available)
+    # Standard OTLP collector ports: 4317 (gRPC), 4318 (HTTP)
     OTEL__ENABLED=false
     OTEL__SERVICE_NAME=rem-api
-    OTEL__COLLECTOR_ENDPOINT=http://localhost:4318
-    OTEL__PROTOCOL=http
+    OTEL__COLLECTOR_ENDPOINT=http://localhost:4317
+    OTEL__PROTOCOL=grpc
 
-    # Arize Phoenix (disabled by default)
-    PHOENIX__ENABLED=false
+    # Arize Phoenix (enabled by default - can be disabled via env var)
+    PHOENIX__ENABLED=true
     PHOENIX__COLLECTOR_ENDPOINT=http://localhost:6006/v1/traces
     PHOENIX__PROJECT_NAME=rem
 
@@ -57,8 +58,10 @@ Example .env file:
 """
 
 import os
-from pydantic import Field, field_validator
+import hashlib
+from pydantic import Field, field_validator, ValidationInfo
 from pydantic_settings import BaseSettings, SettingsConfigDict
+from loguru import logger
 
 
 class LLMSettings(BaseSettings):
@@ -72,7 +75,7 @@ class LLMSettings(BaseSettings):
         LLM__EVALUATOR_MODEL or EVALUATOR_MODEL - Model for LLM-as-judge evaluation
         LLM__OPENAI_API_KEY or OPENAI_API_KEY - OpenAI API key
         LLM__ANTHROPIC_API_KEY or ANTHROPIC_API_KEY - Anthropic API key
-        LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai, cohere, jina, etc.)
+        LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai)
         LLM__EMBEDDING_MODEL or EMBEDDING_MODEL - Default embedding model name
     """
 
@@ -84,7 +87,7 @@ class LLMSettings(BaseSettings):
     )
 
     default_model: str = Field(
-        default="anthropic:claude-sonnet-4-5-20250929",
+        default="openai:gpt-4.1",
         description="Default LLM model (format: provider:model-id)",
     )
 
@@ -127,7 +130,7 @@ class LLMSettings(BaseSettings):
 
     embedding_provider: str = Field(
         default="openai",
-        description="Default embedding provider (openai, cohere, jina, etc.)",
+        description="Default embedding provider (currently only openai supported)",
     )
 
     embedding_model: str = Field(
@@ -239,6 +242,11 @@ class OTELSettings(BaseSettings):
         description="Export timeout in milliseconds",
     )
 
+    insecure: bool = Field(
+        default=True,
+        description="Use insecure (non-TLS) gRPC connection (default: True for local dev)",
+    )
+
 
 class PhoenixSettings(BaseSettings):
     """
@@ -265,8 +273,8 @@ class PhoenixSettings(BaseSettings):
     )
 
     enabled: bool = Field(
-        default=False,
-        description="Enable Phoenix integration (disabled by default for local dev)",
+        default=True,
+        description="Enable Phoenix integration (enabled by default)",
     )
 
     base_url: str = Field(
@@ -359,10 +367,16 @@ class AuthSettings(BaseSettings):
     - Custom OIDC provider
 
     Environment variables:
-        AUTH__ENABLED - Enable authentication (default: false)
+        AUTH__ENABLED - Enable authentication (default: true)
+        AUTH__ALLOW_ANONYMOUS - Allow rate-limited anonymous access (default: true)
         AUTH__SESSION_SECRET - Secret for session cookie signing
         AUTH__GOOGLE__* - Google OAuth settings
         AUTH__MICROSOFT__* - Microsoft OAuth settings
+
+    Access modes:
+    - enabled=true, allow_anonymous=true: Auth available, anonymous gets rate-limited access
+    - enabled=true, allow_anonymous=false: Auth required for all requests
+    - enabled=false: No auth, all requests treated as default user (dev mode)
     """
 
     model_config = SettingsConfigDict(
@@ -373,8 +387,26 @@ class AuthSettings(BaseSettings):
     )
 
     enabled: bool = Field(
-        default=False,
-        description="Enable authentication (disabled by default for testing)",
+        default=True,
+        description="Enable authentication (OAuth endpoints and middleware)",
+    )
+
+    allow_anonymous: bool = Field(
+        default=True,
+        description=(
+            "Allow anonymous (unauthenticated) access with rate limits. "
+            "When true, requests without auth get ANONYMOUS tier rate limits. "
+            "When false, all requests require authentication."
+        ),
+    )
+
+    mcp_requires_auth: bool = Field(
+        default=True,
+        description=(
+            "Require authentication for MCP endpoints. "
+            "MCP is a protected service and should always require login in production. "
+            "Set to false only for local development/testing."
+        ),
     )
 
     session_secret: str = Field(
@@ -386,6 +418,22 @@ class AuthSettings(BaseSettings):
     google: GoogleOAuthSettings = Field(default_factory=GoogleOAuthSettings)
     microsoft: MicrosoftOAuthSettings = Field(default_factory=MicrosoftOAuthSettings)
 
+    @field_validator("session_secret", mode="before")
+    @classmethod
+    def generate_dev_secret(cls, v: str | None, info: ValidationInfo) -> str:
+        # Only generate if not already set and not in production
+        if not v and info.data.get("environment") != "production":
+            # Deterministic secret for development
+            seed_string = f"{info.data.get('team', 'rem')}-{info.data.get('environment', 'development')}-auth-secret-salt"
+            logger.warning(
+                "AUTH__SESSION_SECRET not set. Generating deterministic secret for non-production environment. "
+                "DO NOT use in production."
+            )
+            return hashlib.sha256(seed_string.encode()).hexdigest()
+        elif not v and info.data.get("environment") == "production":
+            raise ValueError("AUTH__SESSION_SECRET must be set in production environment.")
+        return v
+
 
 class PostgresSettings(BaseSettings):
     """
@@ -644,6 +692,91 @@ class S3Settings(BaseSettings):
     )
 
 
+class DataLakeSettings(BaseSettings):
+    """
+    Data lake settings for experiment and dataset storage.
+
+    Data Lake Convention:
+        The data lake provides a standardized structure for storing datasets,
+        experiments, and calibration data in S3. Users bring their own bucket
+        and the version is pinned by default to v0 in the path.
+
+    S3 Path Structure:
+        s3://{bucket}/{version}/datasets/
+        ├── raw/                        # Raw source data + transformers
+        │   └── {dataset_name}/         # e.g., cns_drugs, codes, care
+        ├── tables/                     # Database table data (JSONL)
+        │   ├── resources/              # → resources table
+        │   │   ├── drugs/{category}/   # Psychotropic drugs
+        │   │   ├── care/stages/        # Treatment stages
+        │   │   └── crisis/             # Crisis resources
+        │   └── codes/                  # → codes table
+        │       ├── icd10/{category}/   # ICD-10 codes
+        │       └── cpt/                # CPT codes
+        └── calibration/                # Agent calibration
+            ├── experiments/            # Experiment configs + results
+            │   └── {agent}/{task}/     # e.g., siggy/risk-assessment
+            └── datasets/               # Shared evaluation datasets
+
+    Experiment Storage:
+        - Local: experiments/{agent}/{task}/experiment.yaml
+        - S3: s3://{bucket}/{version}/datasets/calibration/experiments/{agent}/{task}/
+
+    Environment variables:
+        DATA_LAKE__BUCKET_NAME - S3 bucket for data lake (required)
+        DATA_LAKE__VERSION - Path version prefix (default: v0)
+        DATA_LAKE__DATASETS_PREFIX - Datasets directory (default: datasets)
+        DATA_LAKE__EXPERIMENTS_PREFIX - Experiments subdirectory (default: experiments)
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="DATA_LAKE__",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    bucket_name: str | None = Field(
+        default=None,
+        description="S3 bucket for data lake storage (user-provided)",
+    )
+
+    version: str = Field(
+        default="v0",
+        description="API version for data lake paths",
+    )
+
+    datasets_prefix: str = Field(
+        default="datasets",
+        description="Root directory for datasets in the bucket",
+    )
+
+    experiments_prefix: str = Field(
+        default="experiments",
+        description="Subdirectory within calibration for experiments",
+    )
+
+    def get_base_uri(self) -> str | None:
+        """Get the base S3 URI for the data lake."""
+        if not self.bucket_name:
+            return None
+        return f"s3://{self.bucket_name}/{self.version}/{self.datasets_prefix}"
+
+    def get_experiment_uri(self, agent: str, task: str = "general") -> str | None:
+        """Get the S3 URI for an experiment."""
+        base = self.get_base_uri()
+        if not base:
+            return None
+        return f"{base}/calibration/{self.experiments_prefix}/{agent}/{task}"
+
+    def get_tables_uri(self, table: str = "resources") -> str | None:
+        """Get the S3 URI for a table directory."""
+        base = self.get_base_uri()
+        if not base:
+            return None
+        return f"{base}/tables/{table}"
+
+
 class ChunkingSettings(BaseSettings):
     """
     Document chunking settings for semantic text splitting.
@@ -962,6 +1095,123 @@ class APISettings(BaseSettings):
     )
 
 
+class ModelsSettings(BaseSettings):
+    """
+    Custom model registration settings for downstream applications.
+
+    Allows downstream apps to specify Python modules containing custom models
+    that should be imported (and thus registered) before schema generation.
+
+    This enables `rem db schema generate` to discover models registered with
+    `@rem.register_model` in downstream applications.
+
+    Environment variables:
+        MODELS__IMPORT_MODULES - Semicolon-separated list of Python modules to import
+                                 Example: "models;myapp.entities;myapp.custom_models"
+
+    Example:
+        # In downstream app's .env
+        MODELS__IMPORT_MODULES=models
+
+        # In downstream app's models/__init__.py
+        import rem
+        from rem.models.core import CoreModel
+
+        @rem.register_model
+        class MyCustomEntity(CoreModel):
+            name: str
+
+        # Then run schema generation
+        rem db schema generate  # Includes MyCustomEntity
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="MODELS__",
+        extra="ignore",
+    )
+
+    import_modules: str = Field(
+        default="",
+        description=(
+            "Semicolon-separated list of Python modules to import for model registration. "
+            "These modules are imported before schema generation to ensure custom models "
+            "decorated with @rem.register_model are discovered. "
+            "Example: 'models;myapp.entities'"
+        ),
+    )
+
+    @property
+    def module_list(self) -> list[str]:
+        """
+        Get modules as a list, filtering empty strings.
+
+        Auto-detects ./models folder if it exists and is importable.
+        """
+        modules = []
+        if self.import_modules:
+            modules = [m.strip() for m in self.import_modules.split(";") if m.strip()]
+
+        # Auto-detect ./models if it exists and is a Python package (convention over configuration)
+        from pathlib import Path
+
+        models_path = Path("./models")
+        if models_path.exists() and models_path.is_dir():
+            # Check if it's a Python package (has __init__.py)
+            if (models_path / "__init__.py").exists():
+                if "models" not in modules:
+                    modules.insert(0, "models")
+
+        return modules
+
+
+class SchemaSettings(BaseSettings):
+    """
+    Schema search path settings for agent and evaluator schemas.
+
+    Allows extending REM's schema search with custom directories.
+    Custom paths are searched BEFORE built-in package schemas.
+
+    Environment variables:
+        SCHEMA__PATHS - Semicolon-separated list of directories to search
+                        Example: "/app/schemas;/shared/agents;./local-schemas"
+
+    Search Order:
+    1. Exact path (if file exists)
+    2. Custom paths from SCHEMA__PATHS (in order)
+    3. Built-in package schemas (schemas/agents/, schemas/evaluators/, etc.)
+    4. Database LOOKUP (if enabled)
+
+    Example:
+        # In .env or environment
+        SCHEMA__PATHS=/app/custom-agents;/shared/evaluators
+
+        # Then in code
+        from rem.utils.schema_loader import load_agent_schema
+        schema = load_agent_schema("my-custom-agent")  # Found in /app/custom-agents/
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="SCHEMA__",
+        extra="ignore",
+    )
+
+    paths: str = Field(
+        default="",
+        description=(
+            "Semicolon-separated list of directories to search for schemas. "
+            "These paths are searched BEFORE built-in package schemas. "
+            "Example: '/app/schemas;/shared/agents'"
+        ),
+    )
+
+    @property
+    def path_list(self) -> list[str]:
+        """Get paths as a list, filtering empty strings."""
+        if not self.paths:
+            return []
+        return [p.strip() for p in self.paths.split(";") if p.strip()]
+
+
 class GitSettings(BaseSettings):
     """
     Git repository provider settings for versioned schema/experiment syncing.
@@ -1097,6 +1347,110 @@ class GitSettings(BaseSettings):
     )
 
 
+class DBListenerSettings(BaseSettings):
+    """
+    PostgreSQL LISTEN/NOTIFY database listener settings.
+
+    The DB Listener is a lightweight worker that subscribes to PostgreSQL
+    NOTIFY events and dispatches them to external systems (SQS, REST, custom).
+
+    Architecture:
+        - Single-replica deployment (to avoid duplicate processing)
+        - Dedicated connection for LISTEN (not from connection pool)
+        - Automatic reconnection with exponential backoff
+        - Graceful shutdown on SIGTERM
+
+    Use Cases:
+        - Sync data changes to external systems (Phoenix, webhooks)
+        - Trigger async jobs without polling
+        - Event-driven architectures with PostgreSQL as event source
+
+    Example PostgreSQL trigger:
+        CREATE OR REPLACE FUNCTION notify_feedback_insert()
+        RETURNS TRIGGER AS $$
+        BEGIN
+            PERFORM pg_notify('feedback_sync', json_build_object(
+                'id', NEW.id,
+                'table', 'feedbacks',
+                'action', 'insert'
+            )::text);
+            RETURN NEW;
+        END;
+        $$ LANGUAGE plpgsql;
+
+    Environment variables:
+        DB_LISTENER__ENABLED - Enable the listener worker (default: false)
+        DB_LISTENER__CHANNELS - Comma-separated PostgreSQL channels to listen on
+        DB_LISTENER__HANDLER_TYPE - Handler type: 'sqs', 'rest', or 'custom'
+        DB_LISTENER__SQS_QUEUE_URL - SQS queue URL (for handler_type=sqs)
+        DB_LISTENER__REST_ENDPOINT - REST endpoint URL (for handler_type=rest)
+        DB_LISTENER__RECONNECT_DELAY - Initial reconnect delay in seconds
+        DB_LISTENER__MAX_RECONNECT_DELAY - Maximum reconnect delay in seconds
+
+    References:
+        - PostgreSQL NOTIFY: https://www.postgresql.org/docs/current/sql-notify.html
+        - Brandur's Notifier: https://brandur.org/notifier
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="DB_LISTENER__",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    enabled: bool = Field(
+        default=False,
+        description="Enable the DB Listener worker (disabled by default)",
+    )
+
+    channels: str = Field(
+        default="",
+        description=(
+            "Comma-separated list of PostgreSQL channels to LISTEN on. "
+            "Example: 'feedback_sync,entity_update,user_events'"
+        ),
+    )
+
+    handler_type: str = Field(
+        default="rest",
+        description=(
+            "Handler type for dispatching notifications. Options: "
+            "'sqs' (publish to SQS), 'rest' (POST to endpoint), 'custom' (Python handlers)"
+        ),
+    )
+
+    sqs_queue_url: str = Field(
+        default="",
+        description="SQS queue URL for handler_type='sqs'",
+    )
+
+    rest_endpoint: str = Field(
+        default="http://localhost:8000/api/v1/internal/events",
+        description=(
+            "REST endpoint URL for handler_type='rest'. "
+            "Receives POST with {channel, payload, source} JSON body."
+        ),
+    )
+
+    reconnect_delay: float = Field(
+        default=1.0,
+        description="Initial delay (seconds) between reconnection attempts",
+    )
+
+    max_reconnect_delay: float = Field(
+        default=60.0,
+        description="Maximum delay (seconds) between reconnection attempts (exponential backoff cap)",
+    )
+
+    @property
+    def channel_list(self) -> list[str]:
+        """Get channels as a list, filtering empty strings."""
+        if not self.channels:
+            return []
+        return [c.strip() for c in self.channels.split(",") if c.strip()]
+
+
 class TestSettings(BaseSettings):
     """
     Test environment settings.
@@ -1166,6 +1520,11 @@ class Settings(BaseSettings):
         extra="ignore",
     )
 
+    app_name: str = Field(
+        default="REM",
+        description="Application/API name used in docs, titles, and user-facing text",
+    )
+
     team: str = Field(
         default="rem",
         description="Team or project name for observability",
@@ -1186,16 +1545,12 @@ class Settings(BaseSettings):
         description="Root path for reverse proxy (e.g., /rem for ALB routing)",
     )
 
-    sql_dir: str = Field(
-        default="src/rem/sql",
-        description="Directory for SQL files and migrations",
-    )
-
     # Nested settings groups
     api: APISettings = Field(default_factory=APISettings)
     chat: ChatSettings = Field(default_factory=ChatSettings)
     llm: LLMSettings = Field(default_factory=LLMSettings)
     mcp: MCPSettings = Field(default_factory=MCPSettings)
+    models: ModelsSettings = Field(default_factory=ModelsSettings)
     otel: OTELSettings = Field(default_factory=OTELSettings)
     phoenix: PhoenixSettings = Field(default_factory=PhoenixSettings)
     auth: AuthSettings = Field(default_factory=AuthSettings)
@@ -1203,24 +1558,39 @@ class Settings(BaseSettings):
     migration: MigrationSettings = Field(default_factory=MigrationSettings)
     storage: StorageSettings = Field(default_factory=StorageSettings)
     s3: S3Settings = Field(default_factory=S3Settings)
+    data_lake: DataLakeSettings = Field(default_factory=DataLakeSettings)
     git: GitSettings = Field(default_factory=GitSettings)
     sqs: SQSSettings = Field(default_factory=SQSSettings)
+    db_listener: DBListenerSettings = Field(default_factory=DBListenerSettings)
     chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
     content: ContentSettings = Field(default_factory=ContentSettings)
+    schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
     test: TestSettings = Field(default_factory=TestSettings)
 
 
+# Auto-load .env file from current directory if it exists
+# This happens BEFORE config file loading, so .env takes precedence
+from pathlib import Path
+from dotenv import load_dotenv
+
+_dotenv_path = Path(".env")
+if _dotenv_path.exists():
+    load_dotenv(_dotenv_path, override=False)  # Don't override existing env vars
+    logger.debug(f"Loaded environment from {_dotenv_path.resolve()}")
+
 # Load configuration from ~/.rem/config.yaml before initializing settings
 # This allows user configuration to be merged with environment variables
-try:
-    from rem.config import load_config, merge_config_to_env
-
-    _config = load_config()
-    if _config:
-        merge_config_to_env(_config)
-except ImportError:
-    # config module not available (e.g., during initial setup)
-    pass
+# Set REM_SKIP_CONFIG=1 to disable (useful for development with .env)
+if not os.getenv("REM_SKIP_CONFIG", "").lower() in ("true", "1", "yes"):
+    try:
+        from rem.config import load_config, merge_config_to_env
+
+        _config = load_config()
+        if _config:
+            merge_config_to_env(_config)
+    except ImportError:
+        # config module not available (e.g., during initial setup)
+        pass
 
 # Global settings singleton
 settings = Settings()

rem/sql/background_indexes.sql

@@ -1,9 +1,9 @@
 -- Background index creation
 -- Run AFTER initial data load to avoid blocking writes
 
--- HNSW vector index for embeddings_users
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_users_vector_hnsw
-ON embeddings_users
+-- HNSW vector index for embeddings_files
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_files_vector_hnsw
+ON embeddings_files
 USING hnsw (embedding vector_cosine_ops);
 
 -- HNSW vector index for embeddings_image_resources
@@ -11,24 +11,14 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_image_resources_vector_hn
 ON embeddings_image_resources
 USING hnsw (embedding vector_cosine_ops);
 
--- HNSW vector index for embeddings_moments
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_moments_vector_hnsw
-ON embeddings_moments
-USING hnsw (embedding vector_cosine_ops);
-
--- HNSW vector index for embeddings_resources
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_resources_vector_hnsw
-ON embeddings_resources
-USING hnsw (embedding vector_cosine_ops);
-
 -- HNSW vector index for embeddings_messages
 CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_messages_vector_hnsw
 ON embeddings_messages
 USING hnsw (embedding vector_cosine_ops);
 
--- HNSW vector index for embeddings_files
-CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_files_vector_hnsw
-ON embeddings_files
+-- HNSW vector index for embeddings_moments
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_moments_vector_hnsw
+ON embeddings_moments
 USING hnsw (embedding vector_cosine_ops);
 
 -- HNSW vector index for embeddings_ontology_configs
@@ -36,7 +26,22 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_ontology_configs_vector_h
 ON embeddings_ontology_configs
 USING hnsw (embedding vector_cosine_ops);
 
+-- HNSW vector index for embeddings_resources
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_resources_vector_hnsw
+ON embeddings_resources
+USING hnsw (embedding vector_cosine_ops);
+
 -- HNSW vector index for embeddings_schemas
 CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_schemas_vector_hnsw
 ON embeddings_schemas
 USING hnsw (embedding vector_cosine_ops);
+
+-- HNSW vector index for embeddings_sessions
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_sessions_vector_hnsw
+ON embeddings_sessions
+USING hnsw (embedding vector_cosine_ops);
+
+-- HNSW vector index for embeddings_users
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_users_vector_hnsw
+ON embeddings_users
+USING hnsw (embedding vector_cosine_ops);