remdb 0.3.14__py3-none-any.whl → 0.3.157__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rem/agentic/README.md +76 -0
- rem/agentic/__init__.py +15 -0
- rem/agentic/agents/__init__.py +32 -2
- rem/agentic/agents/agent_manager.py +310 -0
- rem/agentic/agents/sse_simulator.py +502 -0
- rem/agentic/context.py +51 -27
- rem/agentic/context_builder.py +5 -3
- rem/agentic/llm_provider_models.py +301 -0
- rem/agentic/mcp/tool_wrapper.py +155 -18
- rem/agentic/otel/setup.py +93 -4
- rem/agentic/providers/phoenix.py +371 -108
- rem/agentic/providers/pydantic_ai.py +280 -57
- rem/agentic/schema.py +361 -21
- rem/agentic/tools/rem_tools.py +3 -3
- rem/api/README.md +215 -1
- rem/api/deps.py +255 -0
- rem/api/main.py +132 -40
- rem/api/mcp_router/resources.py +1 -1
- rem/api/mcp_router/server.py +28 -5
- rem/api/mcp_router/tools.py +555 -7
- rem/api/routers/admin.py +494 -0
- rem/api/routers/auth.py +278 -4
- rem/api/routers/chat/completions.py +402 -20
- rem/api/routers/chat/models.py +88 -10
- rem/api/routers/chat/otel_utils.py +33 -0
- rem/api/routers/chat/sse_events.py +542 -0
- rem/api/routers/chat/streaming.py +697 -45
- rem/api/routers/dev.py +81 -0
- rem/api/routers/feedback.py +268 -0
- rem/api/routers/messages.py +473 -0
- rem/api/routers/models.py +78 -0
- rem/api/routers/query.py +360 -0
- rem/api/routers/shared_sessions.py +406 -0
- rem/auth/__init__.py +13 -3
- rem/auth/middleware.py +186 -22
- rem/auth/providers/__init__.py +4 -1
- rem/auth/providers/email.py +215 -0
- rem/cli/commands/README.md +237 -64
- rem/cli/commands/cluster.py +1808 -0
- rem/cli/commands/configure.py +4 -7
- rem/cli/commands/db.py +386 -143
- rem/cli/commands/experiments.py +468 -76
- rem/cli/commands/process.py +14 -8
- rem/cli/commands/schema.py +97 -50
- rem/cli/commands/session.py +336 -0
- rem/cli/dreaming.py +2 -2
- rem/cli/main.py +29 -6
- rem/config.py +10 -3
- rem/models/core/core_model.py +7 -1
- rem/models/core/experiment.py +58 -14
- rem/models/core/rem_query.py +5 -2
- rem/models/entities/__init__.py +25 -0
- rem/models/entities/domain_resource.py +38 -0
- rem/models/entities/feedback.py +123 -0
- rem/models/entities/message.py +30 -1
- rem/models/entities/ontology.py +1 -1
- rem/models/entities/ontology_config.py +1 -1
- rem/models/entities/session.py +83 -0
- rem/models/entities/shared_session.py +180 -0
- rem/models/entities/subscriber.py +175 -0
- rem/models/entities/user.py +1 -0
- rem/registry.py +10 -4
- rem/schemas/agents/core/agent-builder.yaml +134 -0
- rem/schemas/agents/examples/contract-analyzer.yaml +1 -1
- rem/schemas/agents/examples/contract-extractor.yaml +1 -1
- rem/schemas/agents/examples/cv-parser.yaml +1 -1
- rem/schemas/agents/rem.yaml +7 -3
- rem/services/__init__.py +3 -1
- rem/services/content/service.py +92 -19
- rem/services/email/__init__.py +10 -0
- rem/services/email/service.py +459 -0
- rem/services/email/templates.py +360 -0
- rem/services/embeddings/api.py +4 -4
- rem/services/embeddings/worker.py +16 -16
- rem/services/phoenix/client.py +154 -14
- rem/services/postgres/README.md +197 -15
- rem/services/postgres/__init__.py +2 -1
- rem/services/postgres/diff_service.py +547 -0
- rem/services/postgres/pydantic_to_sqlalchemy.py +470 -140
- rem/services/postgres/repository.py +132 -0
- rem/services/postgres/schema_generator.py +205 -4
- rem/services/postgres/service.py +6 -6
- rem/services/rem/parser.py +44 -9
- rem/services/rem/service.py +36 -2
- rem/services/session/compression.py +137 -51
- rem/services/session/reload.py +15 -8
- rem/settings.py +515 -27
- rem/sql/background_indexes.sql +21 -16
- rem/sql/migrations/001_install.sql +387 -54
- rem/sql/migrations/002_install_models.sql +2304 -377
- rem/sql/migrations/003_optional_extensions.sql +326 -0
- rem/sql/migrations/004_cache_system.sql +548 -0
- rem/sql/migrations/005_schema_update.sql +145 -0
- rem/utils/README.md +45 -0
- rem/utils/__init__.py +18 -0
- rem/utils/date_utils.py +2 -2
- rem/utils/files.py +157 -1
- rem/utils/model_helpers.py +156 -1
- rem/utils/schema_loader.py +220 -22
- rem/utils/sql_paths.py +146 -0
- rem/utils/sql_types.py +3 -1
- rem/utils/vision.py +1 -1
- rem/workers/__init__.py +3 -1
- rem/workers/db_listener.py +579 -0
- rem/workers/unlogged_maintainer.py +463 -0
- {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/METADATA +340 -229
- {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/RECORD +109 -80
- {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/WHEEL +1 -1
- rem/sql/002_install_models.sql +0 -1068
- rem/sql/install_models.sql +0 -1051
- rem/sql/migrations/003_seed_default_user.sql +0 -48
- {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/entry_points.txt +0 -0
rem/settings.py
CHANGED
|
@@ -15,14 +15,14 @@ Example .env file:
|
|
|
15
15
|
API__LOG_LEVEL=info
|
|
16
16
|
|
|
17
17
|
# LLM
|
|
18
|
-
LLM__DEFAULT_MODEL=
|
|
18
|
+
LLM__DEFAULT_MODEL=openai:gpt-4.1
|
|
19
19
|
LLM__DEFAULT_TEMPERATURE=0.5
|
|
20
20
|
LLM__MAX_RETRIES=10
|
|
21
21
|
LLM__OPENAI_API_KEY=sk-...
|
|
22
22
|
LLM__ANTHROPIC_API_KEY=sk-ant-...
|
|
23
23
|
|
|
24
|
-
# Database (port
|
|
25
|
-
POSTGRES__CONNECTION_STRING=postgresql://rem:rem@localhost:
|
|
24
|
+
# Database (port 5051 for Docker Compose prebuilt, 5050 for local dev)
|
|
25
|
+
POSTGRES__CONNECTION_STRING=postgresql://rem:rem@localhost:5051/rem
|
|
26
26
|
POSTGRES__POOL_MIN_SIZE=5
|
|
27
27
|
POSTGRES__POOL_MAX_SIZE=20
|
|
28
28
|
POSTGRES__STATEMENT_TIMEOUT=30000
|
|
@@ -33,14 +33,15 @@ Example .env file:
|
|
|
33
33
|
AUTH__OIDC_CLIENT_ID=your-client-id
|
|
34
34
|
AUTH__SESSION_SECRET=your-secret-key
|
|
35
35
|
|
|
36
|
-
# OpenTelemetry (disabled by default)
|
|
36
|
+
# OpenTelemetry (disabled by default - enable via env var when collector available)
|
|
37
|
+
# Standard OTLP collector ports: 4317 (gRPC), 4318 (HTTP)
|
|
37
38
|
OTEL__ENABLED=false
|
|
38
39
|
OTEL__SERVICE_NAME=rem-api
|
|
39
|
-
OTEL__COLLECTOR_ENDPOINT=http://localhost:
|
|
40
|
-
OTEL__PROTOCOL=
|
|
40
|
+
OTEL__COLLECTOR_ENDPOINT=http://localhost:4317
|
|
41
|
+
OTEL__PROTOCOL=grpc
|
|
41
42
|
|
|
42
|
-
# Arize Phoenix (
|
|
43
|
-
PHOENIX__ENABLED=
|
|
43
|
+
# Arize Phoenix (enabled by default - can be disabled via env var)
|
|
44
|
+
PHOENIX__ENABLED=true
|
|
44
45
|
PHOENIX__COLLECTOR_ENDPOINT=http://localhost:6006/v1/traces
|
|
45
46
|
PHOENIX__PROJECT_NAME=rem
|
|
46
47
|
|
|
@@ -58,7 +59,7 @@ Example .env file:
|
|
|
58
59
|
|
|
59
60
|
import os
|
|
60
61
|
import hashlib
|
|
61
|
-
from pydantic import Field, field_validator,
|
|
62
|
+
from pydantic import Field, field_validator, ValidationInfo
|
|
62
63
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
63
64
|
from loguru import logger
|
|
64
65
|
|
|
@@ -74,7 +75,7 @@ class LLMSettings(BaseSettings):
|
|
|
74
75
|
LLM__EVALUATOR_MODEL or EVALUATOR_MODEL - Model for LLM-as-judge evaluation
|
|
75
76
|
LLM__OPENAI_API_KEY or OPENAI_API_KEY - OpenAI API key
|
|
76
77
|
LLM__ANTHROPIC_API_KEY or ANTHROPIC_API_KEY - Anthropic API key
|
|
77
|
-
LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai
|
|
78
|
+
LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai)
|
|
78
79
|
LLM__EMBEDDING_MODEL or EMBEDDING_MODEL - Default embedding model name
|
|
79
80
|
"""
|
|
80
81
|
|
|
@@ -86,7 +87,7 @@ class LLMSettings(BaseSettings):
|
|
|
86
87
|
)
|
|
87
88
|
|
|
88
89
|
default_model: str = Field(
|
|
89
|
-
default="
|
|
90
|
+
default="openai:gpt-4.1",
|
|
90
91
|
description="Default LLM model (format: provider:model-id)",
|
|
91
92
|
)
|
|
92
93
|
|
|
@@ -129,7 +130,7 @@ class LLMSettings(BaseSettings):
|
|
|
129
130
|
|
|
130
131
|
embedding_provider: str = Field(
|
|
131
132
|
default="openai",
|
|
132
|
-
description="Default embedding provider (
|
|
133
|
+
description="Default embedding provider (currently only openai supported)",
|
|
133
134
|
)
|
|
134
135
|
|
|
135
136
|
embedding_model: str = Field(
|
|
@@ -241,6 +242,11 @@ class OTELSettings(BaseSettings):
|
|
|
241
242
|
description="Export timeout in milliseconds",
|
|
242
243
|
)
|
|
243
244
|
|
|
245
|
+
insecure: bool = Field(
|
|
246
|
+
default=True,
|
|
247
|
+
description="Use insecure (non-TLS) gRPC connection (default: True for local dev)",
|
|
248
|
+
)
|
|
249
|
+
|
|
244
250
|
|
|
245
251
|
class PhoenixSettings(BaseSettings):
|
|
246
252
|
"""
|
|
@@ -267,8 +273,8 @@ class PhoenixSettings(BaseSettings):
|
|
|
267
273
|
)
|
|
268
274
|
|
|
269
275
|
enabled: bool = Field(
|
|
270
|
-
default=
|
|
271
|
-
description="Enable Phoenix integration (
|
|
276
|
+
default=True,
|
|
277
|
+
description="Enable Phoenix integration (enabled by default)",
|
|
272
278
|
)
|
|
273
279
|
|
|
274
280
|
base_url: str = Field(
|
|
@@ -361,10 +367,16 @@ class AuthSettings(BaseSettings):
|
|
|
361
367
|
- Custom OIDC provider
|
|
362
368
|
|
|
363
369
|
Environment variables:
|
|
364
|
-
AUTH__ENABLED - Enable authentication (default:
|
|
370
|
+
AUTH__ENABLED - Enable authentication (default: true)
|
|
371
|
+
AUTH__ALLOW_ANONYMOUS - Allow rate-limited anonymous access (default: true)
|
|
365
372
|
AUTH__SESSION_SECRET - Secret for session cookie signing
|
|
366
373
|
AUTH__GOOGLE__* - Google OAuth settings
|
|
367
374
|
AUTH__MICROSOFT__* - Microsoft OAuth settings
|
|
375
|
+
|
|
376
|
+
Access modes:
|
|
377
|
+
- enabled=true, allow_anonymous=true: Auth available, anonymous gets rate-limited access
|
|
378
|
+
- enabled=true, allow_anonymous=false: Auth required for all requests
|
|
379
|
+
- enabled=false: No auth, all requests treated as default user (dev mode)
|
|
368
380
|
"""
|
|
369
381
|
|
|
370
382
|
model_config = SettingsConfigDict(
|
|
@@ -375,8 +387,26 @@ class AuthSettings(BaseSettings):
|
|
|
375
387
|
)
|
|
376
388
|
|
|
377
389
|
enabled: bool = Field(
|
|
378
|
-
default=
|
|
379
|
-
description="Enable authentication (
|
|
390
|
+
default=True,
|
|
391
|
+
description="Enable authentication (OAuth endpoints and middleware)",
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
allow_anonymous: bool = Field(
|
|
395
|
+
default=True,
|
|
396
|
+
description=(
|
|
397
|
+
"Allow anonymous (unauthenticated) access with rate limits. "
|
|
398
|
+
"When true, requests without auth get ANONYMOUS tier rate limits. "
|
|
399
|
+
"When false, all requests require authentication."
|
|
400
|
+
),
|
|
401
|
+
)
|
|
402
|
+
|
|
403
|
+
mcp_requires_auth: bool = Field(
|
|
404
|
+
default=True,
|
|
405
|
+
description=(
|
|
406
|
+
"Require authentication for MCP endpoints. "
|
|
407
|
+
"MCP is a protected service and should always require login in production. "
|
|
408
|
+
"Set to false only for local development/testing."
|
|
409
|
+
),
|
|
380
410
|
)
|
|
381
411
|
|
|
382
412
|
session_secret: str = Field(
|
|
@@ -390,7 +420,7 @@ class AuthSettings(BaseSettings):
|
|
|
390
420
|
|
|
391
421
|
@field_validator("session_secret", mode="before")
|
|
392
422
|
@classmethod
|
|
393
|
-
def generate_dev_secret(cls, v: str | None, info:
|
|
423
|
+
def generate_dev_secret(cls, v: str | None, info: ValidationInfo) -> str:
|
|
394
424
|
# Only generate if not already set and not in production
|
|
395
425
|
if not v and info.data.get("environment") != "production":
|
|
396
426
|
# Deterministic secret for development
|
|
@@ -434,10 +464,11 @@ class PostgresSettings(BaseSettings):
|
|
|
434
464
|
)
|
|
435
465
|
|
|
436
466
|
connection_string: str = Field(
|
|
437
|
-
default="postgresql://rem:rem@localhost:
|
|
438
|
-
description="PostgreSQL connection string (default uses Docker Compose port
|
|
467
|
+
default="postgresql://rem:rem@localhost:5051/rem",
|
|
468
|
+
description="PostgreSQL connection string (default uses Docker Compose prebuilt port 5051)",
|
|
439
469
|
)
|
|
440
470
|
|
|
471
|
+
|
|
441
472
|
pool_size: int = Field(
|
|
442
473
|
default=10,
|
|
443
474
|
description="Connection pool size (deprecated, use pool_min_size/pool_max_size)",
|
|
@@ -662,6 +693,91 @@ class S3Settings(BaseSettings):
|
|
|
662
693
|
)
|
|
663
694
|
|
|
664
695
|
|
|
696
|
+
class DataLakeSettings(BaseSettings):
|
|
697
|
+
"""
|
|
698
|
+
Data lake settings for experiment and dataset storage.
|
|
699
|
+
|
|
700
|
+
Data Lake Convention:
|
|
701
|
+
The data lake provides a standardized structure for storing datasets,
|
|
702
|
+
experiments, and calibration data in S3. Users bring their own bucket
|
|
703
|
+
and the version is pinned by default to v0 in the path.
|
|
704
|
+
|
|
705
|
+
S3 Path Structure:
|
|
706
|
+
s3://{bucket}/{version}/datasets/
|
|
707
|
+
├── raw/ # Raw source data + transformers
|
|
708
|
+
│ └── {dataset_name}/ # e.g., cns_drugs, codes, care
|
|
709
|
+
├── tables/ # Database table data (JSONL)
|
|
710
|
+
│ ├── resources/ # → resources table
|
|
711
|
+
│ │ ├── drugs/{category}/ # Psychotropic drugs
|
|
712
|
+
│ │ ├── care/stages/ # Treatment stages
|
|
713
|
+
│ │ └── crisis/ # Crisis resources
|
|
714
|
+
│ └── codes/ # → codes table
|
|
715
|
+
│ ├── icd10/{category}/ # ICD-10 codes
|
|
716
|
+
│ └── cpt/ # CPT codes
|
|
717
|
+
└── calibration/ # Agent calibration
|
|
718
|
+
├── experiments/ # Experiment configs + results
|
|
719
|
+
│ └── {agent}/{task}/ # e.g., siggy/risk-assessment
|
|
720
|
+
└── datasets/ # Shared evaluation datasets
|
|
721
|
+
|
|
722
|
+
Experiment Storage:
|
|
723
|
+
- Local: experiments/{agent}/{task}/experiment.yaml
|
|
724
|
+
- S3: s3://{bucket}/{version}/datasets/calibration/experiments/{agent}/{task}/
|
|
725
|
+
|
|
726
|
+
Environment variables:
|
|
727
|
+
DATA_LAKE__BUCKET_NAME - S3 bucket for data lake (required)
|
|
728
|
+
DATA_LAKE__VERSION - Path version prefix (default: v0)
|
|
729
|
+
DATA_LAKE__DATASETS_PREFIX - Datasets directory (default: datasets)
|
|
730
|
+
DATA_LAKE__EXPERIMENTS_PREFIX - Experiments subdirectory (default: experiments)
|
|
731
|
+
"""
|
|
732
|
+
|
|
733
|
+
model_config = SettingsConfigDict(
|
|
734
|
+
env_prefix="DATA_LAKE__",
|
|
735
|
+
env_file=".env",
|
|
736
|
+
env_file_encoding="utf-8",
|
|
737
|
+
extra="ignore",
|
|
738
|
+
)
|
|
739
|
+
|
|
740
|
+
bucket_name: str | None = Field(
|
|
741
|
+
default=None,
|
|
742
|
+
description="S3 bucket for data lake storage (user-provided)",
|
|
743
|
+
)
|
|
744
|
+
|
|
745
|
+
version: str = Field(
|
|
746
|
+
default="v0",
|
|
747
|
+
description="API version for data lake paths",
|
|
748
|
+
)
|
|
749
|
+
|
|
750
|
+
datasets_prefix: str = Field(
|
|
751
|
+
default="datasets",
|
|
752
|
+
description="Root directory for datasets in the bucket",
|
|
753
|
+
)
|
|
754
|
+
|
|
755
|
+
experiments_prefix: str = Field(
|
|
756
|
+
default="experiments",
|
|
757
|
+
description="Subdirectory within calibration for experiments",
|
|
758
|
+
)
|
|
759
|
+
|
|
760
|
+
def get_base_uri(self) -> str | None:
|
|
761
|
+
"""Get the base S3 URI for the data lake."""
|
|
762
|
+
if not self.bucket_name:
|
|
763
|
+
return None
|
|
764
|
+
return f"s3://{self.bucket_name}/{self.version}/{self.datasets_prefix}"
|
|
765
|
+
|
|
766
|
+
def get_experiment_uri(self, agent: str, task: str = "general") -> str | None:
|
|
767
|
+
"""Get the S3 URI for an experiment."""
|
|
768
|
+
base = self.get_base_uri()
|
|
769
|
+
if not base:
|
|
770
|
+
return None
|
|
771
|
+
return f"{base}/calibration/{self.experiments_prefix}/{agent}/{task}"
|
|
772
|
+
|
|
773
|
+
def get_tables_uri(self, table: str = "resources") -> str | None:
|
|
774
|
+
"""Get the S3 URI for a table directory."""
|
|
775
|
+
base = self.get_base_uri()
|
|
776
|
+
if not base:
|
|
777
|
+
return None
|
|
778
|
+
return f"{base}/tables/{table}"
|
|
779
|
+
|
|
780
|
+
|
|
665
781
|
class ChunkingSettings(BaseSettings):
|
|
666
782
|
"""
|
|
667
783
|
Document chunking settings for semantic text splitting.
|
|
@@ -945,6 +1061,8 @@ class APISettings(BaseSettings):
|
|
|
945
1061
|
API__RELOAD - Enable auto-reload for development
|
|
946
1062
|
API__WORKERS - Number of worker processes (production)
|
|
947
1063
|
API__LOG_LEVEL - Logging level (debug, info, warning, error)
|
|
1064
|
+
API__API_KEY_ENABLED - Enable X-API-Key header authentication
|
|
1065
|
+
API__API_KEY - API key for X-API-Key authentication
|
|
948
1066
|
"""
|
|
949
1067
|
|
|
950
1068
|
model_config = SettingsConfigDict(
|
|
@@ -979,6 +1097,92 @@ class APISettings(BaseSettings):
|
|
|
979
1097
|
description="Logging level (debug, info, warning, error, critical)",
|
|
980
1098
|
)
|
|
981
1099
|
|
|
1100
|
+
api_key_enabled: bool = Field(
|
|
1101
|
+
default=False,
|
|
1102
|
+
description=(
|
|
1103
|
+
"Enable X-API-Key header authentication for API endpoints. "
|
|
1104
|
+
"When enabled, requests must include X-API-Key header with valid key. "
|
|
1105
|
+
"This provides simple API key auth independent of OAuth."
|
|
1106
|
+
),
|
|
1107
|
+
)
|
|
1108
|
+
|
|
1109
|
+
api_key: str | None = Field(
|
|
1110
|
+
default=None,
|
|
1111
|
+
description=(
|
|
1112
|
+
"API key for X-API-Key authentication. Required when api_key_enabled=true. "
|
|
1113
|
+
"Generate with: python -c \"import secrets; print(secrets.token_urlsafe(32))\""
|
|
1114
|
+
),
|
|
1115
|
+
)
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
class ModelsSettings(BaseSettings):
|
|
1119
|
+
"""
|
|
1120
|
+
Custom model registration settings for downstream applications.
|
|
1121
|
+
|
|
1122
|
+
Allows downstream apps to specify Python modules containing custom models
|
|
1123
|
+
that should be imported (and thus registered) before schema generation.
|
|
1124
|
+
|
|
1125
|
+
This enables `rem db schema generate` to discover models registered with
|
|
1126
|
+
`@rem.register_model` in downstream applications.
|
|
1127
|
+
|
|
1128
|
+
Environment variables:
|
|
1129
|
+
MODELS__IMPORT_MODULES - Semicolon-separated list of Python modules to import
|
|
1130
|
+
Example: "models;myapp.entities;myapp.custom_models"
|
|
1131
|
+
|
|
1132
|
+
Example:
|
|
1133
|
+
# In downstream app's .env
|
|
1134
|
+
MODELS__IMPORT_MODULES=models
|
|
1135
|
+
|
|
1136
|
+
# In downstream app's models/__init__.py
|
|
1137
|
+
import rem
|
|
1138
|
+
from rem.models.core import CoreModel
|
|
1139
|
+
|
|
1140
|
+
@rem.register_model
|
|
1141
|
+
class MyCustomEntity(CoreModel):
|
|
1142
|
+
name: str
|
|
1143
|
+
|
|
1144
|
+
# Then run schema generation
|
|
1145
|
+
rem db schema generate # Includes MyCustomEntity
|
|
1146
|
+
"""
|
|
1147
|
+
|
|
1148
|
+
model_config = SettingsConfigDict(
|
|
1149
|
+
env_prefix="MODELS__",
|
|
1150
|
+
extra="ignore",
|
|
1151
|
+
)
|
|
1152
|
+
|
|
1153
|
+
import_modules: str = Field(
|
|
1154
|
+
default="",
|
|
1155
|
+
description=(
|
|
1156
|
+
"Semicolon-separated list of Python modules to import for model registration. "
|
|
1157
|
+
"These modules are imported before schema generation to ensure custom models "
|
|
1158
|
+
"decorated with @rem.register_model are discovered. "
|
|
1159
|
+
"Example: 'models;myapp.entities'"
|
|
1160
|
+
),
|
|
1161
|
+
)
|
|
1162
|
+
|
|
1163
|
+
@property
|
|
1164
|
+
def module_list(self) -> list[str]:
|
|
1165
|
+
"""
|
|
1166
|
+
Get modules as a list, filtering empty strings.
|
|
1167
|
+
|
|
1168
|
+
Auto-detects ./models folder if it exists and is importable.
|
|
1169
|
+
"""
|
|
1170
|
+
modules = []
|
|
1171
|
+
if self.import_modules:
|
|
1172
|
+
modules = [m.strip() for m in self.import_modules.split(";") if m.strip()]
|
|
1173
|
+
|
|
1174
|
+
# Auto-detect ./models if it exists and is a Python package (convention over configuration)
|
|
1175
|
+
from pathlib import Path
|
|
1176
|
+
|
|
1177
|
+
models_path = Path("./models")
|
|
1178
|
+
if models_path.exists() and models_path.is_dir():
|
|
1179
|
+
# Check if it's a Python package (has __init__.py)
|
|
1180
|
+
if (models_path / "__init__.py").exists():
|
|
1181
|
+
if "models" not in modules:
|
|
1182
|
+
modules.insert(0, "models")
|
|
1183
|
+
|
|
1184
|
+
return modules
|
|
1185
|
+
|
|
982
1186
|
|
|
983
1187
|
class SchemaSettings(BaseSettings):
|
|
984
1188
|
"""
|
|
@@ -1163,6 +1367,276 @@ class GitSettings(BaseSettings):
|
|
|
1163
1367
|
)
|
|
1164
1368
|
|
|
1165
1369
|
|
|
1370
|
+
class DBListenerSettings(BaseSettings):
|
|
1371
|
+
"""
|
|
1372
|
+
PostgreSQL LISTEN/NOTIFY database listener settings.
|
|
1373
|
+
|
|
1374
|
+
The DB Listener is a lightweight worker that subscribes to PostgreSQL
|
|
1375
|
+
NOTIFY events and dispatches them to external systems (SQS, REST, custom).
|
|
1376
|
+
|
|
1377
|
+
Architecture:
|
|
1378
|
+
- Single-replica deployment (to avoid duplicate processing)
|
|
1379
|
+
- Dedicated connection for LISTEN (not from connection pool)
|
|
1380
|
+
- Automatic reconnection with exponential backoff
|
|
1381
|
+
- Graceful shutdown on SIGTERM
|
|
1382
|
+
|
|
1383
|
+
Use Cases:
|
|
1384
|
+
- Sync data changes to external systems (Phoenix, webhooks)
|
|
1385
|
+
- Trigger async jobs without polling
|
|
1386
|
+
- Event-driven architectures with PostgreSQL as event source
|
|
1387
|
+
|
|
1388
|
+
Example PostgreSQL trigger:
|
|
1389
|
+
CREATE OR REPLACE FUNCTION notify_feedback_insert()
|
|
1390
|
+
RETURNS TRIGGER AS $$
|
|
1391
|
+
BEGIN
|
|
1392
|
+
PERFORM pg_notify('feedback_sync', json_build_object(
|
|
1393
|
+
'id', NEW.id,
|
|
1394
|
+
'table', 'feedbacks',
|
|
1395
|
+
'action', 'insert'
|
|
1396
|
+
)::text);
|
|
1397
|
+
RETURN NEW;
|
|
1398
|
+
END;
|
|
1399
|
+
$$ LANGUAGE plpgsql;
|
|
1400
|
+
|
|
1401
|
+
Environment variables:
|
|
1402
|
+
DB_LISTENER__ENABLED - Enable the listener worker (default: false)
|
|
1403
|
+
DB_LISTENER__CHANNELS - Comma-separated PostgreSQL channels to listen on
|
|
1404
|
+
DB_LISTENER__HANDLER_TYPE - Handler type: 'sqs', 'rest', or 'custom'
|
|
1405
|
+
DB_LISTENER__SQS_QUEUE_URL - SQS queue URL (for handler_type=sqs)
|
|
1406
|
+
DB_LISTENER__REST_ENDPOINT - REST endpoint URL (for handler_type=rest)
|
|
1407
|
+
DB_LISTENER__RECONNECT_DELAY - Initial reconnect delay in seconds
|
|
1408
|
+
DB_LISTENER__MAX_RECONNECT_DELAY - Maximum reconnect delay in seconds
|
|
1409
|
+
|
|
1410
|
+
References:
|
|
1411
|
+
- PostgreSQL NOTIFY: https://www.postgresql.org/docs/current/sql-notify.html
|
|
1412
|
+
- Brandur's Notifier: https://brandur.org/notifier
|
|
1413
|
+
"""
|
|
1414
|
+
|
|
1415
|
+
model_config = SettingsConfigDict(
|
|
1416
|
+
env_prefix="DB_LISTENER__",
|
|
1417
|
+
env_file=".env",
|
|
1418
|
+
env_file_encoding="utf-8",
|
|
1419
|
+
extra="ignore",
|
|
1420
|
+
)
|
|
1421
|
+
|
|
1422
|
+
enabled: bool = Field(
|
|
1423
|
+
default=False,
|
|
1424
|
+
description="Enable the DB Listener worker (disabled by default)",
|
|
1425
|
+
)
|
|
1426
|
+
|
|
1427
|
+
channels: str = Field(
|
|
1428
|
+
default="",
|
|
1429
|
+
description=(
|
|
1430
|
+
"Comma-separated list of PostgreSQL channels to LISTEN on. "
|
|
1431
|
+
"Example: 'feedback_sync,entity_update,user_events'"
|
|
1432
|
+
),
|
|
1433
|
+
)
|
|
1434
|
+
|
|
1435
|
+
handler_type: str = Field(
|
|
1436
|
+
default="rest",
|
|
1437
|
+
description=(
|
|
1438
|
+
"Handler type for dispatching notifications. Options: "
|
|
1439
|
+
"'sqs' (publish to SQS), 'rest' (POST to endpoint), 'custom' (Python handlers)"
|
|
1440
|
+
),
|
|
1441
|
+
)
|
|
1442
|
+
|
|
1443
|
+
sqs_queue_url: str = Field(
|
|
1444
|
+
default="",
|
|
1445
|
+
description="SQS queue URL for handler_type='sqs'",
|
|
1446
|
+
)
|
|
1447
|
+
|
|
1448
|
+
rest_endpoint: str = Field(
|
|
1449
|
+
default="http://localhost:8000/api/v1/internal/events",
|
|
1450
|
+
description=(
|
|
1451
|
+
"REST endpoint URL for handler_type='rest'. "
|
|
1452
|
+
"Receives POST with {channel, payload, source} JSON body."
|
|
1453
|
+
),
|
|
1454
|
+
)
|
|
1455
|
+
|
|
1456
|
+
reconnect_delay: float = Field(
|
|
1457
|
+
default=1.0,
|
|
1458
|
+
description="Initial delay (seconds) between reconnection attempts",
|
|
1459
|
+
)
|
|
1460
|
+
|
|
1461
|
+
max_reconnect_delay: float = Field(
|
|
1462
|
+
default=60.0,
|
|
1463
|
+
description="Maximum delay (seconds) between reconnection attempts (exponential backoff cap)",
|
|
1464
|
+
)
|
|
1465
|
+
|
|
1466
|
+
@property
|
|
1467
|
+
def channel_list(self) -> list[str]:
|
|
1468
|
+
"""Get channels as a list, filtering empty strings."""
|
|
1469
|
+
if not self.channels:
|
|
1470
|
+
return []
|
|
1471
|
+
return [c.strip() for c in self.channels.split(",") if c.strip()]
|
|
1472
|
+
|
|
1473
|
+
|
|
1474
|
+
class EmailSettings(BaseSettings):
|
|
1475
|
+
"""
|
|
1476
|
+
Email service settings for SMTP.
|
|
1477
|
+
|
|
1478
|
+
Supports passwordless login via email codes and transactional emails.
|
|
1479
|
+
Uses Gmail SMTP with App Passwords by default.
|
|
1480
|
+
|
|
1481
|
+
Generate app password at: https://myaccount.google.com/apppasswords
|
|
1482
|
+
|
|
1483
|
+
Environment variables:
|
|
1484
|
+
EMAIL__ENABLED - Enable email service (default: false)
|
|
1485
|
+
EMAIL__SMTP_HOST - SMTP server host (default: smtp.gmail.com)
|
|
1486
|
+
EMAIL__SMTP_PORT - SMTP server port (default: 587 for TLS)
|
|
1487
|
+
EMAIL__SENDER_EMAIL - Sender email address
|
|
1488
|
+
EMAIL__SENDER_NAME - Sender display name
|
|
1489
|
+
EMAIL__APP_PASSWORD - Gmail app password (from secrets)
|
|
1490
|
+
EMAIL__USE_TLS - Use TLS encryption (default: true)
|
|
1491
|
+
EMAIL__LOGIN_CODE_EXPIRY_MINUTES - Login code expiry (default: 10)
|
|
1492
|
+
|
|
1493
|
+
Branding environment variables (for email templates):
|
|
1494
|
+
EMAIL__APP_NAME - Application name in emails (default: REM)
|
|
1495
|
+
EMAIL__LOGO_URL - Logo URL for email templates (40x40 recommended)
|
|
1496
|
+
EMAIL__TAGLINE - Tagline shown in email footer
|
|
1497
|
+
EMAIL__WEBSITE_URL - Main website URL for email links
|
|
1498
|
+
EMAIL__PRIVACY_URL - Privacy policy URL for email footer
|
|
1499
|
+
EMAIL__TERMS_URL - Terms of service URL for email footer
|
|
1500
|
+
"""
|
|
1501
|
+
|
|
1502
|
+
model_config = SettingsConfigDict(
|
|
1503
|
+
env_prefix="EMAIL__",
|
|
1504
|
+
env_file=".env",
|
|
1505
|
+
env_file_encoding="utf-8",
|
|
1506
|
+
extra="ignore",
|
|
1507
|
+
)
|
|
1508
|
+
|
|
1509
|
+
enabled: bool = Field(
|
|
1510
|
+
default=False,
|
|
1511
|
+
description="Enable email service (requires app_password to be set)",
|
|
1512
|
+
)
|
|
1513
|
+
|
|
1514
|
+
smtp_host: str = Field(
|
|
1515
|
+
default="smtp.gmail.com",
|
|
1516
|
+
description="SMTP server host",
|
|
1517
|
+
)
|
|
1518
|
+
|
|
1519
|
+
smtp_port: int = Field(
|
|
1520
|
+
default=587,
|
|
1521
|
+
description="SMTP server port (587 for TLS, 465 for SSL)",
|
|
1522
|
+
)
|
|
1523
|
+
|
|
1524
|
+
sender_email: str = Field(
|
|
1525
|
+
default="",
|
|
1526
|
+
description="Sender email address",
|
|
1527
|
+
)
|
|
1528
|
+
|
|
1529
|
+
sender_name: str = Field(
|
|
1530
|
+
default="REM",
|
|
1531
|
+
description="Sender display name",
|
|
1532
|
+
)
|
|
1533
|
+
|
|
1534
|
+
# Branding settings for email templates
|
|
1535
|
+
app_name: str = Field(
|
|
1536
|
+
default="REM",
|
|
1537
|
+
description="Application name shown in email templates",
|
|
1538
|
+
)
|
|
1539
|
+
|
|
1540
|
+
logo_url: str | None = Field(
|
|
1541
|
+
default=None,
|
|
1542
|
+
description="Logo URL for email templates (40x40 recommended)",
|
|
1543
|
+
)
|
|
1544
|
+
|
|
1545
|
+
tagline: str = Field(
|
|
1546
|
+
default="Your AI-powered platform",
|
|
1547
|
+
description="Tagline shown in email footer",
|
|
1548
|
+
)
|
|
1549
|
+
|
|
1550
|
+
website_url: str = Field(
|
|
1551
|
+
default="https://rem.ai",
|
|
1552
|
+
description="Main website URL for email links",
|
|
1553
|
+
)
|
|
1554
|
+
|
|
1555
|
+
privacy_url: str = Field(
|
|
1556
|
+
default="https://rem.ai/privacy",
|
|
1557
|
+
description="Privacy policy URL for email footer",
|
|
1558
|
+
)
|
|
1559
|
+
|
|
1560
|
+
terms_url: str = Field(
|
|
1561
|
+
default="https://rem.ai/terms",
|
|
1562
|
+
description="Terms of service URL for email footer",
|
|
1563
|
+
)
|
|
1564
|
+
|
|
1565
|
+
app_password: str | None = Field(
|
|
1566
|
+
default=None,
|
|
1567
|
+
description="Gmail app password for SMTP authentication",
|
|
1568
|
+
)
|
|
1569
|
+
|
|
1570
|
+
use_tls: bool = Field(
|
|
1571
|
+
default=True,
|
|
1572
|
+
description="Use TLS encryption for SMTP",
|
|
1573
|
+
)
|
|
1574
|
+
|
|
1575
|
+
login_code_expiry_minutes: int = Field(
|
|
1576
|
+
default=10,
|
|
1577
|
+
description="Login code expiry in minutes",
|
|
1578
|
+
)
|
|
1579
|
+
|
|
1580
|
+
trusted_email_domains: str = Field(
|
|
1581
|
+
default="",
|
|
1582
|
+
description=(
|
|
1583
|
+
"Comma-separated list of trusted email domains for new user registration. "
|
|
1584
|
+
"Existing users can always login regardless of domain. "
|
|
1585
|
+
"New users must have an email from a trusted domain. "
|
|
1586
|
+
"Empty string means all domains are allowed. "
|
|
1587
|
+
"Example: 'siggymd.ai,example.com'"
|
|
1588
|
+
),
|
|
1589
|
+
)
|
|
1590
|
+
|
|
1591
|
+
@property
|
|
1592
|
+
def trusted_domain_list(self) -> list[str]:
|
|
1593
|
+
"""Get trusted domains as a list, filtering empty strings."""
|
|
1594
|
+
if not self.trusted_email_domains:
|
|
1595
|
+
return []
|
|
1596
|
+
return [d.strip().lower() for d in self.trusted_email_domains.split(",") if d.strip()]
|
|
1597
|
+
|
|
1598
|
+
def is_domain_trusted(self, email: str) -> bool:
|
|
1599
|
+
"""Check if an email's domain is in the trusted list.
|
|
1600
|
+
|
|
1601
|
+
Args:
|
|
1602
|
+
email: Email address to check
|
|
1603
|
+
|
|
1604
|
+
Returns:
|
|
1605
|
+
True if domain is trusted (or if no trusted domains configured)
|
|
1606
|
+
"""
|
|
1607
|
+
domains = self.trusted_domain_list
|
|
1608
|
+
if not domains:
|
|
1609
|
+
# No restrictions configured
|
|
1610
|
+
return True
|
|
1611
|
+
|
|
1612
|
+
email_domain = email.lower().split("@")[-1].strip()
|
|
1613
|
+
return email_domain in domains
|
|
1614
|
+
|
|
1615
|
+
@property
|
|
1616
|
+
def is_configured(self) -> bool:
|
|
1617
|
+
"""Check if email service is properly configured."""
|
|
1618
|
+
return bool(self.sender_email and self.app_password)
|
|
1619
|
+
|
|
1620
|
+
@property
|
|
1621
|
+
def template_kwargs(self) -> dict:
|
|
1622
|
+
"""
|
|
1623
|
+
Get branding kwargs for email templates.
|
|
1624
|
+
|
|
1625
|
+
Returns a dict that can be passed to template functions:
|
|
1626
|
+
login_code_template(..., **settings.email.template_kwargs)
|
|
1627
|
+
"""
|
|
1628
|
+
kwargs = {
|
|
1629
|
+
"app_name": self.app_name,
|
|
1630
|
+
"tagline": self.tagline,
|
|
1631
|
+
"website_url": self.website_url,
|
|
1632
|
+
"privacy_url": self.privacy_url,
|
|
1633
|
+
"terms_url": self.terms_url,
|
|
1634
|
+
}
|
|
1635
|
+
if self.logo_url:
|
|
1636
|
+
kwargs["logo_url"] = self.logo_url
|
|
1637
|
+
return kwargs
|
|
1638
|
+
|
|
1639
|
+
|
|
1166
1640
|
class TestSettings(BaseSettings):
|
|
1167
1641
|
"""
|
|
1168
1642
|
Test environment settings.
|
|
@@ -1232,6 +1706,11 @@ class Settings(BaseSettings):
|
|
|
1232
1706
|
extra="ignore",
|
|
1233
1707
|
)
|
|
1234
1708
|
|
|
1709
|
+
app_name: str = Field(
|
|
1710
|
+
default="REM",
|
|
1711
|
+
description="Application/API name used in docs, titles, and user-facing text",
|
|
1712
|
+
)
|
|
1713
|
+
|
|
1235
1714
|
team: str = Field(
|
|
1236
1715
|
default="rem",
|
|
1237
1716
|
description="Team or project name for observability",
|
|
@@ -1252,16 +1731,12 @@ class Settings(BaseSettings):
|
|
|
1252
1731
|
description="Root path for reverse proxy (e.g., /rem for ALB routing)",
|
|
1253
1732
|
)
|
|
1254
1733
|
|
|
1255
|
-
sql_dir: str = Field(
|
|
1256
|
-
default="src/rem/sql",
|
|
1257
|
-
description="Directory for SQL files and migrations",
|
|
1258
|
-
)
|
|
1259
|
-
|
|
1260
1734
|
# Nested settings groups
|
|
1261
1735
|
api: APISettings = Field(default_factory=APISettings)
|
|
1262
1736
|
chat: ChatSettings = Field(default_factory=ChatSettings)
|
|
1263
1737
|
llm: LLMSettings = Field(default_factory=LLMSettings)
|
|
1264
1738
|
mcp: MCPSettings = Field(default_factory=MCPSettings)
|
|
1739
|
+
models: ModelsSettings = Field(default_factory=ModelsSettings)
|
|
1265
1740
|
otel: OTELSettings = Field(default_factory=OTELSettings)
|
|
1266
1741
|
phoenix: PhoenixSettings = Field(default_factory=PhoenixSettings)
|
|
1267
1742
|
auth: AuthSettings = Field(default_factory=AuthSettings)
|
|
@@ -1269,18 +1744,31 @@ class Settings(BaseSettings):
|
|
|
1269
1744
|
migration: MigrationSettings = Field(default_factory=MigrationSettings)
|
|
1270
1745
|
storage: StorageSettings = Field(default_factory=StorageSettings)
|
|
1271
1746
|
s3: S3Settings = Field(default_factory=S3Settings)
|
|
1747
|
+
data_lake: DataLakeSettings = Field(default_factory=DataLakeSettings)
|
|
1272
1748
|
git: GitSettings = Field(default_factory=GitSettings)
|
|
1273
1749
|
sqs: SQSSettings = Field(default_factory=SQSSettings)
|
|
1750
|
+
db_listener: DBListenerSettings = Field(default_factory=DBListenerSettings)
|
|
1274
1751
|
chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
|
|
1275
1752
|
content: ContentSettings = Field(default_factory=ContentSettings)
|
|
1276
1753
|
schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
|
|
1754
|
+
email: EmailSettings = Field(default_factory=EmailSettings)
|
|
1277
1755
|
test: TestSettings = Field(default_factory=TestSettings)
|
|
1278
1756
|
|
|
1279
1757
|
|
|
1758
|
+
# Auto-load .env file from current directory if it exists
|
|
1759
|
+
# This happens BEFORE config file loading, so .env takes precedence
|
|
1760
|
+
from pathlib import Path
|
|
1761
|
+
from dotenv import load_dotenv
|
|
1762
|
+
|
|
1763
|
+
_dotenv_path = Path(".env")
|
|
1764
|
+
if _dotenv_path.exists():
|
|
1765
|
+
load_dotenv(_dotenv_path, override=False) # Don't override existing env vars
|
|
1766
|
+
logger.debug(f"Loaded environment from {_dotenv_path.resolve()}")
|
|
1767
|
+
|
|
1280
1768
|
# Load configuration from ~/.rem/config.yaml before initializing settings
|
|
1281
1769
|
# This allows user configuration to be merged with environment variables
|
|
1282
|
-
# Set
|
|
1283
|
-
if not os.getenv("
|
|
1770
|
+
# Set REM_SKIP_CONFIG=1 to disable (useful for development with .env)
|
|
1771
|
+
if not os.getenv("REM_SKIP_CONFIG", "").lower() in ("true", "1", "yes"):
|
|
1284
1772
|
try:
|
|
1285
1773
|
from rem.config import load_config, merge_config_to_env
|
|
1286
1774
|
|