remdb 0.3.14__py3-none-any.whl → 0.3.157__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. rem/agentic/README.md +76 -0
  2. rem/agentic/__init__.py +15 -0
  3. rem/agentic/agents/__init__.py +32 -2
  4. rem/agentic/agents/agent_manager.py +310 -0
  5. rem/agentic/agents/sse_simulator.py +502 -0
  6. rem/agentic/context.py +51 -27
  7. rem/agentic/context_builder.py +5 -3
  8. rem/agentic/llm_provider_models.py +301 -0
  9. rem/agentic/mcp/tool_wrapper.py +155 -18
  10. rem/agentic/otel/setup.py +93 -4
  11. rem/agentic/providers/phoenix.py +371 -108
  12. rem/agentic/providers/pydantic_ai.py +280 -57
  13. rem/agentic/schema.py +361 -21
  14. rem/agentic/tools/rem_tools.py +3 -3
  15. rem/api/README.md +215 -1
  16. rem/api/deps.py +255 -0
  17. rem/api/main.py +132 -40
  18. rem/api/mcp_router/resources.py +1 -1
  19. rem/api/mcp_router/server.py +28 -5
  20. rem/api/mcp_router/tools.py +555 -7
  21. rem/api/routers/admin.py +494 -0
  22. rem/api/routers/auth.py +278 -4
  23. rem/api/routers/chat/completions.py +402 -20
  24. rem/api/routers/chat/models.py +88 -10
  25. rem/api/routers/chat/otel_utils.py +33 -0
  26. rem/api/routers/chat/sse_events.py +542 -0
  27. rem/api/routers/chat/streaming.py +697 -45
  28. rem/api/routers/dev.py +81 -0
  29. rem/api/routers/feedback.py +268 -0
  30. rem/api/routers/messages.py +473 -0
  31. rem/api/routers/models.py +78 -0
  32. rem/api/routers/query.py +360 -0
  33. rem/api/routers/shared_sessions.py +406 -0
  34. rem/auth/__init__.py +13 -3
  35. rem/auth/middleware.py +186 -22
  36. rem/auth/providers/__init__.py +4 -1
  37. rem/auth/providers/email.py +215 -0
  38. rem/cli/commands/README.md +237 -64
  39. rem/cli/commands/cluster.py +1808 -0
  40. rem/cli/commands/configure.py +4 -7
  41. rem/cli/commands/db.py +386 -143
  42. rem/cli/commands/experiments.py +468 -76
  43. rem/cli/commands/process.py +14 -8
  44. rem/cli/commands/schema.py +97 -50
  45. rem/cli/commands/session.py +336 -0
  46. rem/cli/dreaming.py +2 -2
  47. rem/cli/main.py +29 -6
  48. rem/config.py +10 -3
  49. rem/models/core/core_model.py +7 -1
  50. rem/models/core/experiment.py +58 -14
  51. rem/models/core/rem_query.py +5 -2
  52. rem/models/entities/__init__.py +25 -0
  53. rem/models/entities/domain_resource.py +38 -0
  54. rem/models/entities/feedback.py +123 -0
  55. rem/models/entities/message.py +30 -1
  56. rem/models/entities/ontology.py +1 -1
  57. rem/models/entities/ontology_config.py +1 -1
  58. rem/models/entities/session.py +83 -0
  59. rem/models/entities/shared_session.py +180 -0
  60. rem/models/entities/subscriber.py +175 -0
  61. rem/models/entities/user.py +1 -0
  62. rem/registry.py +10 -4
  63. rem/schemas/agents/core/agent-builder.yaml +134 -0
  64. rem/schemas/agents/examples/contract-analyzer.yaml +1 -1
  65. rem/schemas/agents/examples/contract-extractor.yaml +1 -1
  66. rem/schemas/agents/examples/cv-parser.yaml +1 -1
  67. rem/schemas/agents/rem.yaml +7 -3
  68. rem/services/__init__.py +3 -1
  69. rem/services/content/service.py +92 -19
  70. rem/services/email/__init__.py +10 -0
  71. rem/services/email/service.py +459 -0
  72. rem/services/email/templates.py +360 -0
  73. rem/services/embeddings/api.py +4 -4
  74. rem/services/embeddings/worker.py +16 -16
  75. rem/services/phoenix/client.py +154 -14
  76. rem/services/postgres/README.md +197 -15
  77. rem/services/postgres/__init__.py +2 -1
  78. rem/services/postgres/diff_service.py +547 -0
  79. rem/services/postgres/pydantic_to_sqlalchemy.py +470 -140
  80. rem/services/postgres/repository.py +132 -0
  81. rem/services/postgres/schema_generator.py +205 -4
  82. rem/services/postgres/service.py +6 -6
  83. rem/services/rem/parser.py +44 -9
  84. rem/services/rem/service.py +36 -2
  85. rem/services/session/compression.py +137 -51
  86. rem/services/session/reload.py +15 -8
  87. rem/settings.py +515 -27
  88. rem/sql/background_indexes.sql +21 -16
  89. rem/sql/migrations/001_install.sql +387 -54
  90. rem/sql/migrations/002_install_models.sql +2304 -377
  91. rem/sql/migrations/003_optional_extensions.sql +326 -0
  92. rem/sql/migrations/004_cache_system.sql +548 -0
  93. rem/sql/migrations/005_schema_update.sql +145 -0
  94. rem/utils/README.md +45 -0
  95. rem/utils/__init__.py +18 -0
  96. rem/utils/date_utils.py +2 -2
  97. rem/utils/files.py +157 -1
  98. rem/utils/model_helpers.py +156 -1
  99. rem/utils/schema_loader.py +220 -22
  100. rem/utils/sql_paths.py +146 -0
  101. rem/utils/sql_types.py +3 -1
  102. rem/utils/vision.py +1 -1
  103. rem/workers/__init__.py +3 -1
  104. rem/workers/db_listener.py +579 -0
  105. rem/workers/unlogged_maintainer.py +463 -0
  106. {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/METADATA +340 -229
  107. {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/RECORD +109 -80
  108. {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/WHEEL +1 -1
  109. rem/sql/002_install_models.sql +0 -1068
  110. rem/sql/install_models.sql +0 -1051
  111. rem/sql/migrations/003_seed_default_user.sql +0 -48
  112. {remdb-0.3.14.dist-info → remdb-0.3.157.dist-info}/entry_points.txt +0 -0
rem/settings.py CHANGED
@@ -15,14 +15,14 @@ Example .env file:
15
15
  API__LOG_LEVEL=info
16
16
 
17
17
  # LLM
18
- LLM__DEFAULT_MODEL=anthropic:claude-sonnet-4-5-20250929
18
+ LLM__DEFAULT_MODEL=openai:gpt-4.1
19
19
  LLM__DEFAULT_TEMPERATURE=0.5
20
20
  LLM__MAX_RETRIES=10
21
21
  LLM__OPENAI_API_KEY=sk-...
22
22
  LLM__ANTHROPIC_API_KEY=sk-ant-...
23
23
 
24
- # Database (port 5050 for Docker Compose)
25
- POSTGRES__CONNECTION_STRING=postgresql://rem:rem@localhost:5050/rem
24
+ # Database (port 5051 for Docker Compose prebuilt, 5050 for local dev)
25
+ POSTGRES__CONNECTION_STRING=postgresql://rem:rem@localhost:5051/rem
26
26
  POSTGRES__POOL_MIN_SIZE=5
27
27
  POSTGRES__POOL_MAX_SIZE=20
28
28
  POSTGRES__STATEMENT_TIMEOUT=30000
@@ -33,14 +33,15 @@ Example .env file:
33
33
  AUTH__OIDC_CLIENT_ID=your-client-id
34
34
  AUTH__SESSION_SECRET=your-secret-key
35
35
 
36
- # OpenTelemetry (disabled by default)
36
+ # OpenTelemetry (disabled by default - enable via env var when collector available)
37
+ # Standard OTLP collector ports: 4317 (gRPC), 4318 (HTTP)
37
38
  OTEL__ENABLED=false
38
39
  OTEL__SERVICE_NAME=rem-api
39
- OTEL__COLLECTOR_ENDPOINT=http://localhost:4318
40
- OTEL__PROTOCOL=http
40
+ OTEL__COLLECTOR_ENDPOINT=http://localhost:4317
41
+ OTEL__PROTOCOL=grpc
41
42
 
42
- # Arize Phoenix (disabled by default)
43
- PHOENIX__ENABLED=false
43
+ # Arize Phoenix (enabled by default - can be disabled via env var)
44
+ PHOENIX__ENABLED=true
44
45
  PHOENIX__COLLECTOR_ENDPOINT=http://localhost:6006/v1/traces
45
46
  PHOENIX__PROJECT_NAME=rem
46
47
 
@@ -58,7 +59,7 @@ Example .env file:
58
59
 
59
60
  import os
60
61
  import hashlib
61
- from pydantic import Field, field_validator, FieldValidationInfo
62
+ from pydantic import Field, field_validator, ValidationInfo
62
63
  from pydantic_settings import BaseSettings, SettingsConfigDict
63
64
  from loguru import logger
64
65
 
@@ -74,7 +75,7 @@ class LLMSettings(BaseSettings):
74
75
  LLM__EVALUATOR_MODEL or EVALUATOR_MODEL - Model for LLM-as-judge evaluation
75
76
  LLM__OPENAI_API_KEY or OPENAI_API_KEY - OpenAI API key
76
77
  LLM__ANTHROPIC_API_KEY or ANTHROPIC_API_KEY - Anthropic API key
77
- LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai, cohere, jina, etc.)
78
+ LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai)
78
79
  LLM__EMBEDDING_MODEL or EMBEDDING_MODEL - Default embedding model name
79
80
  """
80
81
 
@@ -86,7 +87,7 @@ class LLMSettings(BaseSettings):
86
87
  )
87
88
 
88
89
  default_model: str = Field(
89
- default="anthropic:claude-sonnet-4-5-20250929",
90
+ default="openai:gpt-4.1",
90
91
  description="Default LLM model (format: provider:model-id)",
91
92
  )
92
93
 
@@ -129,7 +130,7 @@ class LLMSettings(BaseSettings):
129
130
 
130
131
  embedding_provider: str = Field(
131
132
  default="openai",
132
- description="Default embedding provider (openai, cohere, jina, etc.)",
133
+ description="Default embedding provider (currently only openai supported)",
133
134
  )
134
135
 
135
136
  embedding_model: str = Field(
@@ -241,6 +242,11 @@ class OTELSettings(BaseSettings):
241
242
  description="Export timeout in milliseconds",
242
243
  )
243
244
 
245
+ insecure: bool = Field(
246
+ default=True,
247
+ description="Use insecure (non-TLS) gRPC connection (default: True for local dev)",
248
+ )
249
+
244
250
 
245
251
  class PhoenixSettings(BaseSettings):
246
252
  """
@@ -267,8 +273,8 @@ class PhoenixSettings(BaseSettings):
267
273
  )
268
274
 
269
275
  enabled: bool = Field(
270
- default=False,
271
- description="Enable Phoenix integration (disabled by default for local dev)",
276
+ default=True,
277
+ description="Enable Phoenix integration (enabled by default)",
272
278
  )
273
279
 
274
280
  base_url: str = Field(
@@ -361,10 +367,16 @@ class AuthSettings(BaseSettings):
361
367
  - Custom OIDC provider
362
368
 
363
369
  Environment variables:
364
- AUTH__ENABLED - Enable authentication (default: false)
370
+ AUTH__ENABLED - Enable authentication (default: true)
371
+ AUTH__ALLOW_ANONYMOUS - Allow rate-limited anonymous access (default: true)
365
372
  AUTH__SESSION_SECRET - Secret for session cookie signing
366
373
  AUTH__GOOGLE__* - Google OAuth settings
367
374
  AUTH__MICROSOFT__* - Microsoft OAuth settings
375
+
376
+ Access modes:
377
+ - enabled=true, allow_anonymous=true: Auth available, anonymous gets rate-limited access
378
+ - enabled=true, allow_anonymous=false: Auth required for all requests
379
+ - enabled=false: No auth, all requests treated as default user (dev mode)
368
380
  """
369
381
 
370
382
  model_config = SettingsConfigDict(
@@ -375,8 +387,26 @@ class AuthSettings(BaseSettings):
375
387
  )
376
388
 
377
389
  enabled: bool = Field(
378
- default=False,
379
- description="Enable authentication (disabled by default for testing)",
390
+ default=True,
391
+ description="Enable authentication (OAuth endpoints and middleware)",
392
+ )
393
+
394
+ allow_anonymous: bool = Field(
395
+ default=True,
396
+ description=(
397
+ "Allow anonymous (unauthenticated) access with rate limits. "
398
+ "When true, requests without auth get ANONYMOUS tier rate limits. "
399
+ "When false, all requests require authentication."
400
+ ),
401
+ )
402
+
403
+ mcp_requires_auth: bool = Field(
404
+ default=True,
405
+ description=(
406
+ "Require authentication for MCP endpoints. "
407
+ "MCP is a protected service and should always require login in production. "
408
+ "Set to false only for local development/testing."
409
+ ),
380
410
  )
381
411
 
382
412
  session_secret: str = Field(
@@ -390,7 +420,7 @@ class AuthSettings(BaseSettings):
390
420
 
391
421
  @field_validator("session_secret", mode="before")
392
422
  @classmethod
393
- def generate_dev_secret(cls, v: str | None, info: FieldValidationInfo) -> str:
423
+ def generate_dev_secret(cls, v: str | None, info: ValidationInfo) -> str:
394
424
  # Only generate if not already set and not in production
395
425
  if not v and info.data.get("environment") != "production":
396
426
  # Deterministic secret for development
@@ -434,10 +464,11 @@ class PostgresSettings(BaseSettings):
434
464
  )
435
465
 
436
466
  connection_string: str = Field(
437
- default="postgresql://rem:rem@localhost:5050/rem",
438
- description="PostgreSQL connection string (default uses Docker Compose port 5050)",
467
+ default="postgresql://rem:rem@localhost:5051/rem",
468
+ description="PostgreSQL connection string (default uses Docker Compose prebuilt port 5051)",
439
469
  )
440
470
 
471
+
441
472
  pool_size: int = Field(
442
473
  default=10,
443
474
  description="Connection pool size (deprecated, use pool_min_size/pool_max_size)",
@@ -662,6 +693,91 @@ class S3Settings(BaseSettings):
662
693
  )
663
694
 
664
695
 
696
+ class DataLakeSettings(BaseSettings):
697
+ """
698
+ Data lake settings for experiment and dataset storage.
699
+
700
+ Data Lake Convention:
701
+ The data lake provides a standardized structure for storing datasets,
702
+ experiments, and calibration data in S3. Users bring their own bucket
703
+ and the version is pinned by default to v0 in the path.
704
+
705
+ S3 Path Structure:
706
+ s3://{bucket}/{version}/datasets/
707
+ ├── raw/ # Raw source data + transformers
708
+ │ └── {dataset_name}/ # e.g., cns_drugs, codes, care
709
+ ├── tables/ # Database table data (JSONL)
710
+ │ ├── resources/ # → resources table
711
+ │ │ ├── drugs/{category}/ # Psychotropic drugs
712
+ │ │ ├── care/stages/ # Treatment stages
713
+ │ │ └── crisis/ # Crisis resources
714
+ │ └── codes/ # → codes table
715
+ │ ├── icd10/{category}/ # ICD-10 codes
716
+ │ └── cpt/ # CPT codes
717
+ └── calibration/ # Agent calibration
718
+ ├── experiments/ # Experiment configs + results
719
+ │ └── {agent}/{task}/ # e.g., siggy/risk-assessment
720
+ └── datasets/ # Shared evaluation datasets
721
+
722
+ Experiment Storage:
723
+ - Local: experiments/{agent}/{task}/experiment.yaml
724
+ - S3: s3://{bucket}/{version}/datasets/calibration/experiments/{agent}/{task}/
725
+
726
+ Environment variables:
727
+ DATA_LAKE__BUCKET_NAME - S3 bucket for data lake (required)
728
+ DATA_LAKE__VERSION - Path version prefix (default: v0)
729
+ DATA_LAKE__DATASETS_PREFIX - Datasets directory (default: datasets)
730
+ DATA_LAKE__EXPERIMENTS_PREFIX - Experiments subdirectory (default: experiments)
731
+ """
732
+
733
+ model_config = SettingsConfigDict(
734
+ env_prefix="DATA_LAKE__",
735
+ env_file=".env",
736
+ env_file_encoding="utf-8",
737
+ extra="ignore",
738
+ )
739
+
740
+ bucket_name: str | None = Field(
741
+ default=None,
742
+ description="S3 bucket for data lake storage (user-provided)",
743
+ )
744
+
745
+ version: str = Field(
746
+ default="v0",
747
+ description="API version for data lake paths",
748
+ )
749
+
750
+ datasets_prefix: str = Field(
751
+ default="datasets",
752
+ description="Root directory for datasets in the bucket",
753
+ )
754
+
755
+ experiments_prefix: str = Field(
756
+ default="experiments",
757
+ description="Subdirectory within calibration for experiments",
758
+ )
759
+
760
+ def get_base_uri(self) -> str | None:
761
+ """Get the base S3 URI for the data lake."""
762
+ if not self.bucket_name:
763
+ return None
764
+ return f"s3://{self.bucket_name}/{self.version}/{self.datasets_prefix}"
765
+
766
+ def get_experiment_uri(self, agent: str, task: str = "general") -> str | None:
767
+ """Get the S3 URI for an experiment."""
768
+ base = self.get_base_uri()
769
+ if not base:
770
+ return None
771
+ return f"{base}/calibration/{self.experiments_prefix}/{agent}/{task}"
772
+
773
+ def get_tables_uri(self, table: str = "resources") -> str | None:
774
+ """Get the S3 URI for a table directory."""
775
+ base = self.get_base_uri()
776
+ if not base:
777
+ return None
778
+ return f"{base}/tables/{table}"
779
+
780
+
665
781
  class ChunkingSettings(BaseSettings):
666
782
  """
667
783
  Document chunking settings for semantic text splitting.
@@ -945,6 +1061,8 @@ class APISettings(BaseSettings):
945
1061
  API__RELOAD - Enable auto-reload for development
946
1062
  API__WORKERS - Number of worker processes (production)
947
1063
  API__LOG_LEVEL - Logging level (debug, info, warning, error)
1064
+ API__API_KEY_ENABLED - Enable X-API-Key header authentication
1065
+ API__API_KEY - API key for X-API-Key authentication
948
1066
  """
949
1067
 
950
1068
  model_config = SettingsConfigDict(
@@ -979,6 +1097,92 @@ class APISettings(BaseSettings):
979
1097
  description="Logging level (debug, info, warning, error, critical)",
980
1098
  )
981
1099
 
1100
+ api_key_enabled: bool = Field(
1101
+ default=False,
1102
+ description=(
1103
+ "Enable X-API-Key header authentication for API endpoints. "
1104
+ "When enabled, requests must include X-API-Key header with valid key. "
1105
+ "This provides simple API key auth independent of OAuth."
1106
+ ),
1107
+ )
1108
+
1109
+ api_key: str | None = Field(
1110
+ default=None,
1111
+ description=(
1112
+ "API key for X-API-Key authentication. Required when api_key_enabled=true. "
1113
+ "Generate with: python -c \"import secrets; print(secrets.token_urlsafe(32))\""
1114
+ ),
1115
+ )
1116
+
1117
+
1118
+ class ModelsSettings(BaseSettings):
1119
+ """
1120
+ Custom model registration settings for downstream applications.
1121
+
1122
+ Allows downstream apps to specify Python modules containing custom models
1123
+ that should be imported (and thus registered) before schema generation.
1124
+
1125
+ This enables `rem db schema generate` to discover models registered with
1126
+ `@rem.register_model` in downstream applications.
1127
+
1128
+ Environment variables:
1129
+ MODELS__IMPORT_MODULES - Semicolon-separated list of Python modules to import
1130
+ Example: "models;myapp.entities;myapp.custom_models"
1131
+
1132
+ Example:
1133
+ # In downstream app's .env
1134
+ MODELS__IMPORT_MODULES=models
1135
+
1136
+ # In downstream app's models/__init__.py
1137
+ import rem
1138
+ from rem.models.core import CoreModel
1139
+
1140
+ @rem.register_model
1141
+ class MyCustomEntity(CoreModel):
1142
+ name: str
1143
+
1144
+ # Then run schema generation
1145
+ rem db schema generate # Includes MyCustomEntity
1146
+ """
1147
+
1148
+ model_config = SettingsConfigDict(
1149
+ env_prefix="MODELS__",
1150
+ extra="ignore",
1151
+ )
1152
+
1153
+ import_modules: str = Field(
1154
+ default="",
1155
+ description=(
1156
+ "Semicolon-separated list of Python modules to import for model registration. "
1157
+ "These modules are imported before schema generation to ensure custom models "
1158
+ "decorated with @rem.register_model are discovered. "
1159
+ "Example: 'models;myapp.entities'"
1160
+ ),
1161
+ )
1162
+
1163
+ @property
1164
+ def module_list(self) -> list[str]:
1165
+ """
1166
+ Get modules as a list, filtering empty strings.
1167
+
1168
+ Auto-detects ./models folder if it exists and is importable.
1169
+ """
1170
+ modules = []
1171
+ if self.import_modules:
1172
+ modules = [m.strip() for m in self.import_modules.split(";") if m.strip()]
1173
+
1174
+ # Auto-detect ./models if it exists and is a Python package (convention over configuration)
1175
+ from pathlib import Path
1176
+
1177
+ models_path = Path("./models")
1178
+ if models_path.exists() and models_path.is_dir():
1179
+ # Check if it's a Python package (has __init__.py)
1180
+ if (models_path / "__init__.py").exists():
1181
+ if "models" not in modules:
1182
+ modules.insert(0, "models")
1183
+
1184
+ return modules
1185
+
982
1186
 
983
1187
  class SchemaSettings(BaseSettings):
984
1188
  """
@@ -1163,6 +1367,276 @@ class GitSettings(BaseSettings):
1163
1367
  )
1164
1368
 
1165
1369
 
1370
+ class DBListenerSettings(BaseSettings):
1371
+ """
1372
+ PostgreSQL LISTEN/NOTIFY database listener settings.
1373
+
1374
+ The DB Listener is a lightweight worker that subscribes to PostgreSQL
1375
+ NOTIFY events and dispatches them to external systems (SQS, REST, custom).
1376
+
1377
+ Architecture:
1378
+ - Single-replica deployment (to avoid duplicate processing)
1379
+ - Dedicated connection for LISTEN (not from connection pool)
1380
+ - Automatic reconnection with exponential backoff
1381
+ - Graceful shutdown on SIGTERM
1382
+
1383
+ Use Cases:
1384
+ - Sync data changes to external systems (Phoenix, webhooks)
1385
+ - Trigger async jobs without polling
1386
+ - Event-driven architectures with PostgreSQL as event source
1387
+
1388
+ Example PostgreSQL trigger:
1389
+ CREATE OR REPLACE FUNCTION notify_feedback_insert()
1390
+ RETURNS TRIGGER AS $$
1391
+ BEGIN
1392
+ PERFORM pg_notify('feedback_sync', json_build_object(
1393
+ 'id', NEW.id,
1394
+ 'table', 'feedbacks',
1395
+ 'action', 'insert'
1396
+ )::text);
1397
+ RETURN NEW;
1398
+ END;
1399
+ $$ LANGUAGE plpgsql;
1400
+
1401
+ Environment variables:
1402
+ DB_LISTENER__ENABLED - Enable the listener worker (default: false)
1403
+ DB_LISTENER__CHANNELS - Comma-separated PostgreSQL channels to listen on
1404
+ DB_LISTENER__HANDLER_TYPE - Handler type: 'sqs', 'rest', or 'custom'
1405
+ DB_LISTENER__SQS_QUEUE_URL - SQS queue URL (for handler_type=sqs)
1406
+ DB_LISTENER__REST_ENDPOINT - REST endpoint URL (for handler_type=rest)
1407
+ DB_LISTENER__RECONNECT_DELAY - Initial reconnect delay in seconds
1408
+ DB_LISTENER__MAX_RECONNECT_DELAY - Maximum reconnect delay in seconds
1409
+
1410
+ References:
1411
+ - PostgreSQL NOTIFY: https://www.postgresql.org/docs/current/sql-notify.html
1412
+ - Brandur's Notifier: https://brandur.org/notifier
1413
+ """
1414
+
1415
+ model_config = SettingsConfigDict(
1416
+ env_prefix="DB_LISTENER__",
1417
+ env_file=".env",
1418
+ env_file_encoding="utf-8",
1419
+ extra="ignore",
1420
+ )
1421
+
1422
+ enabled: bool = Field(
1423
+ default=False,
1424
+ description="Enable the DB Listener worker (disabled by default)",
1425
+ )
1426
+
1427
+ channels: str = Field(
1428
+ default="",
1429
+ description=(
1430
+ "Comma-separated list of PostgreSQL channels to LISTEN on. "
1431
+ "Example: 'feedback_sync,entity_update,user_events'"
1432
+ ),
1433
+ )
1434
+
1435
+ handler_type: str = Field(
1436
+ default="rest",
1437
+ description=(
1438
+ "Handler type for dispatching notifications. Options: "
1439
+ "'sqs' (publish to SQS), 'rest' (POST to endpoint), 'custom' (Python handlers)"
1440
+ ),
1441
+ )
1442
+
1443
+ sqs_queue_url: str = Field(
1444
+ default="",
1445
+ description="SQS queue URL for handler_type='sqs'",
1446
+ )
1447
+
1448
+ rest_endpoint: str = Field(
1449
+ default="http://localhost:8000/api/v1/internal/events",
1450
+ description=(
1451
+ "REST endpoint URL for handler_type='rest'. "
1452
+ "Receives POST with {channel, payload, source} JSON body."
1453
+ ),
1454
+ )
1455
+
1456
+ reconnect_delay: float = Field(
1457
+ default=1.0,
1458
+ description="Initial delay (seconds) between reconnection attempts",
1459
+ )
1460
+
1461
+ max_reconnect_delay: float = Field(
1462
+ default=60.0,
1463
+ description="Maximum delay (seconds) between reconnection attempts (exponential backoff cap)",
1464
+ )
1465
+
1466
+ @property
1467
+ def channel_list(self) -> list[str]:
1468
+ """Get channels as a list, filtering empty strings."""
1469
+ if not self.channels:
1470
+ return []
1471
+ return [c.strip() for c in self.channels.split(",") if c.strip()]
1472
+
1473
+
1474
+ class EmailSettings(BaseSettings):
1475
+ """
1476
+ Email service settings for SMTP.
1477
+
1478
+ Supports passwordless login via email codes and transactional emails.
1479
+ Uses Gmail SMTP with App Passwords by default.
1480
+
1481
+ Generate app password at: https://myaccount.google.com/apppasswords
1482
+
1483
+ Environment variables:
1484
+ EMAIL__ENABLED - Enable email service (default: false)
1485
+ EMAIL__SMTP_HOST - SMTP server host (default: smtp.gmail.com)
1486
+ EMAIL__SMTP_PORT - SMTP server port (default: 587 for TLS)
1487
+ EMAIL__SENDER_EMAIL - Sender email address
1488
+ EMAIL__SENDER_NAME - Sender display name
1489
+ EMAIL__APP_PASSWORD - Gmail app password (from secrets)
1490
+ EMAIL__USE_TLS - Use TLS encryption (default: true)
1491
+ EMAIL__LOGIN_CODE_EXPIRY_MINUTES - Login code expiry (default: 10)
1492
+
1493
+ Branding environment variables (for email templates):
1494
+ EMAIL__APP_NAME - Application name in emails (default: REM)
1495
+ EMAIL__LOGO_URL - Logo URL for email templates (40x40 recommended)
1496
+ EMAIL__TAGLINE - Tagline shown in email footer
1497
+ EMAIL__WEBSITE_URL - Main website URL for email links
1498
+ EMAIL__PRIVACY_URL - Privacy policy URL for email footer
1499
+ EMAIL__TERMS_URL - Terms of service URL for email footer
1500
+ """
1501
+
1502
+ model_config = SettingsConfigDict(
1503
+ env_prefix="EMAIL__",
1504
+ env_file=".env",
1505
+ env_file_encoding="utf-8",
1506
+ extra="ignore",
1507
+ )
1508
+
1509
+ enabled: bool = Field(
1510
+ default=False,
1511
+ description="Enable email service (requires app_password to be set)",
1512
+ )
1513
+
1514
+ smtp_host: str = Field(
1515
+ default="smtp.gmail.com",
1516
+ description="SMTP server host",
1517
+ )
1518
+
1519
+ smtp_port: int = Field(
1520
+ default=587,
1521
+ description="SMTP server port (587 for TLS, 465 for SSL)",
1522
+ )
1523
+
1524
+ sender_email: str = Field(
1525
+ default="",
1526
+ description="Sender email address",
1527
+ )
1528
+
1529
+ sender_name: str = Field(
1530
+ default="REM",
1531
+ description="Sender display name",
1532
+ )
1533
+
1534
+ # Branding settings for email templates
1535
+ app_name: str = Field(
1536
+ default="REM",
1537
+ description="Application name shown in email templates",
1538
+ )
1539
+
1540
+ logo_url: str | None = Field(
1541
+ default=None,
1542
+ description="Logo URL for email templates (40x40 recommended)",
1543
+ )
1544
+
1545
+ tagline: str = Field(
1546
+ default="Your AI-powered platform",
1547
+ description="Tagline shown in email footer",
1548
+ )
1549
+
1550
+ website_url: str = Field(
1551
+ default="https://rem.ai",
1552
+ description="Main website URL for email links",
1553
+ )
1554
+
1555
+ privacy_url: str = Field(
1556
+ default="https://rem.ai/privacy",
1557
+ description="Privacy policy URL for email footer",
1558
+ )
1559
+
1560
+ terms_url: str = Field(
1561
+ default="https://rem.ai/terms",
1562
+ description="Terms of service URL for email footer",
1563
+ )
1564
+
1565
+ app_password: str | None = Field(
1566
+ default=None,
1567
+ description="Gmail app password for SMTP authentication",
1568
+ )
1569
+
1570
+ use_tls: bool = Field(
1571
+ default=True,
1572
+ description="Use TLS encryption for SMTP",
1573
+ )
1574
+
1575
+ login_code_expiry_minutes: int = Field(
1576
+ default=10,
1577
+ description="Login code expiry in minutes",
1578
+ )
1579
+
1580
+ trusted_email_domains: str = Field(
1581
+ default="",
1582
+ description=(
1583
+ "Comma-separated list of trusted email domains for new user registration. "
1584
+ "Existing users can always login regardless of domain. "
1585
+ "New users must have an email from a trusted domain. "
1586
+ "Empty string means all domains are allowed. "
1587
+ "Example: 'siggymd.ai,example.com'"
1588
+ ),
1589
+ )
1590
+
1591
+ @property
1592
+ def trusted_domain_list(self) -> list[str]:
1593
+ """Get trusted domains as a list, filtering empty strings."""
1594
+ if not self.trusted_email_domains:
1595
+ return []
1596
+ return [d.strip().lower() for d in self.trusted_email_domains.split(",") if d.strip()]
1597
+
1598
+ def is_domain_trusted(self, email: str) -> bool:
1599
+ """Check if an email's domain is in the trusted list.
1600
+
1601
+ Args:
1602
+ email: Email address to check
1603
+
1604
+ Returns:
1605
+ True if domain is trusted (or if no trusted domains configured)
1606
+ """
1607
+ domains = self.trusted_domain_list
1608
+ if not domains:
1609
+ # No restrictions configured
1610
+ return True
1611
+
1612
+ email_domain = email.lower().split("@")[-1].strip()
1613
+ return email_domain in domains
1614
+
1615
+ @property
1616
+ def is_configured(self) -> bool:
1617
+ """Check if email service is properly configured."""
1618
+ return bool(self.sender_email and self.app_password)
1619
+
1620
+ @property
1621
+ def template_kwargs(self) -> dict:
1622
+ """
1623
+ Get branding kwargs for email templates.
1624
+
1625
+ Returns a dict that can be passed to template functions:
1626
+ login_code_template(..., **settings.email.template_kwargs)
1627
+ """
1628
+ kwargs = {
1629
+ "app_name": self.app_name,
1630
+ "tagline": self.tagline,
1631
+ "website_url": self.website_url,
1632
+ "privacy_url": self.privacy_url,
1633
+ "terms_url": self.terms_url,
1634
+ }
1635
+ if self.logo_url:
1636
+ kwargs["logo_url"] = self.logo_url
1637
+ return kwargs
1638
+
1639
+
1166
1640
  class TestSettings(BaseSettings):
1167
1641
  """
1168
1642
  Test environment settings.
@@ -1232,6 +1706,11 @@ class Settings(BaseSettings):
1232
1706
  extra="ignore",
1233
1707
  )
1234
1708
 
1709
+ app_name: str = Field(
1710
+ default="REM",
1711
+ description="Application/API name used in docs, titles, and user-facing text",
1712
+ )
1713
+
1235
1714
  team: str = Field(
1236
1715
  default="rem",
1237
1716
  description="Team or project name for observability",
@@ -1252,16 +1731,12 @@ class Settings(BaseSettings):
1252
1731
  description="Root path for reverse proxy (e.g., /rem for ALB routing)",
1253
1732
  )
1254
1733
 
1255
- sql_dir: str = Field(
1256
- default="src/rem/sql",
1257
- description="Directory for SQL files and migrations",
1258
- )
1259
-
1260
1734
  # Nested settings groups
1261
1735
  api: APISettings = Field(default_factory=APISettings)
1262
1736
  chat: ChatSettings = Field(default_factory=ChatSettings)
1263
1737
  llm: LLMSettings = Field(default_factory=LLMSettings)
1264
1738
  mcp: MCPSettings = Field(default_factory=MCPSettings)
1739
+ models: ModelsSettings = Field(default_factory=ModelsSettings)
1265
1740
  otel: OTELSettings = Field(default_factory=OTELSettings)
1266
1741
  phoenix: PhoenixSettings = Field(default_factory=PhoenixSettings)
1267
1742
  auth: AuthSettings = Field(default_factory=AuthSettings)
@@ -1269,18 +1744,31 @@ class Settings(BaseSettings):
1269
1744
  migration: MigrationSettings = Field(default_factory=MigrationSettings)
1270
1745
  storage: StorageSettings = Field(default_factory=StorageSettings)
1271
1746
  s3: S3Settings = Field(default_factory=S3Settings)
1747
+ data_lake: DataLakeSettings = Field(default_factory=DataLakeSettings)
1272
1748
  git: GitSettings = Field(default_factory=GitSettings)
1273
1749
  sqs: SQSSettings = Field(default_factory=SQSSettings)
1750
+ db_listener: DBListenerSettings = Field(default_factory=DBListenerSettings)
1274
1751
  chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
1275
1752
  content: ContentSettings = Field(default_factory=ContentSettings)
1276
1753
  schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
1754
+ email: EmailSettings = Field(default_factory=EmailSettings)
1277
1755
  test: TestSettings = Field(default_factory=TestSettings)
1278
1756
 
1279
1757
 
1758
+ # Auto-load .env file from current directory if it exists
1759
+ # This happens BEFORE config file loading, so .env takes precedence
1760
+ from pathlib import Path
1761
+ from dotenv import load_dotenv
1762
+
1763
+ _dotenv_path = Path(".env")
1764
+ if _dotenv_path.exists():
1765
+ load_dotenv(_dotenv_path, override=False) # Don't override existing env vars
1766
+ logger.debug(f"Loaded environment from {_dotenv_path.resolve()}")
1767
+
1280
1768
  # Load configuration from ~/.rem/config.yaml before initializing settings
1281
1769
  # This allows user configuration to be merged with environment variables
1282
- # Set REM_SKIP_CONFIG_FILE=true to disable (useful for development with .env)
1283
- if not os.getenv("REM_SKIP_CONFIG_FILE", "").lower() in ("true", "1", "yes"):
1770
+ # Set REM_SKIP_CONFIG=1 to disable (useful for development with .env)
1771
+ if not os.getenv("REM_SKIP_CONFIG", "").lower() in ("true", "1", "yes"):
1284
1772
  try:
1285
1773
  from rem.config import load_config, merge_config_to_env
1286
1774