remdb 0.2.6__py3-none-any.whl → 0.3.118__py3-none-any.whl

rem/cli/commands/README.md

@@ -1,5 +1,191 @@
 # REM CLI Commands
 
+## Database Management (`rem db`)
+
+REM uses a **code-as-source-of-truth** approach to database schema management. Pydantic models define the schema, and the database is kept in sync via diff-based migrations.
+
+### Quick Reference
+
+```bash
+rem db schema generate   # Regenerate schema SQL from registered models
+rem db diff              # Compare models vs database (detect drift)
+rem db diff --check      # CI mode: exit 1 if drift detected
+rem db apply <file>      # Apply SQL file to database
+```
+
+### Schema Management Workflow
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                    Source of Truth                          │
+│                                                             │
+│   Pydantic Models (CoreModel subclasses)                   │
+│   └── src/rem/models/entities/*.py                         │
+│                                                             │
+│   Model Registry                                            │
+│   └── Core models auto-registered                          │
+│   └── Custom models via @rem.register_model                │
+└─────────────────────────────────────────────────────────────┘
+                           │
+                           ▼
+┌─────────────────────────────────────────────────────────────┐
+│                 rem db schema generate                      │
+│                                                             │
+│   Generates SQL from registry → 002_install_models.sql     │
+└─────────────────────────────────────────────────────────────┘
+                           │
+                           ▼
+┌─────────────────────────────────────────────────────────────┐
+│                      rem db diff                            │
+│                                                             │
+│   Compares models ↔ database using Alembic autogenerate    │
+│   Shows: + additions, - removals, ~ modifications          │
+└─────────────────────────────────────────────────────────────┘
+                           │
+                           ▼
+┌─────────────────────────────────────────────────────────────┐
+│                    rem db apply                             │
+│                                                             │
+│   Executes SQL directly against database                    │
+│   Optionally logs to rem_migrations (audit only)           │
+└─────────────────────────────────────────────────────────────┘
+```
+
+### File Structure
+
+```
+src/rem/sql/
+├── migrations/
+│   ├── 001_install.sql          # Core infrastructure (manual)
+│   └── 002_install_models.sql   # Entity tables (auto-generated)
+└── background_indexes.sql       # HNSW vector indexes (optional)
+```
+
+**Key principle**: Only two migration files. No incremental `003_`, `004_` files. The models file is always regenerated to match code.
+
+### Commands
+
+#### `rem db schema generate`
+
+Regenerate `002_install_models.sql` from the model registry:
+
+```bash
+rem db schema generate
+```
+
+This reads all registered models (core + custom) and generates:
+- CREATE TABLE statements for each entity
+- Embeddings tables (`embeddings_<table>`)
+- KV_STORE triggers for cache maintenance
+- Foreground indexes (GIN for JSONB, B-tree for lookups)
+
+#### `rem db diff`
+
+Compare Pydantic models against the live database:
+
+```bash
+# Show differences
+rem db diff
+
+# CI mode: exit 1 if drift detected
+rem db diff --check
+```
+
+Output shows:
+- `+ ADD COLUMN` - Column exists in model but not in DB
+- `- DROP COLUMN` - Column exists in DB but not in model
+- `~ ALTER COLUMN` - Column type or constraints differ
+- `+ CREATE TABLE` - Table exists in model but not in DB
+- `- DROP TABLE` - Table exists in DB but not in model
+
+#### `rem db apply`
+
+Apply a SQL file directly to the database:
+
+```bash
+# Apply with audit logging (default)
+rem db apply src/rem/sql/migrations/002_install_models.sql
+
+# Preview without executing
+rem db apply --dry-run src/rem/sql/migrations/002_install_models.sql
+
+# Apply without logging to rem_migrations
+rem db apply --no-log src/rem/sql/migrations/002_install_models.sql
+```
+
+### Typical Workflows
+
+#### Initial Setup
+
+```bash
+# 1. Generate schema from models
+rem db schema generate
+
+# 2. Apply infrastructure (extensions, kv_store)
+rem db apply src/rem/sql/migrations/001_install.sql
+
+# 3. Apply entity tables
+rem db apply src/rem/sql/migrations/002_install_models.sql
+
+# 4. Verify no drift
+rem db diff
+```
+
+#### Adding a New Entity
+
+```bash
+# 1. Create model in src/rem/models/entities/
+# 2. Register in src/rem/registry.py (add to core_models list)
+# 3. Regenerate schema
+rem db schema generate
+
+# 4. Check what changed
+rem db diff
+
+# 5. Apply changes
+rem db apply src/rem/sql/migrations/002_install_models.sql
+```
+
+#### Modifying an Existing Entity
+
+```bash
+# 1. Modify model in src/rem/models/entities/
+# 2. Regenerate schema
+rem db schema generate
+
+# 3. Check what changed
+rem db diff
+
+# 4. Apply changes (idempotent - uses IF NOT EXISTS)
+rem db apply src/rem/sql/migrations/002_install_models.sql
+```
+
+#### CI/CD Pipeline
+
+```bash
+# Fail build if schema drift detected
+rem db diff --check
+```
+
+### Registering Custom Models
+
+```python
+import rem
+from rem.models.core import CoreModel
+
+@rem.register_model
+class MyEntity(CoreModel):
+    name: str
+    description: str  # Auto-embeds (content field)
+
+# Or with options:
+@rem.register_model(table_name="custom_table")
+class AnotherEntity(CoreModel):
+    title: str
+```
+
+---
+
 ## Configuration (`rem configure`)
 
 Interactive configuration wizard for REM setup.
@@ -13,114 +199,101 @@ rem configure
 # Configure + install database tables
 rem configure --install
 
-# Configure + install + register with Claude Desktop
-rem configure --install --claude-desktop
-```
-
-### Managing Configuration
-
-```bash
 # View current configuration
 rem configure --show
 
 # Edit configuration file
-rem configure --edit  # Opens in $EDITOR (defaults to vim)
-
-# Or edit manually
-vim ~/.rem/config.yaml
+rem configure --edit
 ```
 
-### Configuration File Structure
+### Configuration File
 
 `~/.rem/config.yaml`:
 
 ```yaml
 postgres:
   connection_string: postgresql://user:pass@localhost:5432/rem
-  pool_min_size: 5
-  pool_max_size: 20
 
 llm:
   default_model: anthropic:claude-sonnet-4-5-20250929
-  default_temperature: 0.5
   openai_api_key: sk-...
   anthropic_api_key: sk-ant-...
 
 s3:
   bucket_name: rem-storage
   region: us-east-1
-  # Optional: for MinIO/LocalStack
-  endpoint_url: http://localhost:9000
-  access_key_id: minioadmin
-  secret_access_key: minioadmin
 ```
 
 ### Environment Variables
 
-All configuration can be overridden via environment variables using double underscore delimiter:
+All configuration can be overridden via environment variables:
 
 ```bash
-# Postgres
 export POSTGRES__CONNECTION_STRING=postgresql://user:pass@host:5432/db
-export POSTGRES__POOL_MIN_SIZE=5
-export POSTGRES__POOL_MAX_SIZE=20
-
-# LLM
 export LLM__DEFAULT_MODEL=anthropic:claude-sonnet-4-5-20250929
 export LLM__OPENAI_API_KEY=sk-...
-export LLM__ANTHROPIC_API_KEY=sk-ant-...
-
-# S3
-export S3__BUCKET_NAME=rem-storage
-export S3__REGION=us-east-1
 ```
 
-### Configuration Precedence
+**Precedence**: Environment variables > Config file > Defaults
 
-1. **Environment variables** (highest priority)
-2. **Configuration file** (`~/.rem/config.yaml`)
-3. **Default values** (from `rem/settings.py`)
+---
 
-### Docker/Kubernetes
+## Cluster Management (`rem cluster`)
 
-In containerized environments, use environment variables exclusively:
+Commands for deploying REM to Kubernetes.
 
-```yaml
-# docker-compose.yml
-services:
-  rem-api:
-    image: rem:latest
-    environment:
-      POSTGRES__CONNECTION_STRING: postgresql://rem:rem@postgres:5432/rem
-      LLM__OPENAI_API_KEY: ${OPENAI_API_KEY}
+### Quick Reference
+
+```bash
+rem cluster init           # Initialize cluster config
+rem cluster generate       # Generate all manifests (ArgoCD, ConfigMaps, etc.)
+rem cluster setup-ssm      # Create required SSM parameters in AWS
+rem cluster validate       # Validate deployment prerequisites
+rem cluster env check      # Validate .env for cluster deployment
 ```
 
-```yaml
-# Kubernetes ConfigMap/Secret
-apiVersion: v1
-kind: Secret
-metadata:
-  name: rem-secrets
-stringData:
-  POSTGRES__CONNECTION_STRING: postgresql://rem:rem@postgres:5432/rem
-  LLM__OPENAI_API_KEY: sk-...
+### `rem cluster generate`
+
+Generates all Kubernetes manifests from cluster config:
+
+```bash
+rem cluster generate
 ```
 
+This generates/updates:
+- ArgoCD Application manifests
+- ClusterSecretStore configurations
+- SQL init ConfigMap (from `rem/sql/migrations/*.sql`)
+
+The SQL ConfigMap is used by CloudNativePG for database initialization on first cluster bootstrap.
+
+### `rem cluster env`
+
+Environment configuration management:
+
+```bash
+rem cluster env check              # Validate .env for staging
+rem cluster env check --env prod   # Validate for production
+rem cluster env generate           # Generate ConfigMap from .env
+rem cluster env diff               # Compare .env with cluster ConfigMap
+```
+
+---
+
 ## Other Commands
 
-- **`rem ask`** - Interactive chat with REM memory
-- **`rem serve`** - Start FastAPI server
-- **`rem db`** - Database management (migrate, seed, etc.)
-- **`rem schema`** - Schema generation and validation
-- **`rem mcp`** - MCP server commands
-- **`rem dreaming`** - Background knowledge processing
-- **`rem process`** - File processing utilities
-- **`rem experiments`** - Experiment management (datasets, prompts, traces, runs)
+| Command | Description |
+|---------|-------------|
+| `rem ask` | Interactive chat with REM agents |
+| `rem serve` | Start FastAPI server |
+| `rem mcp` | MCP server commands |
+| `rem dreaming` | Background knowledge processing |
+| `rem process` | File processing utilities |
 
-Run `rem COMMAND --help` for detailed usage of each command.
+Run `rem COMMAND --help` for detailed usage.
 
 ## See Also
 
-- [README.md](../../../../../README.md) - Main documentation
+- [rem/README.md](../../../../../README.md) - Main documentation
 - [CLAUDE.md](../../../../../CLAUDE.md) - Architecture overview
-- [settings.py](../../settings.py) - All available settings
+- [postgres/README.md](../../services/postgres/README.md) - Database service details

rem/cli/commands/ask.py

@@ -89,8 +89,8 @@ async def run_agent_streaming(
         context: Optional AgentContext for session persistence
         max_iterations: Maximum iterations/requests (from agent schema or settings)
     """
-    from datetime import datetime, timezone
     from pydantic_ai import UsageLimits
+    from rem.utils.date_utils import to_iso_with_z, utc_now
 
     logger.info("Running agent in streaming mode...")
 
@@ -151,13 +151,13 @@ async def run_agent_streaming(
             user_message = {
                 "role": "user",
                 "content": user_message_content,
-                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "timestamp": to_iso_with_z(utc_now()),
             }
 
             assistant_message = {
                 "role": "assistant",
                 "content": "".join(assistant_response_parts),
-                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "timestamp": to_iso_with_z(utc_now()),
             }
 
             # Store messages with compression
@@ -200,8 +200,8 @@ async def run_agent_non_streaming(
     Returns:
         Output data if successful, None otherwise
     """
-    from datetime import datetime, timezone
     from pydantic_ai import UsageLimits
+    from rem.utils.date_utils import to_iso_with_z, utc_now
 
     logger.info("Running agent in non-streaming mode...")
 
@@ -248,13 +248,13 @@ async def run_agent_non_streaming(
             user_message = {
                 "role": "user",
                 "content": user_message_content,
-                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "timestamp": to_iso_with_z(utc_now()),
             }
 
             assistant_message = {
                 "role": "assistant",
                 "content": assistant_content,
-                "timestamp": datetime.now(timezone.utc).isoformat(),
+                "timestamp": to_iso_with_z(utc_now()),
             }
 
             # Store messages with compression
@@ -357,8 +357,8 @@ async def _save_output_file(file_path: Path, data: dict[str, Any]) -> None:
 )
 @click.option(
     "--user-id",
-    default="test-user",
-    help="User ID for context (default: test-user)",
+    default=None,
+    help="User ID for context (default: from settings.test.effective_user_id)",
 )
 @click.option(
     "--session-id",
@@ -393,7 +393,7 @@ def ask(
     max_turns: int,
     version: str | None,
     stream: bool,
-    user_id: str,
+    user_id: str | None,
     session_id: str | None,
     input_file: Path | None,
     output_file: Path | None,
@@ -434,6 +434,9 @@ def ask(
         # Two arguments provided
         name = name_or_query
 
+    # Resolve user_id from settings if not provided
+    effective_user_id = user_id or settings.test.effective_user_id
+
     asyncio.run(
         _ask_async(
             name=name,
@@ -443,7 +446,7 @@ def ask(
             max_turns=max_turns,
             version=version,
             stream=stream,
-            user_id=user_id,
+            user_id=effective_user_id,
             session_id=session_id,
             input_file=input_file,
             output_file=output_file,
@@ -486,9 +489,10 @@ async def _ask_async(
 
     # Load schema using centralized utility
     # Handles both file paths and schema names automatically
+    # Falls back to database LOOKUP if not found in filesystem
     logger.info(f"Loading schema: {name} (version: {version or 'latest'})")
     try:
-        schema = load_agent_schema(name)
+        schema = load_agent_schema(name, user_id=user_id)
     except FileNotFoundError as e:
         logger.error(str(e))
         sys.exit(1)