remdb 0.3.7__py3-none-any.whl → 0.3.14__py3-none-any.whl

rem/workers/db_maintainer.py

@@ -0,0 +1,74 @@
+"""
+Database Maintainer Worker.
+
+Handles background maintenance tasks for PostgreSQL:
+1. Cleaning up expired rate limit counters (UNLOGGED table).
+2. Refreshing materialized views (if any).
+3. Vacuuming specific tables (if needed).
+
+Usage:
+    python -m rem.workers.db_maintainer
+    
+    # Or via docker-compose:
+    # command: python -m rem.workers.db_maintainer
+"""
+
+import asyncio
+import signal
+from loguru import logger
+
+from ..services.postgres.service import PostgresService
+from ..services.rate_limit import RateLimitService
+
+class DatabaseMaintainer:
+    def __init__(self):
+        self.running = False
+        self.db = PostgresService()
+        self.rate_limiter = RateLimitService(self.db)
+
+    async def start(self):
+        """Start maintenance loop."""
+        self.running = True
+        logger.info("Starting Database Maintainer Worker")
+        
+        await self.db.connect()
+        
+        try:
+            while self.running:
+                await self._run_maintenance_cycle()
+                # Sleep for 5 minutes
+                await asyncio.sleep(300)
+        finally:
+            await self.db.disconnect()
+
+    async def _run_maintenance_cycle(self):
+        """Execute maintenance tasks."""
+        logger.debug("Running maintenance cycle...")
+        
+        try:
+            # 1. Cleanup Rate Limits
+            await self.rate_limiter.cleanup_expired()
+            
+            # 2. (Future) Refresh Views
+            # await self.db.execute("REFRESH MATERIALIZED VIEW ...")
+            
+        except Exception as e:
+            logger.error(f"Maintenance cycle failed: {e}")
+
+    def stop(self):
+        """Stop worker gracefully."""
+        self.running = False
+        logger.info("Stopping Database Maintainer Worker...")
+
+async def main():
+    worker = DatabaseMaintainer()
+    
+    # Handle signals
+    loop = asyncio.get_running_loop()
+    for sig in (signal.SIGTERM, signal.SIGINT):
+        loop.add_signal_handler(sig, worker.stop)
+        
+    await worker.start()
+
+if __name__ == "__main__":
+    asyncio.run(main())

{remdb-0.3.7.dist-info → remdb-0.3.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: remdb
-Version: 0.3.7
+Version: 0.3.14
 Summary: Resources Entities Moments - Bio-inspired memory system for agentic AI workloads
 Project-URL: Homepage, https://github.com/Percolation-Labs/reminiscent
 Project-URL: Documentation, https://github.com/Percolation-Labs/reminiscent/blob/main/README.md
@@ -23,11 +23,10 @@ Requires-Dist: click>=8.1.0
 Requires-Dist: fastapi>=0.115.0
 Requires-Dist: fastmcp>=0.5.0
 Requires-Dist: gitpython>=3.1.45
-Requires-Dist: gmft
 Requires-Dist: hypercorn>=0.17.0
 Requires-Dist: itsdangerous>=2.0.0
 Requires-Dist: json-schema-to-pydantic>=0.2.0
-Requires-Dist: kreuzberg>=3.21.0
+Requires-Dist: kreuzberg<4.0.0,>=3.21.0
 Requires-Dist: loguru>=0.7.0
 Requires-Dist: openinference-instrumentation-pydantic-ai>=0.1.0
 Requires-Dist: opentelemetry-api>=1.28.0
@@ -48,7 +47,6 @@ Requires-Dist: requests>=2.32.0
 Requires-Dist: semchunk>=2.2.0
 Requires-Dist: tenacity>=9.0.0
 Requires-Dist: tiktoken>=0.5.0
-Requires-Dist: torch>=2.0.0
 Requires-Dist: uvicorn[standard]>=0.32.0
 Provides-Extra: all
 Requires-Dist: ipdb>=0.13.0; extra == 'all'
@@ -137,59 +135,50 @@ Choose your path:
 **Best for**: First-time users who want to explore REM with curated example datasets.
 
 ```bash
-# Install system dependencies
-# macOS:
-brew install tesseract
-
-# Linux:
-sudo apt-get install tesseract-ocr
+# Install system dependencies (tesseract for OCR)
+brew install tesseract  # macOS (Linux/Windows: see tesseract-ocr.github.io)
 
 # Install remdb
-pip install remdb[all]
+pip install "remdb[all]"
 
 # Clone example datasets
 git clone https://github.com/Percolation-Labs/remstack-lab.git
 cd remstack-lab
 
-# Configure REM (interactive wizard)
-rem configure --install
+# Start PostgreSQL with docker-compose
+curl -O https://gist.githubusercontent.com/percolating-sirsh/d117b673bc0edfdef1a5068ccd3cf3e5/raw/docker-compose.prebuilt.yml
+docker compose -f docker-compose.prebuilt.yml up -d postgres
 
-# Start PostgreSQL
-docker run -d \
-  --name rem-postgres \
-  -e POSTGRES_USER=rem \
-  -e POSTGRES_PASSWORD=rem \
-  -e POSTGRES_DB=rem \
-  -p 5050:5432 \
-  pgvector/pgvector:pg18
+# Configure REM (creates ~/.rem/config.yaml and installs database schema)
+# Add --claude-desktop to register with Claude Desktop app
+rem configure --install --claude-desktop
 
-# Load quickstart dataset
-rem db load datasets/quickstart/sample_data.yaml --user-id demo-user
+# Load quickstart dataset (uses default user)
+rem db load datasets/quickstart/sample_data.yaml
 
 # Optional: Set default LLM provider via environment variable
 # export LLM__DEFAULT_MODEL="openai:gpt-4.1-nano"  # Fast and cheap
 # export LLM__DEFAULT_MODEL="anthropic:claude-sonnet-4-5-20250929"  # High quality (default)
 
 # Ask questions
-rem ask --user-id demo-user "What documents exist in the system?"
-rem ask --user-id demo-user "Show me meetings about API design"
+rem ask "What documents exist in the system?"
+rem ask "Show me meetings about API design"
 
 # Ingest files (PDF, DOCX, images, etc.) - note: requires remstack-lab
-rem process ingest datasets/formats/files/bitcoin_whitepaper.pdf --user-id demo-user --category research --tags bitcoin,whitepaper
+rem process ingest datasets/formats/files/bitcoin_whitepaper.pdf --category research --tags bitcoin,whitepaper
 
 # Query ingested content
-rem ask --user-id demo-user "What is the Bitcoin whitepaper about?"
+rem ask "What is the Bitcoin whitepaper about?"
 
-# Try other datasets
-rem db load --file datasets/domains/recruitment/scenarios/candidate_pipeline/data.yaml --user-id my-company
-rem ask --user-id my-company "Show me candidates with Python experience"
+# Try other datasets (use --user-id for multi-tenant scenarios)
+rem db load datasets/domains/recruitment/scenarios/candidate_pipeline/data.yaml --user-id acme-corp
+rem ask --user-id acme-corp "Show me candidates with Python experience"
 ```
 
 **What you get:**
 - Quickstart: 3 users, 3 resources, 3 moments, 4 messages
 - Domain datasets: recruitment, legal, enterprise, misc
 - Format examples: engrams, documents, conversations, files
-- Jupyter notebooks and experiments
 
 **Learn more**: [remstack-lab repository](https://github.com/Percolation-Labs/remstack-lab)
 
@@ -270,28 +259,28 @@ Configuration saved to `~/.rem/config.yaml` (can edit with `rem configure --edit
 # Clone datasets repository
 git clone https://github.com/Percolation-Labs/remstack-lab.git
 
-# Load quickstart dataset
-rem db load --file remstack-lab/datasets/quickstart/sample_data.yaml --user-id demo-user
+# Load quickstart dataset (uses default user)
+rem db load --file remstack-lab/datasets/quickstart/sample_data.yaml
 
 # Test with sample queries
-rem ask --user-id demo-user "What documents exist in the system?"
-rem ask --user-id demo-user "Show me meetings about API design"
-rem ask --user-id demo-user "Who is Sarah Chen?"
+rem ask "What documents exist in the system?"
+rem ask "Show me meetings about API design"
+rem ask "Who is Sarah Chen?"
 
-# Try domain-specific datasets
-rem db load --file remstack-lab/datasets/domains/recruitment/scenarios/candidate_pipeline/data.yaml --user-id my-company
-rem ask --user-id my-company "Show me candidates with Python experience"
+# Try domain-specific datasets (use --user-id for multi-tenant scenarios)
+rem db load --file remstack-lab/datasets/domains/recruitment/scenarios/candidate_pipeline/data.yaml --user-id acme-corp
+rem ask --user-id acme-corp "Show me candidates with Python experience"
 ```
 
 **Option B: Bring your own data**
 
 ```bash
-# Ingest your own files
+# Ingest your own files (uses default user)
 echo "REM is a bio-inspired memory system for agentic AI workloads." > test-doc.txt
-rem process ingest test-doc.txt --user-id test-user --category documentation --tags rem,ai
+rem process ingest test-doc.txt --category documentation --tags rem,ai
 
 # Query your ingested data
-rem ask --user-id test-user "What do you know about REM from my knowledge base?"
+rem ask "What do you know about REM from my knowledge base?"
 ```
 
 ### Step 4: Test the API
@@ -328,13 +317,13 @@ curl -X POST http://localhost:8000/api/v1/chat/completions \
 ```bash
 cd remstack-lab
 
-# Load any dataset
-rem db load --file datasets/quickstart/sample_data.yaml --user-id demo-user
+# Load any dataset (uses default user)
+rem db load --file datasets/quickstart/sample_data.yaml
 
 # Explore formats
-rem db load --file datasets/formats/engrams/scenarios/team_meeting/team_standup_meeting.yaml --user-id demo-user
+rem db load --file datasets/formats/engrams/scenarios/team_meeting/team_standup_meeting.yaml
 
-# Try domain-specific examples
+# Try domain-specific examples (use --user-id for multi-tenant scenarios)
 rem db load --file datasets/domains/recruitment/scenarios/candidate_pipeline/data.yaml --user-id acme-corp
 ```
 
@@ -429,30 +418,24 @@ json_schema_extra:
 ```bash
 # Ingest the schema (stores in database schemas table)
 rem process ingest my-research-assistant.yaml \
-  --user-id my-user \
   --category agents \
   --tags custom,research
 
 # Verify schema is in database (should show schema details)
-rem ask "LOOKUP 'my-research-assistant' FROM schemas" --user-id my-user
+rem ask "LOOKUP 'my-research-assistant' FROM schemas"
 ```
 
 **Step 3: Use Your Custom Agent**
 
 ```bash
 # Run a query with your custom agent
-rem ask research-assistant "Find documents about machine learning architecture" \
-  --user-id my-user
+rem ask research-assistant "Find documents about machine learning architecture"
 
 # With streaming
-rem ask research-assistant "Summarize recent API design documents" \
-  --user-id my-user \
-  --stream
+rem ask research-assistant "Summarize recent API design documents" --stream
 
 # With session continuity
-rem ask research-assistant "What did we discuss about ML?" \
-  --user-id my-user \
-  --session-id abc-123
+rem ask research-assistant "What did we discuss about ML?" --session-id abc-123
 ```
 
 ### Agent Schema Structure
@@ -523,10 +506,10 @@ Custom agents can also be used as **ontology extractors** to extract structured
 **Schema not found error:**
 ```bash
 # Check if schema was ingested correctly
-rem ask "SEARCH 'my-agent' FROM schemas" --user-id my-user
+rem ask "SEARCH 'my-agent' FROM schemas"
 
-# List all schemas for your user
-rem ask "SELECT name, category, created_at FROM schemas ORDER BY created_at DESC LIMIT 10" --user-id my-user
+# List all schemas
+rem ask "SELECT name, category, created_at FROM schemas ORDER BY created_at DESC LIMIT 10"
 ```
 
 **Agent not loading tools:**
@@ -959,22 +942,14 @@ rem db schema validate --models src/rem/models/entities
 Process files with optional custom extractor (ontology extraction).
 
 ```bash
-# Process all completed files for tenant
-rem process files \
-  --tenant-id acme-corp \
-  --status completed \
-  --limit 10
+# Process all completed files
+rem process files --status completed --limit 10
 
 # Process with custom extractor
-rem process files \
-  --tenant-id acme-corp \
-  --extractor cv-parser-v1 \
-  --limit 50
+rem process files --extractor cv-parser-v1 --limit 50
 
-# Process files from the last 7 days
-rem process files \
-  --tenant-id acme-corp \
-  --lookback-hours 168
+# Process files for specific user
+rem process files --user-id user-123 --status completed
 ```
 
 #### `rem process ingest` - Ingest File into REM
@@ -982,14 +957,13 @@ rem process files \
 Ingest a file into REM with full pipeline (storage + parsing + embedding + database).
 
 ```bash
-# Ingest local file
+# Ingest local file with metadata
 rem process ingest /path/to/document.pdf \
-  --user-id user-123 \
   --category legal \
   --tags contract,2024
 
 # Ingest with minimal options
-rem process ingest ./meeting-notes.md --user-id user-123
+rem process ingest ./meeting-notes.md
 ```
 
 #### `rem process uri` - Parse File (Read-Only)
@@ -1014,28 +988,17 @@ rem process uri s3://bucket/key.docx --output text
 Run full dreaming workflow: extractors → moments → affinity → user model.
 
 ```bash
-# Full workflow for user
-rem dreaming full \
-  --user-id user-123 \
-  --tenant-id acme-corp
+# Full workflow (uses default user from settings)
+rem dreaming full
 
 # Skip ontology extractors
-rem dreaming full \
-  --user-id user-123 \
-  --tenant-id acme-corp \
-  --skip-extractors
+rem dreaming full --skip-extractors
 
 # Process last 24 hours only
-rem dreaming full \
-  --user-id user-123 \
-  --tenant-id acme-corp \
-  --lookback-hours 24
+rem dreaming full --lookback-hours 24
 
-# Limit resources processed
-rem dreaming full \
-  --user-id user-123 \
-  --tenant-id acme-corp \
-  --limit 100
+# Limit resources processed for specific user
+rem dreaming full --user-id user-123 --limit 100
 ```
 
 #### `rem dreaming custom` - Custom Extractor
@@ -1043,16 +1006,11 @@ rem dreaming full \
 Run specific ontology extractor on user's data.
 
 ```bash
-# Run CV parser on user's files
-rem dreaming custom \
-  --user-id user-123 \
-  --tenant-id acme-corp \
-  --extractor cv-parser-v1
+# Run CV parser on files
+rem dreaming custom --extractor cv-parser-v1
 
-# Process last week's files
+# Process last week's files with limit
 rem dreaming custom \
-  --user-id user-123 \
-  --tenant-id acme-corp \
   --extractor contract-analyzer-v1 \
   --lookback-hours 168 \
   --limit 50
@@ -1063,17 +1021,11 @@ rem dreaming custom \
 Extract temporal narratives from resources.
 
 ```bash
-# Generate moments for user
-rem dreaming moments \
-  --user-id user-123 \
-  --tenant-id acme-corp \
-  --limit 50
+# Generate moments
+rem dreaming moments --limit 50
 
 # Process last 7 days
-rem dreaming moments \
-  --user-id user-123 \
-  --tenant-id acme-corp \
-  --lookback-hours 168
+rem dreaming moments --lookback-hours 168
 ```
 
 #### `rem dreaming affinity` - Build Relationships
@@ -1081,17 +1033,11 @@ rem dreaming moments \
 Build semantic relationships between resources using embeddings.
 
 ```bash
-# Build affinity graph for user
-rem dreaming affinity \
-  --user-id user-123 \
-  --tenant-id acme-corp \
-  --limit 100
+# Build affinity graph
+rem dreaming affinity --limit 100
 
 # Process recent resources only
-rem dreaming affinity \
-  --user-id user-123 \
-  --tenant-id acme-corp \
-  --lookback-hours 24
+rem dreaming affinity --lookback-hours 24
 ```
 
 #### `rem dreaming user-model` - Update User Model
@@ -1100,9 +1046,7 @@ Update user model from recent activity (preferences, interests, patterns).
 
 ```bash
 # Update user model
-rem dreaming user-model \
-  --user-id user-123 \
-  --tenant-id acme-corp
+rem dreaming user-model
 ```
 
 ### Evaluation & Experiments
@@ -1468,6 +1412,110 @@ TraverseQuery ::= TRAVERSE [<edge_types:list>] WITH <initial_query:Query> [DEPTH
 
 **Stage 4** (100% answerable): Mature graph with rich historical data. All query types fully functional with high-quality results.
 
+## Troubleshooting
+
+### Apple Silicon Mac: "Failed to build kreuzberg" Error
+
+**Problem**: Installation fails with `ERROR: Failed building wheel for kreuzberg` on Apple Silicon Macs.
+
+**Root Cause**: REM uses `kreuzberg>=4.0.0rc1` for document parsing with native ONNX/Rust table extraction. Kreuzberg 4.0.0rc1 provides pre-built wheels for ARM64 macOS (`macosx_14_0_arm64.whl`) but NOT for x86_64 (Intel) macOS. If you're using an x86_64 Python binary (running under Rosetta 2), pip cannot find a compatible wheel and attempts to build from source, which fails.
+
+**Solution**: Use ARM64 (native) Python instead of x86_64 Python.
+
+**Step 1: Verify your Python architecture**
+
+```bash
+python3 -c "import platform; print(f'Machine: {platform.machine()}')"
+```
+
+- **Correct**: `Machine: arm64` (native ARM Python)
+- **Wrong**: `Machine: x86_64` (Intel Python under Rosetta)
+
+**Step 2: Install ARM Python via Homebrew** (if not already installed)
+
+```bash
+# Install ARM Python
+brew install python@3.12
+
+# Verify it's ARM
+/opt/homebrew/bin/python3.12 -c "import platform; print(platform.machine())"
+# Should output: arm64
+```
+
+**Step 3: Create venv with ARM Python**
+
+```bash
+# Use full path to ARM Python
+/opt/homebrew/bin/python3.12 -m venv .venv
+
+# Activate and install
+source .venv/bin/activate
+pip install "remdb[all]"
+```
+
+**Why This Happens**: Some users have both Intel Homebrew (`/usr/local`) and ARM Homebrew (`/opt/homebrew`) installed. If your system `python3` points to the Intel version at `/usr/local/bin/python3`, you'll hit this issue. The fix is to explicitly use the ARM Python from `/opt/homebrew/bin/python3.12`.
+
+**Verification**: After successful installation, you should see:
+```
+Using cached kreuzberg-4.0.0rc1-cp310-abi3-macosx_14_0_arm64.whl (19.8 MB)
+Successfully installed ... kreuzberg-4.0.0rc1 ... remdb-0.3.10
+```
+
+## Using REM as a Library
+
+REM wraps FastAPI - extend it exactly as you would any FastAPI app.
+
+```python
+import rem
+from rem import create_app
+from rem.models.core import CoreModel
+
+# 1. Register models (for schema generation)
+rem.register_models(MyModel, AnotherModel)
+
+# 2. Register schema paths (for custom agents/evaluators)
+rem.register_schema_path("./schemas")
+
+# 3. Create app
+app = create_app()
+
+# 4. Extend like normal FastAPI
+app.include_router(my_router)
+
+@app.mcp_server.tool()
+async def my_tool(query: str) -> dict:
+    """Custom MCP tool."""
+    return {"result": query}
+```
+
+### Project Structure
+
+```
+my-rem-app/
+├── my_app/
+│   ├── main.py           # Entry point (create_app + extensions)
+│   ├── models.py         # Custom models (inherit CoreModel)
+│   └── routers/          # Custom FastAPI routers
+├── schemas/
+│   ├── agents/           # Custom agent YAML schemas
+│   └── evaluators/       # Custom evaluator schemas
+├── sql/migrations/       # Custom SQL migrations
+└── pyproject.toml
+```
+
+Generate this structure with: `rem scaffold my-app` *(coming soon)*
+
+### Extension Points
+
+| Extension | How |
+|-----------|-----|
+| **Routes** | `app.include_router(router)` or `@app.get()` |
+| **MCP Tools** | `@app.mcp_server.tool()` decorator or `app.mcp_server.add_tool(fn)` |
+| **MCP Resources** | `@app.mcp_server.resource("uri://...")` or `app.mcp_server.add_resource(fn)` |
+| **MCP Prompts** | `@app.mcp_server.prompt()` or `app.mcp_server.add_prompt(fn)` |
+| **Models** | `rem.register_models(Model)` then `rem db schema generate` |
+| **Agent Schemas** | `rem.register_schema_path("./schemas")` or `SCHEMA__PATHS` env var |
+
 ## License
 
 MIT