remdb 0.3.172__py3-none-any.whl → 0.3.223__py3-none-any.whl

rem/models/entities/ontology.py

@@ -1,63 +1,55 @@
-"""Ontology entity for tenant-specific knowledge extensions.
+"""Ontology entity for domain-specific knowledge.
 
-**What is Ontology Extraction?**
+**What are Ontologies?**
 
-Ontologies are **domain-specific structured knowledge** extracted from files using custom
-agent schemas. They extend REM's normal file processing pipeline with tenant-specific
-parsers that extract structured data the standard chunking pipeline would miss.
+Ontologies are **domain-specific structured knowledge** that can be:
+1. **Extracted** from files using custom agent schemas (agent-extracted)
+2. **Loaded directly** from external sources like git repos or S3 (direct-loaded)
 
-**Normal File Processing:**
-File → extract text → chunk → embed → resources (semantic search ready)
+**Use Case 1: Agent-Extracted Ontologies**
 
-**Ontology Processing (Tenant Knowledge Extensions):**
 File → custom agent → structured JSON → ontology (domain knowledge)
 
-**Why Ontologies?**
-- Standard chunking gives you semantic search over raw content
-- Ontologies give you **structured queryable fields** from domain logic
-- Example: A contract PDF becomes both searchable chunks AND a structured record with
-  parties, dates, payment terms, obligations as queryable fields
+Example: A contract PDF becomes a structured record with parties, dates, payment terms.
+
+**Use Case 2: Direct-Loaded Ontologies (Knowledge Bases)**
+
+External source (git/S3) → load → ontology (reference knowledge)
+
+Example: A psychiatric ontology of disorders, symptoms, and drugs loaded from markdown
+files in a git repository. Each markdown file becomes an ontology node with:
+- `uri`: git path (e.g., `git://org/repo/ontology/disorders/anxiety/panic-disorder.md`)
+- `content`: markdown content for embedding/search
+- `extracted_data`: parsed frontmatter or structure
 
 **Architecture:**
-- Runs as part of dreaming worker (background knowledge extraction)
-- OntologyConfig defines which files trigger which extractors (MIME type, URI pattern, tags)
+- Runs as part of dreaming worker (background knowledge extraction) OR
+- Loaded directly via `rem db load` for external knowledge bases
+- OntologyConfig defines which files trigger which extractors
 - Multiple ontologies per file (apply different domain lenses)
-- Tenant-scoped: Each tenant can define their own extractors
+- Tenant-scoped: Each tenant can define their own extractors and knowledge bases
 
 **Use Cases:**
 
-1. **Recruitment (CV Parsing)**
-   - Standard pipeline: Chunks for "find me candidates with Python experience"
-   - Ontology: Structured fields for filtering/sorting (years_experience, seniority_level, skills[])
-
-2. **Legal (Contract Analysis)**
-   - Standard pipeline: Semantic search over contract text
-   - Ontology: Queryable fields (parties, effective_date, payment_amount, key_obligations[])
+1. **Recruitment (CV Parsing)** - Agent-extracted
+   - Ontology: Structured fields for filtering/sorting (years_experience, skills[])
 
-3. **Medical (Health Records)**
-   - Standard pipeline: Find mentions of conditions
-   - Ontology: Structured diagnoses, medications, dosages, treatment plans
+2. **Legal (Contract Analysis)** - Agent-extracted
+   - Ontology: Queryable fields (parties, effective_date, payment_amount)
 
-4. **Finance (Report Analysis)**
-   - Standard pipeline: Search for financial terms
-   - Ontology: Extracted metrics, risk_flags, trends, forecasts
+3. **Medical Knowledge Base** - Direct-loaded
+   - Ontology: Disorders, symptoms, medications from curated markdown files
+   - Enables semantic search over psychiatric/medical domain knowledge
 
-**Example Flow:**
-1. Tenant creates OntologyConfig: "Run cv-parser-v1 on files with mime_type='application/pdf' and tags=['resume']"
-2. File uploaded with tags=["resume"]
-3. Normal processing: File → chunks → resources
-4. Dreaming worker detects matching OntologyConfig
-5. Loads cv-parser-v1 agent schema from database
-6. Runs agent on file content → extracts structured data
-7. Stores Ontology with extracted_data = {candidate_name, skills, experience, education, ...}
-8. Ontology is now queryable via LOOKUP, SEARCH, or direct SQL
+4. **Documentation/Procedures** - Direct-loaded
+   - Ontology: Clinical procedures (e.g., SCID-5 assessment steps)
+   - Reference material accessible via RAG
 
 **Design:**
-- Each ontology links to a File via file_id
-- Agent schema tracked via agent_schema_id (human-readable label, not UUID)
-- Structured data in `extracted_data` (arbitrary JSON, schema defined by agent)
-- Embeddings generated for semantic search (configurable fields via agent schema)
-- Multiple ontologies per file using different schemas
+- `file_id` and `agent_schema_id` are optional (only needed for agent-extracted)
+- `uri` field for external source references (git://, s3://, https://)
+- Structured data in `extracted_data` (arbitrary JSON)
+- Embeddings generated for semantic search via `content` field
 - Tenant-isolated: OntologyConfigs are tenant-scoped
 """
 
@@ -70,18 +62,19 @@ from ..core.core_model import CoreModel
 
 
 class Ontology(CoreModel):
-    """Domain-specific knowledge extracted from files using custom agents.
+    """Domain-specific knowledge - either agent-extracted or direct-loaded.
 
     Attributes:
         name: Human-readable label for this ontology instance
-        file_id: Foreign key to File entity that was processed
-        agent_schema_id: Foreign key to Schema entity that performed extraction
-        provider_name: LLM provider used for extraction (e.g., "anthropic", "openai")
-        model_name: Specific model used (e.g., "claude-sonnet-4-5")
-        extracted_data: Structured data extracted by agent (arbitrary JSON)
+        uri: External source reference (git://, s3://, https://) for direct-loaded ontologies
+        file_id: Foreign key to File entity (optional - only for agent-extracted)
+        agent_schema_id: Schema that performed extraction (optional - only for agent-extracted)
+        provider_name: LLM provider used for extraction (optional)
+        model_name: Specific model used (optional)
+        extracted_data: Structured data - either extracted by agent or parsed from source
         confidence_score: Optional confidence score from extraction (0.0-1.0)
         extraction_timestamp: When extraction was performed
-        embedding_text: Text used for generating embedding (derived from extracted_data)
+        content: Text used for generating embedding
 
     Inherited from CoreModel:
         id: UUID or string identifier
@@ -93,10 +86,9 @@ class Ontology(CoreModel):
         graph_edges: Relationships to other entities
         metadata: Flexible metadata storage
         tags: Classification tags
-        column: Database schema metadata
 
     Example Usage:
-        # CV extraction
+        # Agent-extracted: CV parsing
         cv_ontology = Ontology(
             name="john-doe-cv-2024",
             file_id="file-uuid-123",
@@ -105,73 +97,72 @@ class Ontology(CoreModel):
             model_name="claude-sonnet-4-5-20250929",
             extracted_data={
                 "candidate_name": "John Doe",
-                "email": "john@example.com",
                 "skills": ["Python", "PostgreSQL", "Kubernetes"],
-                "experience": [
-                    {
-                        "company": "TechCorp",
-                        "role": "Senior Engineer",
-                        "years": 3,
-                        "achievements": ["Led migration to k8s", "Reduced costs 40%"]
-                    }
-                ],
-                "education": [
-                    {"degree": "BS Computer Science", "institution": "MIT", "year": 2018}
-                ]
             },
             confidence_score=0.95,
-            tags=["cv", "engineering", "senior-level"]
+            tags=["cv", "engineering"]
         )
 
-        # Contract extraction
-        contract_ontology = Ontology(
-            name="acme-supplier-agreement-2024",
-            file_id="file-uuid-456",
-            agent_schema_id="contract-parser-v2",
-            provider_name="openai",
-            model_name="gpt-4.1",
+        # Direct-loaded: Knowledge base from git
+        api_docs = Ontology(
+            name="rest-api-guide",
+            uri="git://example-org/docs/api/rest-api-guide.md",
+            content="# REST API Guide\\n\\nThis guide covers RESTful API design...",
             extracted_data={
-                "contract_type": "supplier_agreement",
-                "parties": [
-                    {"name": "ACME Corp", "role": "buyer"},
-                    {"name": "SupplyChain Inc", "role": "supplier"}
-                ],
-                "effective_date": "2024-01-01",
-                "termination_date": "2026-12-31",
-                "payment_terms": {
-                    "amount": 500000,
-                    "currency": "USD",
-                    "frequency": "quarterly"
-                },
-                "key_obligations": [
-                    "Supplier must deliver within 30 days",
-                    "Buyer must pay within 60 days of invoice"
-                ]
+                "type": "documentation",
+                "category": "api",
+                "version": "2.0",
+            },
+            tags=["api", "rest", "documentation"]
+        )
+
+        # Direct-loaded: Technical spec from git
+        config_spec = Ontology(
+            name="config-schema",
+            uri="git://example-org/docs/specs/config-schema.md",
+            content="# Configuration Schema\\n\\nThis document defines...",
+            extracted_data={
+                "type": "specification",
+                "format": "yaml",
+                "version": "1.0",
             },
-            confidence_score=0.92,
-            tags=["contract", "supplier", "procurement"]
+            tags=["config", "schema", "specification"]
         )
     """
 
     # Core fields
     name: str
-    file_id: UUID | str
-    agent_schema_id: str  # Natural language label of Schema entity
+    uri: Optional[str] = None  # External source: git://, s3://, https://
 
-    # Extraction metadata
-    provider_name: str  # LLM provider (anthropic, openai, etc.)
-    model_name: str  # Specific model used
-    extracted_data: dict[str, Any]  # Arbitrary structured data from agent
+    # Agent extraction fields (optional - only for agent-extracted ontologies)
+    file_id: Optional[UUID | str] = None  # FK to File entity
+    agent_schema_id: Optional[str] = None  # Schema that performed extraction
+    provider_name: Optional[str] = None  # LLM provider (anthropic, openai, etc.)
+    model_name: Optional[str] = None  # Specific model used
+
+    # Data fields
+    extracted_data: Optional[dict[str, Any]] = None  # Structured data
     confidence_score: Optional[float] = None  # 0.0-1.0 if provided by agent
     extraction_timestamp: Optional[str] = None  # ISO8601 timestamp
 
-    # Semantic search support
-    embedding_text: Optional[str] = None  # Text for embedding generation
+    # Semantic search support - 'content' is a default embeddable field name
+    content: Optional[str] = None  # Text for embedding generation
 
     model_config = ConfigDict(
         json_schema_extra={
-            "description": "Domain-specific knowledge extracted from files using custom agents",
+            "description": "Domain-specific knowledge - agent-extracted or direct-loaded from external sources",
             "examples": [
+                {
+                    "name": "panic-disorder",
+                    "uri": "git://org/repo/ontology/disorders/anxiety/panic-disorder.md",
+                    "content": "# Panic Disorder\n\nPanic disorder is characterized by...",
+                    "extracted_data": {
+                        "type": "disorder",
+                        "category": "anxiety",
+                        "icd10": "F41.0"
+                    },
+                    "tags": ["disorder", "anxiety"]
+                },
                 {
                     "name": "john-doe-cv-2024",
                     "file_id": "550e8400-e29b-41d4-a716-446655440000",
@@ -180,8 +171,7 @@ class Ontology(CoreModel):
                     "model_name": "claude-sonnet-4-5-20250929",
                     "extracted_data": {
                         "candidate_name": "John Doe",
-                        "skills": ["Python", "PostgreSQL"],
-                        "experience": []
+                        "skills": ["Python", "PostgreSQL"]
                     },
                     "confidence_score": 0.95,
                     "tags": ["cv", "engineering"]

rem/schemas/agents/rem.yaml

@@ -124,7 +124,7 @@ json_schema_extra:
 
   # Explicit resource declarations for reference data
   resources:
-    - uri: rem://schemas
+    - uri: rem://agents
       name: Agent Schemas List
       description: List all available agent schemas in the system
     - uri: rem://status

rem/services/content/service.py

@@ -274,7 +274,7 @@ class ContentService:
     async def ingest_file(
         self,
         file_uri: str,
-        user_id: str,
+        user_id: str | None = None,
         category: str | None = None,
         tags: list[str] | None = None,
         is_local_server: bool = False,
@@ -283,6 +283,10 @@ class ContentService:
         """
         Complete file ingestion pipeline: read → store → parse → chunk → embed.
 
+        **IMPORTANT: Data is PUBLIC by default (user_id=None).**
+        This is correct for shared knowledge bases (ontologies, procedures, reference data).
+        Private user-scoped data is rarely needed - only set user_id for truly personal content.
+
         **CENTRALIZED INGESTION**: This is the single entry point for all file ingestion
         in REM. It handles:
 
@@ -319,7 +323,9 @@ class ContentService:
 
         Args:
             file_uri: Source file location (local path, s3://, or https://)
-            user_id: User identifier for data isolation and ownership
+            user_id: User identifier for PRIVATE data only. Default None = PUBLIC/shared.
+                Leave as None for shared knowledge bases, ontologies, reference data.
+                Only set for truly private user-specific content.
             category: Optional category tag (document, code, audio, etc.)
             tags: Optional list of tags
             is_local_server: True if running as local/stdio MCP server
@@ -347,12 +353,19 @@ class ContentService:
 
         Example:
             >>> service = ContentService()
+            >>> # PUBLIC data (default) - visible to all users
             >>> result = await service.ingest_file(
-            ...     file_uri="s3://bucket/contract.pdf",
-            ...     user_id="user-123",
-            ...     category="legal"
+            ...     file_uri="s3://bucket/procedure.pdf",
+            ...     category="medical"
             ... )
             >>> print(f"Created {result['resources_created']} searchable chunks")
+            >>>
+            >>> # PRIVATE data (rare) - only for user-specific content
+            >>> result = await service.ingest_file(
+            ...     file_uri="s3://bucket/personal-notes.pdf",
+            ...     user_id="user-123",  # Only this user can access
+            ...     category="personal"
+            ... )
         """
         from pathlib import Path
         from uuid import uuid4

rem/services/email/service.py

@@ -376,8 +376,17 @@ class EmailService:
             await user_repo.upsert(existing_user)
             return {"allowed": True, "error": None}
         else:
-            # New user - check if domain is trusted
-            if settings and hasattr(settings, 'email') and settings.email.trusted_domain_list:
+            # New user - first check if they're a subscriber (by email lookup)
+            from ...models.entities import Subscriber
+            subscriber_repo = Repository(Subscriber, db=db)
+            existing_subscriber = await subscriber_repo.find_one({"email": email})
+
+            if existing_subscriber:
+                # Subscriber exists - allow them to create account
+                # (approved field may not exist in older schemas, so just check existence)
+                logger.info(f"Subscriber {email} creating user account")
+            elif settings and hasattr(settings, 'email') and settings.email.trusted_domain_list:
+                # Not an approved subscriber - check if domain is trusted
                 if not settings.email.is_domain_trusted(email):
                     email_domain = email.split("@")[-1]
                     logger.warning(f"Untrusted domain attempted signup: {email_domain}")

rem/services/embeddings/worker.py

@@ -23,6 +23,8 @@ Future:
 import asyncio
 import os
 from typing import Any, Optional
+import hashlib
+import uuid
 from uuid import uuid4
 
 import httpx
@@ -108,6 +110,7 @@ class EmbeddingWorker:
         self.task_queue: asyncio.Queue = asyncio.Queue()
         self.workers: list[asyncio.Task] = []
         self.running = False
+        self._in_flight_count = 0  # Track tasks being processed (not just in queue)
 
         # Store API key for direct HTTP requests
         from ...settings import settings
@@ -143,17 +146,18 @@ class EmbeddingWorker:
             return
 
         queue_size = self.task_queue.qsize()
-        logger.debug(f"Stopping EmbeddingWorker (processing {queue_size} queued tasks first)")
+        in_flight = self._in_flight_count
+        logger.debug(f"Stopping EmbeddingWorker (queue={queue_size}, in_flight={in_flight})")
 
-        # Wait for queue to drain (with timeout)
+        # Wait for both queue to drain AND in-flight tasks to complete
         max_wait = 30  # 30 seconds max
         waited = 0.0
-        while not self.task_queue.empty() and waited < max_wait:
+        while (not self.task_queue.empty() or self._in_flight_count > 0) and waited < max_wait:
             await asyncio.sleep(0.5)
             waited += 0.5
 
-        if not self.task_queue.empty():
-            remaining = self.task_queue.qsize()
+        if not self.task_queue.empty() or self._in_flight_count > 0:
+            remaining = self.task_queue.qsize() + self._in_flight_count
             logger.warning(
                 f"EmbeddingWorker timeout: {remaining} tasks remaining after {max_wait}s"
             )
@@ -205,12 +209,18 @@ class EmbeddingWorker:
                 if not batch:
                     continue
 
-                logger.debug(f"Worker {worker_id} processing batch of {len(batch)} tasks")
+                # Track in-flight tasks
+                self._in_flight_count += len(batch)
 
-                # Generate embeddings for batch
-                await self._process_batch(batch)
+                logger.debug(f"Worker {worker_id} processing batch of {len(batch)} tasks")
 
-                logger.debug(f"Worker {worker_id} completed batch")
+                try:
+                    # Generate embeddings for batch
+                    await self._process_batch(batch)
+                    logger.debug(f"Worker {worker_id} completed batch")
+                finally:
+                    # Always decrement in-flight count, even on error
+                    self._in_flight_count -= len(batch)
 
             except asyncio.CancelledError:
                 logger.debug(f"Worker {worker_id} cancelled")
@@ -373,7 +383,11 @@ class EmbeddingWorker:
         for task, embedding in zip(tasks, embeddings):
             table_name = f"embeddings_{task.table_name}"
 
-            # Build upsert SQL
+            # Generate deterministic ID from key fields (entity_id, field_name, provider)
+            key_string = f"{task.entity_id}:{task.field_name}:{task.provider}"
+            embedding_id = str(uuid.UUID(hashlib.md5(key_string.encode()).hexdigest()))
+
+            # Build upsert SQL - conflict on deterministic ID
             sql = f"""
                 INSERT INTO {table_name} (
                     id,
@@ -386,7 +400,7 @@ class EmbeddingWorker:
                     updated_at
                 )
                 VALUES ($1, $2, $3, $4, $5, $6, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
-                ON CONFLICT (entity_id, field_name, provider)
+                ON CONFLICT (id)
                 DO UPDATE SET
                     model = EXCLUDED.model,
                     embedding = EXCLUDED.embedding,
@@ -400,7 +414,7 @@ class EmbeddingWorker:
                 await self.postgres_service.execute(
                     sql,
                     (
-                        str(uuid4()),
+                        embedding_id,
                         task.entity_id,
                         task.field_name,
                         task.provider,

rem/services/postgres/__init__.py

@@ -3,22 +3,47 @@ PostgreSQL service for CloudNativePG database operations.
 """
 
 from .diff_service import DiffService, SchemaDiff
+from .programmable_diff_service import (
+    DiffResult,
+    ObjectDiff,
+    ObjectType,
+    ProgrammableDiffService,
+)
 from .repository import Repository
 from .service import PostgresService
 
 
+_postgres_instance: PostgresService | None = None
+
+
 def get_postgres_service() -> PostgresService | None:
     """
-    Get PostgresService instance.
+    Get PostgresService singleton instance.
 
     Returns None if Postgres is disabled.
+    Uses singleton pattern to prevent connection pool exhaustion.
     """
+    global _postgres_instance
+
     from ...settings import settings
 
     if not settings.postgres.enabled:
         return None
 
-    return PostgresService()
+    if _postgres_instance is None:
+        _postgres_instance = PostgresService()
+
+    return _postgres_instance
 
 
-__all__ = ["PostgresService", "get_postgres_service", "Repository", "DiffService", "SchemaDiff"]
+__all__ = [
+    "DiffResult",
+    "DiffService",
+    "ObjectDiff",
+    "ObjectType",
+    "PostgresService",
+    "ProgrammableDiffService",
+    "Repository",
+    "SchemaDiff",
+    "get_postgres_service",
+]

rem/services/postgres/diff_service.py

@@ -5,12 +5,17 @@ Uses Alembic autogenerate to detect differences between:
 - Target schema (derived from Pydantic models)
 - Current database schema
 
+Also compares programmable objects (functions, triggers, views) which
+Alembic does not track.
+
 This enables:
 1. Local development: See what would change before applying migrations
 2. CI validation: Detect drift between code and database (--check mode)
 3. Migration generation: Create incremental migration files
 """
 
+import asyncio
+import re
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Optional
@@ -51,11 +56,14 @@ class SchemaDiff:
     sql: str = ""
     upgrade_ops: Optional[ops.UpgradeOps] = None
     filtered_count: int = 0  # Number of operations filtered out by strategy
+    # Programmable objects (functions, triggers, views)
+    programmable_summary: list[str] = field(default_factory=list)
+    programmable_sql: str = ""
 
     @property
     def change_count(self) -> int:
         """Total number of detected changes."""
-        return len(self.summary)
+        return len(self.summary) + len(self.programmable_summary)
 
 
 class DiffService:
@@ -127,10 +135,13 @@ class DiffService:
             # These are now generated in pydantic_to_sqlalchemy
         return True
 
-    def compute_diff(self) -> SchemaDiff:
+    def compute_diff(self, include_programmable: bool = True) -> SchemaDiff:
         """
         Compare Pydantic models against database and return differences.
 
+        Args:
+            include_programmable: If True, also diff functions/triggers/views
+
         Returns:
             SchemaDiff with detected changes
         """
@@ -167,21 +178,62 @@ class DiffService:
                 for op in filtered_ops:
                     summary.extend(self._describe_operation(op))
 
-        has_changes = len(summary) > 0
-
         # Generate SQL if there are changes
         sql = ""
-        if has_changes and upgrade_ops:
+        if summary and upgrade_ops:
             sql = self._render_sql(upgrade_ops, engine)
 
+        # Programmable objects diff (functions, triggers, views)
+        programmable_summary = []
+        programmable_sql = ""
+        if include_programmable:
+            prog_summary, prog_sql = self._compute_programmable_diff()
+            programmable_summary = prog_summary
+            programmable_sql = prog_sql
+
+        has_changes = len(summary) > 0 or len(programmable_summary) > 0
+
         return SchemaDiff(
             has_changes=has_changes,
             summary=summary,
             sql=sql,
             upgrade_ops=upgrade_ops,
             filtered_count=filtered_count,
+            programmable_summary=programmable_summary,
+            programmable_sql=programmable_sql,
         )
 
+    def _compute_programmable_diff(self) -> tuple[list[str], str]:
+        """
+        Compute diff for programmable objects (functions, triggers, views).
+
+        Returns:
+            Tuple of (summary_lines, sync_sql)
+        """
+        from .programmable_diff_service import ProgrammableDiffService
+
+        service = ProgrammableDiffService()
+
+        # Run async diff in sync context
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
+
+        result = loop.run_until_complete(service.compute_diff())
+
+        summary = []
+        for diff in result.diffs:
+            if diff.status == "missing":
+                summary.append(f"+ {diff.object_type.value.upper()} {diff.name} (missing)")
+            elif diff.status == "different":
+                summary.append(f"~ {diff.object_type.value.upper()} {diff.name} (different)")
+            elif diff.status == "extra":
+                summary.append(f"- {diff.object_type.value.upper()} {diff.name} (extra in db)")
+
+        return summary, result.sync_sql
+
     def _filter_operations(self, operations: list) -> tuple[list, int]:
         """
         Filter operations based on migration strategy.