remdb 0.3.146__py3-none-any.whl → 0.3.181__py3-none-any.whl

rem/models/entities/ontology.py

@@ -1,63 +1,55 @@
-"""Ontology entity for tenant-specific knowledge extensions.
+"""Ontology entity for domain-specific knowledge.
 
-**What is Ontology Extraction?**
+**What are Ontologies?**
 
-Ontologies are **domain-specific structured knowledge** extracted from files using custom
-agent schemas. They extend REM's normal file processing pipeline with tenant-specific
-parsers that extract structured data the standard chunking pipeline would miss.
+Ontologies are **domain-specific structured knowledge** that can be:
+1. **Extracted** from files using custom agent schemas (agent-extracted)
+2. **Loaded directly** from external sources like git repos or S3 (direct-loaded)
 
-**Normal File Processing:**
-File → extract text → chunk → embed → resources (semantic search ready)
+**Use Case 1: Agent-Extracted Ontologies**
 
-**Ontology Processing (Tenant Knowledge Extensions):**
 File → custom agent → structured JSON → ontology (domain knowledge)
 
-**Why Ontologies?**
-- Standard chunking gives you semantic search over raw content
-- Ontologies give you **structured queryable fields** from domain logic
-- Example: A contract PDF becomes both searchable chunks AND a structured record with
-  parties, dates, payment terms, obligations as queryable fields
+Example: A contract PDF becomes a structured record with parties, dates, payment terms.
+
+**Use Case 2: Direct-Loaded Ontologies (Knowledge Bases)**
+
+External source (git/S3) → load → ontology (reference knowledge)
+
+Example: A psychiatric ontology of disorders, symptoms, and drugs loaded from markdown
+files in a git repository. Each markdown file becomes an ontology node with:
+- `uri`: git path (e.g., `git://org/repo/ontology/disorders/anxiety/panic-disorder.md`)
+- `content`: markdown content for embedding/search
+- `extracted_data`: parsed frontmatter or structure
 
 **Architecture:**
-- Runs as part of dreaming worker (background knowledge extraction)
-- OntologyConfig defines which files trigger which extractors (MIME type, URI pattern, tags)
+- Runs as part of dreaming worker (background knowledge extraction) OR
+- Loaded directly via `rem db load` for external knowledge bases
+- OntologyConfig defines which files trigger which extractors
 - Multiple ontologies per file (apply different domain lenses)
-- Tenant-scoped: Each tenant can define their own extractors
+- Tenant-scoped: Each tenant can define their own extractors and knowledge bases
 
 **Use Cases:**
 
-1. **Recruitment (CV Parsing)**
-   - Standard pipeline: Chunks for "find me candidates with Python experience"
-   - Ontology: Structured fields for filtering/sorting (years_experience, seniority_level, skills[])
-
-2. **Legal (Contract Analysis)**
-   - Standard pipeline: Semantic search over contract text
-   - Ontology: Queryable fields (parties, effective_date, payment_amount, key_obligations[])
+1. **Recruitment (CV Parsing)** - Agent-extracted
+   - Ontology: Structured fields for filtering/sorting (years_experience, skills[])
 
-3. **Medical (Health Records)**
-   - Standard pipeline: Find mentions of conditions
-   - Ontology: Structured diagnoses, medications, dosages, treatment plans
+2. **Legal (Contract Analysis)** - Agent-extracted
+   - Ontology: Queryable fields (parties, effective_date, payment_amount)
 
-4. **Finance (Report Analysis)**
-   - Standard pipeline: Search for financial terms
-   - Ontology: Extracted metrics, risk_flags, trends, forecasts
+3. **Medical Knowledge Base** - Direct-loaded
+   - Ontology: Disorders, symptoms, medications from curated markdown files
+   - Enables semantic search over psychiatric/medical domain knowledge
 
-**Example Flow:**
-1. Tenant creates OntologyConfig: "Run cv-parser-v1 on files with mime_type='application/pdf' and tags=['resume']"
-2. File uploaded with tags=["resume"]
-3. Normal processing: File → chunks → resources
-4. Dreaming worker detects matching OntologyConfig
-5. Loads cv-parser-v1 agent schema from database
-6. Runs agent on file content → extracts structured data
-7. Stores Ontology with extracted_data = {candidate_name, skills, experience, education, ...}
-8. Ontology is now queryable via LOOKUP, SEARCH, or direct SQL
+4. **Documentation/Procedures** - Direct-loaded
+   - Ontology: Clinical procedures (e.g., SCID-5 assessment steps)
+   - Reference material accessible via RAG
 
 **Design:**
-- Each ontology links to a File via file_id
-- Agent schema tracked via agent_schema_id (human-readable label, not UUID)
-- Structured data in `extracted_data` (arbitrary JSON, schema defined by agent)
-- Embeddings generated for semantic search (configurable fields via agent schema)
-- Multiple ontologies per file using different schemas
+- `file_id` and `agent_schema_id` are optional (only needed for agent-extracted)
+- `uri` field for external source references (git://, s3://, https://)
+- Structured data in `extracted_data` (arbitrary JSON)
+- Embeddings generated for semantic search via `content` field
 - Tenant-isolated: OntologyConfigs are tenant-scoped
 """
 
@@ -70,18 +62,19 @@ from ..core.core_model import CoreModel
 
 
 class Ontology(CoreModel):
-    """Domain-specific knowledge extracted from files using custom agents.
+    """Domain-specific knowledge - either agent-extracted or direct-loaded.
 
     Attributes:
         name: Human-readable label for this ontology instance
-        file_id: Foreign key to File entity that was processed
-        agent_schema_id: Foreign key to Schema entity that performed extraction
-        provider_name: LLM provider used for extraction (e.g., "anthropic", "openai")
-        model_name: Specific model used (e.g., "claude-sonnet-4-5")
-        extracted_data: Structured data extracted by agent (arbitrary JSON)
+        uri: External source reference (git://, s3://, https://) for direct-loaded ontologies
+        file_id: Foreign key to File entity (optional - only for agent-extracted)
+        agent_schema_id: Schema that performed extraction (optional - only for agent-extracted)
+        provider_name: LLM provider used for extraction (optional)
+        model_name: Specific model used (optional)
+        extracted_data: Structured data - either extracted by agent or parsed from source
         confidence_score: Optional confidence score from extraction (0.0-1.0)
         extraction_timestamp: When extraction was performed
-        embedding_text: Text used for generating embedding (derived from extracted_data)
+        content: Text used for generating embedding
 
     Inherited from CoreModel:
         id: UUID or string identifier
@@ -93,10 +86,9 @@ class Ontology(CoreModel):
         graph_edges: Relationships to other entities
         metadata: Flexible metadata storage
         tags: Classification tags
-        column: Database schema metadata
 
     Example Usage:
-        # CV extraction
+        # Agent-extracted: CV parsing
         cv_ontology = Ontology(
             name="john-doe-cv-2024",
             file_id="file-uuid-123",
@@ -105,73 +97,74 @@ class Ontology(CoreModel):
             model_name="claude-sonnet-4-5-20250929",
             extracted_data={
                 "candidate_name": "John Doe",
-                "email": "john@example.com",
                 "skills": ["Python", "PostgreSQL", "Kubernetes"],
-                "experience": [
-                    {
-                        "company": "TechCorp",
-                        "role": "Senior Engineer",
-                        "years": 3,
-                        "achievements": ["Led migration to k8s", "Reduced costs 40%"]
-                    }
-                ],
-                "education": [
-                    {"degree": "BS Computer Science", "institution": "MIT", "year": 2018}
-                ]
             },
             confidence_score=0.95,
-            tags=["cv", "engineering", "senior-level"]
+            tags=["cv", "engineering"]
         )
 
-        # Contract extraction
-        contract_ontology = Ontology(
-            name="acme-supplier-agreement-2024",
-            file_id="file-uuid-456",
-            agent_schema_id="contract-parser-v2",
-            provider_name="openai",
-            model_name="gpt-4.1",
+        # Direct-loaded: Medical knowledge base from git
+        disorder_ontology = Ontology(
+            name="panic-disorder",
+            uri="git://bwolfson-siggie/Siggy-MVP/ontology/disorders/anxiety/panic-disorder.md",
+            content="# Panic Disorder\\n\\nPanic disorder is characterized by...",
             extracted_data={
-                "contract_type": "supplier_agreement",
-                "parties": [
-                    {"name": "ACME Corp", "role": "buyer"},
-                    {"name": "SupplyChain Inc", "role": "supplier"}
-                ],
-                "effective_date": "2024-01-01",
-                "termination_date": "2026-12-31",
-                "payment_terms": {
-                    "amount": 500000,
-                    "currency": "USD",
-                    "frequency": "quarterly"
-                },
-                "key_obligations": [
-                    "Supplier must deliver within 30 days",
-                    "Buyer must pay within 60 days of invoice"
-                ]
+                "type": "disorder",
+                "category": "anxiety",
+                "icd10": "F41.0",
+                "dsm5_criteria": ["A", "B", "C", "D"],
+            },
+            tags=["disorder", "anxiety", "dsm5"]
+        )
+
+        # Direct-loaded: Clinical procedure from git
+        scid_node = Ontology(
+            name="scid-5-f1",
+            uri="git://bwolfson-siggie/Siggy-MVP/ontology/procedures/scid-5/module-f/scid-5-f1.md",
+            content="# scid-5-f1: Panic Attack Screening\\n\\n...",
+            extracted_data={
+                "type": "procedure",
+                "module": "F",
+                "section": "Panic Disorder",
+                "dsm5_criterion": "Panic Attack Specifier",
             },
-            confidence_score=0.92,
-            tags=["contract", "supplier", "procurement"]
+            tags=["scid-5", "procedure", "anxiety"]
         )
     """
 
     # Core fields
     name: str
-    file_id: UUID | str
-    agent_schema_id: str  # Natural language label of Schema entity
+    uri: Optional[str] = None  # External source: git://, s3://, https://
 
-    # Extraction metadata
-    provider_name: str  # LLM provider (anthropic, openai, etc.)
-    model_name: str  # Specific model used
-    extracted_data: dict[str, Any]  # Arbitrary structured data from agent
+    # Agent extraction fields (optional - only for agent-extracted ontologies)
+    file_id: Optional[UUID | str] = None  # FK to File entity
+    agent_schema_id: Optional[str] = None  # Schema that performed extraction
+    provider_name: Optional[str] = None  # LLM provider (anthropic, openai, etc.)
+    model_name: Optional[str] = None  # Specific model used
+
+    # Data fields
+    extracted_data: Optional[dict[str, Any]] = None  # Structured data
     confidence_score: Optional[float] = None  # 0.0-1.0 if provided by agent
     extraction_timestamp: Optional[str] = None  # ISO8601 timestamp
 
-    # Semantic search support
-    embedding_text: Optional[str] = None  # Text for embedding generation
+    # Semantic search support - 'content' is a default embeddable field name
+    content: Optional[str] = None  # Text for embedding generation
 
     model_config = ConfigDict(
         json_schema_extra={
-            "description": "Domain-specific knowledge extracted from files using custom agents",
+            "description": "Domain-specific knowledge - agent-extracted or direct-loaded from external sources",
             "examples": [
+                {
+                    "name": "panic-disorder",
+                    "uri": "git://org/repo/ontology/disorders/anxiety/panic-disorder.md",
+                    "content": "# Panic Disorder\n\nPanic disorder is characterized by...",
+                    "extracted_data": {
+                        "type": "disorder",
+                        "category": "anxiety",
+                        "icd10": "F41.0"
+                    },
+                    "tags": ["disorder", "anxiety"]
+                },
                 {
                     "name": "john-doe-cv-2024",
                     "file_id": "550e8400-e29b-41d4-a716-446655440000",
@@ -180,8 +173,7 @@ class Ontology(CoreModel):
                     "model_name": "claude-sonnet-4-5-20250929",
                     "extracted_data": {
                         "candidate_name": "John Doe",
-                        "skills": ["Python", "PostgreSQL"],
-                        "experience": []
+                        "skills": ["Python", "PostgreSQL"]
                     },
                     "confidence_score": 0.95,
                     "tags": ["cv", "engineering"]

rem/models/entities/subscriber.py

@@ -0,0 +1,175 @@
+"""
+Subscriber - Email subscription management.
+
+This model stores subscribers who sign up via websites/apps.
+Subscribers can be collected before user registration for newsletters,
+updates, and approval-based access control.
+
+Key features:
+- Deterministic UUID from email (same email = same ID)
+- Approval workflow for access control
+- Tags for segmentation
+- Origin tracking for analytics
+"""
+
+import uuid
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Optional
+
+from pydantic import Field, EmailStr, model_validator
+
+from ..core import CoreModel
+
+
+class SubscriberStatus(str, Enum):
+    """Subscription status."""
+
+    ACTIVE = "active"           # Actively subscribed
+    UNSUBSCRIBED = "unsubscribed"  # User unsubscribed
+    BOUNCED = "bounced"         # Email bounced
+    PENDING = "pending"         # Pending confirmation (if double opt-in)
+
+
+class SubscriberOrigin(str, Enum):
+    """Where the subscription originated from."""
+
+    WEBSITE = "website"         # Main website subscribe form
+    LANDING_PAGE = "landing_page"  # Campaign landing page
+    APP = "app"                 # In-app subscription
+    IMPORT = "import"           # Bulk import
+    REFERRAL = "referral"       # Referred by another user
+    OTHER = "other"
+
+
+class Subscriber(CoreModel):
+    """
+    Email subscriber for newsletters and access control.
+
+    This model captures subscribers who sign up via the website, landing pages,
+    or in-app prompts. Uses deterministic UUID from email for natural upserts.
+
+    Access control via `approved` field:
+    - When email auth checks subscriber status, only approved subscribers
+      can complete login (if approval is enabled in settings).
+    - Subscribers can be pre-approved, or approved manually/automatically.
+
+    Usage:
+        from rem.services.postgres import Repository
+        from rem.models.entities import Subscriber, SubscriberStatus
+
+        repo = Repository(Subscriber, db=db)
+
+        # Create subscriber (ID auto-generated from email)
+        subscriber = Subscriber(
+            email="user@example.com",
+            name="John Doe",
+            origin=SubscriberOrigin.WEBSITE,
+        )
+        await repo.upsert(subscriber)
+
+        # Check if approved for login
+        subscriber = await repo.get_by_id(subscriber.id, tenant_id="default")
+        if subscriber and subscriber.approved:
+            # Allow login
+            pass
+    """
+
+    # Required field
+    email: EmailStr = Field(
+        description="Subscriber's email address (unique identifier)"
+    )
+
+    # Optional fields
+    name: Optional[str] = Field(
+        default=None,
+        description="Subscriber's name (optional)"
+    )
+
+    comment: Optional[str] = Field(
+        default=None,
+        max_length=500,
+        description="Optional comment or message from subscriber"
+    )
+
+    status: SubscriberStatus = Field(
+        default=SubscriberStatus.ACTIVE,
+        description="Current subscription status"
+    )
+
+    # Access control
+    approved: bool = Field(
+        default=False,
+        description="Whether subscriber is approved for login (for approval workflows)"
+    )
+
+    approved_at: Optional[datetime] = Field(
+        default=None,
+        description="When the subscriber was approved"
+    )
+
+    approved_by: Optional[str] = Field(
+        default=None,
+        description="Who approved the subscriber (user ID or 'system')"
+    )
+
+    # Origin tracking
+    origin: SubscriberOrigin = Field(
+        default=SubscriberOrigin.WEBSITE,
+        description="Where the subscription originated"
+    )
+
+    origin_detail: Optional[str] = Field(
+        default=None,
+        description="Additional origin context (e.g., campaign name, page URL)"
+    )
+
+    # Timestamps
+    subscribed_at: datetime = Field(
+        default_factory=lambda: datetime.now(timezone.utc),
+        description="When the subscription was created"
+    )
+
+    unsubscribed_at: Optional[datetime] = Field(
+        default=None,
+        description="When the user unsubscribed (if applicable)"
+    )
+
+    # Compliance
+    ip_address: Optional[str] = Field(
+        default=None,
+        description="IP address at subscription time (for compliance)"
+    )
+
+    user_agent: Optional[str] = Field(
+        default=None,
+        description="Browser user agent at subscription time"
+    )
+
+    # Segmentation
+    tags: list[str] = Field(
+        default_factory=list,
+        description="Tags for segmentation (e.g., ['early-access', 'beta'])"
+    )
+
+    @staticmethod
+    def email_to_uuid(email: str) -> uuid.UUID:
+        """Generate a deterministic UUID from an email address.
+
+        Uses UUID v5 with DNS namespace for consistency with
+        EmailService.generate_user_id_from_email().
+
+        Args:
+            email: Email address
+
+        Returns:
+            Deterministic UUID
+        """
+        return uuid.uuid5(uuid.NAMESPACE_DNS, email.lower().strip())
+
+    @model_validator(mode="after")
+    def set_id_from_email(self) -> "Subscriber":
+        """Auto-generate deterministic ID from email for natural upsert."""
+        if self.email:
+            self.id = self.email_to_uuid(self.email)
+        return self

rem/models/entities/user.py

@@ -22,6 +22,7 @@ from ..core import CoreModel
 class UserTier(str, Enum):
     """User subscription tier for feature gating."""
 
+    BLOCKED = "blocked"  # User is blocked from logging in
     ANONYMOUS = "anonymous"
     FREE = "free"
     BASIC = "basic"