remdb 0.3.163__py3-none-any.whl → 0.3.181__py3-none-any.whl

rem/models/entities/ontology.py

@@ -1,63 +1,55 @@
-"""Ontology entity for tenant-specific knowledge extensions.
+"""Ontology entity for domain-specific knowledge.
 
-**What is Ontology Extraction?**
+**What are Ontologies?**
 
-Ontologies are **domain-specific structured knowledge** extracted from files using custom
-agent schemas. They extend REM's normal file processing pipeline with tenant-specific
-parsers that extract structured data the standard chunking pipeline would miss.
+Ontologies are **domain-specific structured knowledge** that can be:
+1. **Extracted** from files using custom agent schemas (agent-extracted)
+2. **Loaded directly** from external sources like git repos or S3 (direct-loaded)
 
-**Normal File Processing:**
-File → extract text → chunk → embed → resources (semantic search ready)
+**Use Case 1: Agent-Extracted Ontologies**
 
-**Ontology Processing (Tenant Knowledge Extensions):**
 File → custom agent → structured JSON → ontology (domain knowledge)
 
-**Why Ontologies?**
-- Standard chunking gives you semantic search over raw content
-- Ontologies give you **structured queryable fields** from domain logic
-- Example: A contract PDF becomes both searchable chunks AND a structured record with
-  parties, dates, payment terms, obligations as queryable fields
+Example: A contract PDF becomes a structured record with parties, dates, payment terms.
+
+**Use Case 2: Direct-Loaded Ontologies (Knowledge Bases)**
+
+External source (git/S3) → load → ontology (reference knowledge)
+
+Example: A psychiatric ontology of disorders, symptoms, and drugs loaded from markdown
+files in a git repository. Each markdown file becomes an ontology node with:
+- `uri`: git path (e.g., `git://org/repo/ontology/disorders/anxiety/panic-disorder.md`)
+- `content`: markdown content for embedding/search
+- `extracted_data`: parsed frontmatter or structure
 
 **Architecture:**
-- Runs as part of dreaming worker (background knowledge extraction)
-- OntologyConfig defines which files trigger which extractors (MIME type, URI pattern, tags)
+- Runs as part of dreaming worker (background knowledge extraction) OR
+- Loaded directly via `rem db load` for external knowledge bases
+- OntologyConfig defines which files trigger which extractors
 - Multiple ontologies per file (apply different domain lenses)
-- Tenant-scoped: Each tenant can define their own extractors
+- Tenant-scoped: Each tenant can define their own extractors and knowledge bases
 
 **Use Cases:**
 
-1. **Recruitment (CV Parsing)**
-   - Standard pipeline: Chunks for "find me candidates with Python experience"
-   - Ontology: Structured fields for filtering/sorting (years_experience, seniority_level, skills[])
-
-2. **Legal (Contract Analysis)**
-   - Standard pipeline: Semantic search over contract text
-   - Ontology: Queryable fields (parties, effective_date, payment_amount, key_obligations[])
+1. **Recruitment (CV Parsing)** - Agent-extracted
+   - Ontology: Structured fields for filtering/sorting (years_experience, skills[])
 
-3. **Medical (Health Records)**
-   - Standard pipeline: Find mentions of conditions
-   - Ontology: Structured diagnoses, medications, dosages, treatment plans
+2. **Legal (Contract Analysis)** - Agent-extracted
+   - Ontology: Queryable fields (parties, effective_date, payment_amount)
 
-4. **Finance (Report Analysis)**
-   - Standard pipeline: Search for financial terms
-   - Ontology: Extracted metrics, risk_flags, trends, forecasts
+3. **Medical Knowledge Base** - Direct-loaded
+   - Ontology: Disorders, symptoms, medications from curated markdown files
+   - Enables semantic search over psychiatric/medical domain knowledge
 
-**Example Flow:**
-1. Tenant creates OntologyConfig: "Run cv-parser-v1 on files with mime_type='application/pdf' and tags=['resume']"
-2. File uploaded with tags=["resume"]
-3. Normal processing: File → chunks → resources
-4. Dreaming worker detects matching OntologyConfig
-5. Loads cv-parser-v1 agent schema from database
-6. Runs agent on file content → extracts structured data
-7. Stores Ontology with extracted_data = {candidate_name, skills, experience, education, ...}
-8. Ontology is now queryable via LOOKUP, SEARCH, or direct SQL
+4. **Documentation/Procedures** - Direct-loaded
+   - Ontology: Clinical procedures (e.g., SCID-5 assessment steps)
+   - Reference material accessible via RAG
 
 **Design:**
-- Each ontology links to a File via file_id
-- Agent schema tracked via agent_schema_id (human-readable label, not UUID)
-- Structured data in `extracted_data` (arbitrary JSON, schema defined by agent)
-- Embeddings generated for semantic search (configurable fields via agent schema)
-- Multiple ontologies per file using different schemas
+- `file_id` and `agent_schema_id` are optional (only needed for agent-extracted)
+- `uri` field for external source references (git://, s3://, https://)
+- Structured data in `extracted_data` (arbitrary JSON)
+- Embeddings generated for semantic search via `content` field
 - Tenant-isolated: OntologyConfigs are tenant-scoped
 """
 
@@ -70,18 +62,19 @@ from ..core.core_model import CoreModel
 
 
 class Ontology(CoreModel):
-    """Domain-specific knowledge extracted from files using custom agents.
+    """Domain-specific knowledge - either agent-extracted or direct-loaded.
 
     Attributes:
         name: Human-readable label for this ontology instance
-        file_id: Foreign key to File entity that was processed
-        agent_schema_id: Foreign key to Schema entity that performed extraction
-        provider_name: LLM provider used for extraction (e.g., "anthropic", "openai")
-        model_name: Specific model used (e.g., "claude-sonnet-4-5")
-        extracted_data: Structured data extracted by agent (arbitrary JSON)
+        uri: External source reference (git://, s3://, https://) for direct-loaded ontologies
+        file_id: Foreign key to File entity (optional - only for agent-extracted)
+        agent_schema_id: Schema that performed extraction (optional - only for agent-extracted)
+        provider_name: LLM provider used for extraction (optional)
+        model_name: Specific model used (optional)
+        extracted_data: Structured data - either extracted by agent or parsed from source
         confidence_score: Optional confidence score from extraction (0.0-1.0)
         extraction_timestamp: When extraction was performed
-        embedding_text: Text used for generating embedding (derived from extracted_data)
+        content: Text used for generating embedding
 
     Inherited from CoreModel:
         id: UUID or string identifier
@@ -93,10 +86,9 @@ class Ontology(CoreModel):
         graph_edges: Relationships to other entities
         metadata: Flexible metadata storage
         tags: Classification tags
-        column: Database schema metadata
 
     Example Usage:
-        # CV extraction
+        # Agent-extracted: CV parsing
         cv_ontology = Ontology(
             name="john-doe-cv-2024",
             file_id="file-uuid-123",
@@ -105,73 +97,74 @@ class Ontology(CoreModel):
             model_name="claude-sonnet-4-5-20250929",
             extracted_data={
                 "candidate_name": "John Doe",
-                "email": "john@example.com",
                 "skills": ["Python", "PostgreSQL", "Kubernetes"],
-                "experience": [
-                    {
-                        "company": "TechCorp",
-                        "role": "Senior Engineer",
-                        "years": 3,
-                        "achievements": ["Led migration to k8s", "Reduced costs 40%"]
-                    }
-                ],
-                "education": [
-                    {"degree": "BS Computer Science", "institution": "MIT", "year": 2018}
-                ]
             },
             confidence_score=0.95,
-            tags=["cv", "engineering", "senior-level"]
+            tags=["cv", "engineering"]
         )
 
-        # Contract extraction
-        contract_ontology = Ontology(
-            name="acme-supplier-agreement-2024",
-            file_id="file-uuid-456",
-            agent_schema_id="contract-parser-v2",
-            provider_name="openai",
-            model_name="gpt-4.1",
+        # Direct-loaded: Medical knowledge base from git
+        disorder_ontology = Ontology(
+            name="panic-disorder",
+            uri="git://bwolfson-siggie/Siggy-MVP/ontology/disorders/anxiety/panic-disorder.md",
+            content="# Panic Disorder\\n\\nPanic disorder is characterized by...",
             extracted_data={
-                "contract_type": "supplier_agreement",
-                "parties": [
-                    {"name": "ACME Corp", "role": "buyer"},
-                    {"name": "SupplyChain Inc", "role": "supplier"}
-                ],
-                "effective_date": "2024-01-01",
-                "termination_date": "2026-12-31",
-                "payment_terms": {
-                    "amount": 500000,
-                    "currency": "USD",
-                    "frequency": "quarterly"
-                },
-                "key_obligations": [
-                    "Supplier must deliver within 30 days",
-                    "Buyer must pay within 60 days of invoice"
-                ]
+                "type": "disorder",
+                "category": "anxiety",
+                "icd10": "F41.0",
+                "dsm5_criteria": ["A", "B", "C", "D"],
+            },
+            tags=["disorder", "anxiety", "dsm5"]
+        )
+
+        # Direct-loaded: Clinical procedure from git
+        scid_node = Ontology(
+            name="scid-5-f1",
+            uri="git://bwolfson-siggie/Siggy-MVP/ontology/procedures/scid-5/module-f/scid-5-f1.md",
+            content="# scid-5-f1: Panic Attack Screening\\n\\n...",
+            extracted_data={
+                "type": "procedure",
+                "module": "F",
+                "section": "Panic Disorder",
+                "dsm5_criterion": "Panic Attack Specifier",
             },
-            confidence_score=0.92,
-            tags=["contract", "supplier", "procurement"]
+            tags=["scid-5", "procedure", "anxiety"]
         )
     """
 
     # Core fields
     name: str
-    file_id: UUID | str
-    agent_schema_id: str  # Natural language label of Schema entity
+    uri: Optional[str] = None  # External source: git://, s3://, https://
 
-    # Extraction metadata
-    provider_name: str  # LLM provider (anthropic, openai, etc.)
-    model_name: str  # Specific model used
-    extracted_data: dict[str, Any]  # Arbitrary structured data from agent
+    # Agent extraction fields (optional - only for agent-extracted ontologies)
+    file_id: Optional[UUID | str] = None  # FK to File entity
+    agent_schema_id: Optional[str] = None  # Schema that performed extraction
+    provider_name: Optional[str] = None  # LLM provider (anthropic, openai, etc.)
+    model_name: Optional[str] = None  # Specific model used
+
+    # Data fields
+    extracted_data: Optional[dict[str, Any]] = None  # Structured data
     confidence_score: Optional[float] = None  # 0.0-1.0 if provided by agent
     extraction_timestamp: Optional[str] = None  # ISO8601 timestamp
 
-    # Semantic search support
-    embedding_text: Optional[str] = None  # Text for embedding generation
+    # Semantic search support - 'content' is a default embeddable field name
+    content: Optional[str] = None  # Text for embedding generation
 
     model_config = ConfigDict(
         json_schema_extra={
-            "description": "Domain-specific knowledge extracted from files using custom agents",
+            "description": "Domain-specific knowledge - agent-extracted or direct-loaded from external sources",
             "examples": [
+                {
+                    "name": "panic-disorder",
+                    "uri": "git://org/repo/ontology/disorders/anxiety/panic-disorder.md",
+                    "content": "# Panic Disorder\n\nPanic disorder is characterized by...",
+                    "extracted_data": {
+                        "type": "disorder",
+                        "category": "anxiety",
+                        "icd10": "F41.0"
+                    },
+                    "tags": ["disorder", "anxiety"]
+                },
                 {
                     "name": "john-doe-cv-2024",
                     "file_id": "550e8400-e29b-41d4-a716-446655440000",
@@ -180,8 +173,7 @@ class Ontology(CoreModel):
                     "model_name": "claude-sonnet-4-5-20250929",
                     "extracted_data": {
                         "candidate_name": "John Doe",
-                        "skills": ["Python", "PostgreSQL"],
-                        "experience": []
+                        "skills": ["Python", "PostgreSQL"]
                     },
                     "confidence_score": 0.95,
                     "tags": ["cv", "engineering"]

rem/schemas/agents/core/agent-builder.yaml

@@ -2,65 +2,148 @@ type: object
 description: |
   # Agent Builder - Create Custom AI Agents Through Conversation
 
-  You help users create custom AI agents by chatting with them naturally.
-  Gather requirements conversationally, show previews, and save the agent when ready.
+  You help users create custom AI agents for the REM platform through natural conversation.
+  Guide them step-by-step, gather requirements, show previews, and save when ready.
 
   ## Your Workflow
 
   1. **Understand the need**: Ask what they want the agent to do
-  2. **Define personality**: Help them choose tone and style
-  3. **Structure outputs**: If needed, define what data the agent captures
-  4. **Preview**: Show them what the agent will look like
-  5. **Save**: Use `save_agent` tool to persist it
+  2. **Define personality**: Help them choose tone and communication style
+  3. **Set guardrails**: What should the agent NOT do?
+  4. **Structure outputs**: Define what data the agent captures (optional)
+  5. **Preview**: Show them what the agent will look like
+  6. **Save**: Use `save_agent` tool to persist it
 
   ## Conversation Style
 
   Be friendly and helpful. Ask one or two questions at a time.
   Don't overwhelm with options - guide them step by step.
 
-  ## Gathering Requirements
+  ## IMPORTANT: Tool Usage
+
+  - `save_agent` - Use ONLY in Step 6 when user approves the preview
+  - `get_agents_list` - Use if user asks to see existing agents as examples
+  - `get_agent_schema` - Use to load a specific agent (like "rem") as reference
+
+  DO NOT loop on tools. If a user asks for examples, call get_agents_list ONCE,
+  then discuss what you found. This is a conversational workflow.
+
+  ## Step 1: Identity & Purpose
 
   Ask about:
-  - What should this agent help with?
-  - What tone should it have? (casual, professional, empathetic, etc.)
-  - Should it capture any specific information? (optional)
-  - What should it be called?
+  - What should this agent help with? (primary purpose)
+  - What would you like to call it? (suggest kebab-case like "sales-assistant")
+  - What role/persona should it embody?
+
+  ## Step 2: Tone & Communication Style
+
+  Help define tone using this framework:
+
+  | Dimension | Options |
+  |-----------|---------|
+  | Formality | casual, conversational, professional, formal |
+  | Warmth | empathetic, friendly, neutral, businesslike |
+  | Pace | patient, balanced, efficient, direct |
+  | Expertise | peer, guide, expert, authority |
+
+  Ask: "What tone feels right? For example, should it be friendly and casual, or more professional?"
+
+  ## Step 3: Guardrails
+
+  Ask what the agent should NOT do:
+  - Topics to avoid?
+  - Actions it shouldn't take?
+  - Boundaries to respect?
 
-  ## Preview Format
+  Example guardrails:
+  - "Never provide medical/legal/financial advice"
+  - "Don't make promises about timelines"
+  - "Always recommend consulting a professional for serious issues"
 
-  Before saving, show a preview using markdown:
+  ## Step 4: Structured Outputs (Optional)
+
+  Most agents just need an `answer` field. But some use cases benefit from structured data:
+
+  | Field | Type | Description |
+  |-------|------|-------------|
+  | answer | string | Natural language response (always required) |
+  | confidence | number | 0.0-1.0 confidence score |
+  | category | string | Classification of the request |
+  | follow_up_needed | boolean | Whether follow-up is required |
+
+  Field types available:
+  - `string` - text values
+  - `number` - numeric values (can add minimum/maximum)
+  - `boolean` - true/false
+  - `array` - list of items
+  - `string` with `enum` - fixed set of choices
+
+  Only suggest structured outputs if the use case clearly benefits from them.
+
+  ## Step 5: Preview
+
+  Before saving, show a preview:
 
   ```
   ## Agent Preview: {name}
 
-  **Personality:**
-  {brief description of tone and approach}
+  **Purpose:** {brief description}
+
+  **Personality:** {tone and approach}
 
   **System Prompt:**
   {the actual prompt that will guide the agent}
 
-  **Structured Fields:** (if any)
+  **Guardrails:**
+  - {guardrail 1}
+  - {guardrail 2}
+
+  **Structured Fields:** (if any beyond answer)
   | Field | Type | Description |
   |-------|------|-------------|
   | answer | string | Response to user |
-  | ... | ... | ... |
   ```
 
-  Ask: "Does this look good? I can save it now or we can adjust anything."
+  Ask: "Does this look good? I can save it now or adjust anything."
 
-  ## Saving the Agent
+  ## Step 6: Save the Agent
 
   When the user approves, call `save_agent` with:
   - `name`: kebab-case name (e.g., "customer-support-bot")
-  - `description`: The full system prompt
+  - `description`: The full system prompt (this is the most important part!)
   - `properties`: Structured output fields (optional, defaults to just "answer")
   - `required`: Which fields are required (optional, defaults to ["answer"])
   - `tags`: Relevant tags for organization
 
-  ## Default Tools
+  ## Writing Great System Prompts
+
+  The `description` field IS the system prompt. Make it:
+
+  1. **Clear about role**: "You are a [role] that helps users [do what]"
+  2. **Specific about behavior**: How should it respond? What's the communication style?
+  3. **Explicit about guardrails**: What should it NOT do?
+  4. **Structured with sections**: Use markdown headers for organization
+
+  Example structure:
+  ```
+  # [Agent Name]
 
-  All agents automatically get `search_rem` and `register_metadata` tools.
-  You don't need to specify these.
+  You are a [role description]. Your purpose is to [main goal].
+
+  ## Communication Style
+  [Tone, approach, language preferences]
+
+  ## Core Responsibilities
+  - [Responsibility 1]
+  - [Responsibility 2]
+
+  ## Guardrails
+  - Never [thing to avoid]
+  - Always [thing to ensure]
+
+  ## How to Respond
+  [Guidelines for responses]
+  ```
 
   ## Example Conversation
 
@@ -68,47 +151,62 @@ description: |
 
   You: "Great idea! Let me help you create an email writing assistant.
 
-  What tone should it have? For example:
+  First, what tone should it have? For example:
   - Professional and formal
   - Friendly but polished
   - Direct and concise"
 
   User: "Friendly but polished"
 
-  You: "Perfect! Should it just write emails, or also help with things like:
+  You: "Perfect! Should it focus just on writing emails, or also help with:
   - Suggesting subject lines
   - Adjusting tone for different recipients
   - Summarizing long email threads"
 
   User: "All of those would be helpful"
 
-  You: "Let me show you a preview:
+  You: "Great! Any things it should NOT do? For example, some email assistants avoid:
+  - Writing emails that could be seen as legally binding
+  - Making commitments on behalf of the user"
+
+  User: "Yeah, it shouldn't make commitments or promises"
+
+  You: "Perfect! Here's a preview:
 
   ## Agent Preview: email-assistant
 
-  **Personality:** Friendly, professional, helpful writing partner
+  **Purpose:** Help users write polished, professional emails
+
+  **Personality:** Friendly, helpful, professional writing partner
 
   **System Prompt:**
+  # Email Writing Assistant
+
   You are a friendly email writing assistant. Help users craft polished,
-  professional emails while maintaining a warm tone. You can:
-  - Write new emails from scratch
+  professional emails while maintaining a warm, approachable tone.
+
+  ## What You Can Do
+  - Write new emails from scratch based on user's intent
   - Suggest compelling subject lines
-  - Adjust tone for different audiences
+  - Adjust tone for different audiences (colleagues, executives, clients)
   - Summarize long email threads
+  - Proofread and improve existing drafts
 
-  Always ask clarifying questions if the request is unclear.
+  ## Communication Style
+  Be helpful and collaborative. Suggest improvements but respect the user's voice.
+  Ask clarifying questions when the request is ambiguous.
 
-  **Structured Fields:**
-  | Field | Type | Description |
-  |-------|------|-------------|
-  | answer | string | Your response or the drafted email |
+  ## Guardrails
+  - Never write emails that make commitments or promises on behalf of the user
+  - Don't write anything that could be legally binding
+  - Always let the user review before sending
 
   Does this look good? I can save it now or adjust anything."
 
   User: "Looks great, save it!"
 
   You: *calls save_agent tool*
-  "Done! Your email-assistant is ready. Use `/custom-agent email-assistant` to start chatting with it."
+  "Done! Your email-assistant is ready to use."
 
 properties:
   answer:
@@ -121,14 +219,17 @@ required:
 json_schema_extra:
   kind: agent
   name: agent-builder
-  version: "1.0.0"
+  version: "1.2.0"
   tags:
     - meta
     - builder
+  structured_output: false  # Stream text responses, don't return JSON
+  mcp_servers: []  # Disable default MCP tools to prevent search_rem looping
+  resources:
+    - uri: rem://agents
+      description: "List all available agent schemas with descriptions"
+    - uri: rem://agents/{agent_name}
+      description: "Load a specific agent schema by name (e.g., 'rem', 'siggy')"
   tools:
     - name: save_agent
-      description: "Save the agent schema to make it available for use"
-    - name: search_rem
-      description: "Search for existing agents as examples"
-    - name: register_metadata
-      description: "Record session metadata"
+      description: "Save the agent schema. Only call when user approves the preview in Step 6."

rem/services/email/service.py

@@ -200,8 +200,8 @@ class EmailService:
         """
         Generate a deterministic UUID from email address.
 
-        Uses UUID v5 with DNS namespace for consistency.
-        Same email always produces same UUID.
+        Uses the centralized email_to_user_id() for consistency.
+        Same email always produces same UUID (bijection).
 
         Args:
             email: Email address
@@ -209,7 +209,8 @@ class EmailService:
         Returns:
             UUID string
         """
-        return str(uuid.uuid5(uuid.NAMESPACE_DNS, email.lower().strip()))
+        from rem.utils.user_id import email_to_user_id
+        return email_to_user_id(email)
 
     async def send_login_code(
         self,
@@ -375,8 +376,17 @@ class EmailService:
             await user_repo.upsert(existing_user)
             return {"allowed": True, "error": None}
         else:
-            # New user - check if domain is trusted
-            if settings and hasattr(settings, 'email') and settings.email.trusted_domain_list:
+            # New user - first check if they're a subscriber (by email lookup)
+            from ...models.entities import Subscriber
+            subscriber_repo = Repository(Subscriber, db=db)
+            existing_subscriber = await subscriber_repo.find_one({"email": email})
+
+            if existing_subscriber:
+                # Subscriber exists - allow them to create account
+                # (approved field may not exist in older schemas, so just check existence)
+                logger.info(f"Subscriber {email} creating user account")
+            elif settings and hasattr(settings, 'email') and settings.email.trusted_domain_list:
+                # Not an approved subscriber - check if domain is trusted
                 if not settings.email.is_domain_trusted(email):
                     email_domain = email.split("@")[-1]
                     logger.warning(f"Untrusted domain attempted signup: {email_domain}")
@@ -393,7 +403,8 @@ class EmailService:
             new_user = User(
                 id=uuid.UUID(user_id),
                 tenant_id=tenant_id,
-                name=email.split("@")[0],  # Default name from email
+                user_id=user_id,  # UUID5 hash of email (same as id)
+                name=email,  # Full email as entity_key for LOOKUP
                 email=email,
                 role=user_role,
                 metadata=login_metadata,