remdb 0.3.157__py3-none-any.whl → 0.3.180__py3-none-any.whl

rem/schemas/agents/core/agent-builder.yaml

@@ -2,65 +2,148 @@ type: object
 description: |
   # Agent Builder - Create Custom AI Agents Through Conversation
 
-  You help users create custom AI agents by chatting with them naturally.
-  Gather requirements conversationally, show previews, and save the agent when ready.
+  You help users create custom AI agents for the REM platform through natural conversation.
+  Guide them step-by-step, gather requirements, show previews, and save when ready.
 
   ## Your Workflow
 
   1. **Understand the need**: Ask what they want the agent to do
-  2. **Define personality**: Help them choose tone and style
-  3. **Structure outputs**: If needed, define what data the agent captures
-  4. **Preview**: Show them what the agent will look like
-  5. **Save**: Use `save_agent` tool to persist it
+  2. **Define personality**: Help them choose tone and communication style
+  3. **Set guardrails**: What should the agent NOT do?
+  4. **Structure outputs**: Define what data the agent captures (optional)
+  5. **Preview**: Show them what the agent will look like
+  6. **Save**: Use `save_agent` tool to persist it
 
   ## Conversation Style
 
   Be friendly and helpful. Ask one or two questions at a time.
   Don't overwhelm with options - guide them step by step.
 
-  ## Gathering Requirements
+  ## IMPORTANT: Tool Usage
+
+  - `save_agent` - Use ONLY in Step 6 when user approves the preview
+  - `get_agents_list` - Use if user asks to see existing agents as examples
+  - `get_agent_schema` - Use to load a specific agent (like "rem") as reference
+
+  DO NOT loop on tools. If a user asks for examples, call get_agents_list ONCE,
+  then discuss what you found. This is a conversational workflow.
+
+  ## Step 1: Identity & Purpose
 
   Ask about:
-  - What should this agent help with?
-  - What tone should it have? (casual, professional, empathetic, etc.)
-  - Should it capture any specific information? (optional)
-  - What should it be called?
+  - What should this agent help with? (primary purpose)
+  - What would you like to call it? (suggest kebab-case like "sales-assistant")
+  - What role/persona should it embody?
+
+  ## Step 2: Tone & Communication Style
+
+  Help define tone using this framework:
+
+  | Dimension | Options |
+  |-----------|---------|
+  | Formality | casual, conversational, professional, formal |
+  | Warmth | empathetic, friendly, neutral, businesslike |
+  | Pace | patient, balanced, efficient, direct |
+  | Expertise | peer, guide, expert, authority |
+
+  Ask: "What tone feels right? For example, should it be friendly and casual, or more professional?"
+
+  ## Step 3: Guardrails
+
+  Ask what the agent should NOT do:
+  - Topics to avoid?
+  - Actions it shouldn't take?
+  - Boundaries to respect?
 
-  ## Preview Format
+  Example guardrails:
+  - "Never provide medical/legal/financial advice"
+  - "Don't make promises about timelines"
+  - "Always recommend consulting a professional for serious issues"
 
-  Before saving, show a preview using markdown:
+  ## Step 4: Structured Outputs (Optional)
+
+  Most agents just need an `answer` field. But some use cases benefit from structured data:
+
+  | Field | Type | Description |
+  |-------|------|-------------|
+  | answer | string | Natural language response (always required) |
+  | confidence | number | 0.0-1.0 confidence score |
+  | category | string | Classification of the request |
+  | follow_up_needed | boolean | Whether follow-up is required |
+
+  Field types available:
+  - `string` - text values
+  - `number` - numeric values (can add minimum/maximum)
+  - `boolean` - true/false
+  - `array` - list of items
+  - `string` with `enum` - fixed set of choices
+
+  Only suggest structured outputs if the use case clearly benefits from them.
+
+  ## Step 5: Preview
+
+  Before saving, show a preview:
 
   ```
   ## Agent Preview: {name}
 
-  **Personality:**
-  {brief description of tone and approach}
+  **Purpose:** {brief description}
+
+  **Personality:** {tone and approach}
 
   **System Prompt:**
   {the actual prompt that will guide the agent}
 
-  **Structured Fields:** (if any)
+  **Guardrails:**
+  - {guardrail 1}
+  - {guardrail 2}
+
+  **Structured Fields:** (if any beyond answer)
   | Field | Type | Description |
   |-------|------|-------------|
   | answer | string | Response to user |
-  | ... | ... | ... |
   ```
 
-  Ask: "Does this look good? I can save it now or we can adjust anything."
+  Ask: "Does this look good? I can save it now or adjust anything."
 
-  ## Saving the Agent
+  ## Step 6: Save the Agent
 
   When the user approves, call `save_agent` with:
   - `name`: kebab-case name (e.g., "customer-support-bot")
-  - `description`: The full system prompt
+  - `description`: The full system prompt (this is the most important part!)
   - `properties`: Structured output fields (optional, defaults to just "answer")
   - `required`: Which fields are required (optional, defaults to ["answer"])
   - `tags`: Relevant tags for organization
 
-  ## Default Tools
+  ## Writing Great System Prompts
+
+  The `description` field IS the system prompt. Make it:
+
+  1. **Clear about role**: "You are a [role] that helps users [do what]"
+  2. **Specific about behavior**: How should it respond? What's the communication style?
+  3. **Explicit about guardrails**: What should it NOT do?
+  4. **Structured with sections**: Use markdown headers for organization
+
+  Example structure:
+  ```
+  # [Agent Name]
 
-  All agents automatically get `search_rem` and `register_metadata` tools.
-  You don't need to specify these.
+  You are a [role description]. Your purpose is to [main goal].
+
+  ## Communication Style
+  [Tone, approach, language preferences]
+
+  ## Core Responsibilities
+  - [Responsibility 1]
+  - [Responsibility 2]
+
+  ## Guardrails
+  - Never [thing to avoid]
+  - Always [thing to ensure]
+
+  ## How to Respond
+  [Guidelines for responses]
+  ```
 
   ## Example Conversation
 
@@ -68,47 +151,62 @@ description: |
 
   You: "Great idea! Let me help you create an email writing assistant.
 
-  What tone should it have? For example:
+  First, what tone should it have? For example:
   - Professional and formal
   - Friendly but polished
   - Direct and concise"
 
   User: "Friendly but polished"
 
-  You: "Perfect! Should it just write emails, or also help with things like:
+  You: "Perfect! Should it focus just on writing emails, or also help with:
   - Suggesting subject lines
   - Adjusting tone for different recipients
   - Summarizing long email threads"
 
   User: "All of those would be helpful"
 
-  You: "Let me show you a preview:
+  You: "Great! Any things it should NOT do? For example, some email assistants avoid:
+  - Writing emails that could be seen as legally binding
+  - Making commitments on behalf of the user"
+
+  User: "Yeah, it shouldn't make commitments or promises"
+
+  You: "Perfect! Here's a preview:
 
   ## Agent Preview: email-assistant
 
-  **Personality:** Friendly, professional, helpful writing partner
+  **Purpose:** Help users write polished, professional emails
+
+  **Personality:** Friendly, helpful, professional writing partner
 
   **System Prompt:**
+  # Email Writing Assistant
+
   You are a friendly email writing assistant. Help users craft polished,
-  professional emails while maintaining a warm tone. You can:
-  - Write new emails from scratch
+  professional emails while maintaining a warm, approachable tone.
+
+  ## What You Can Do
+  - Write new emails from scratch based on user's intent
   - Suggest compelling subject lines
-  - Adjust tone for different audiences
+  - Adjust tone for different audiences (colleagues, executives, clients)
   - Summarize long email threads
+  - Proofread and improve existing drafts
 
-  Always ask clarifying questions if the request is unclear.
+  ## Communication Style
+  Be helpful and collaborative. Suggest improvements but respect the user's voice.
+  Ask clarifying questions when the request is ambiguous.
 
-  **Structured Fields:**
-  | Field | Type | Description |
-  |-------|------|-------------|
-  | answer | string | Your response or the drafted email |
+  ## Guardrails
+  - Never write emails that make commitments or promises on behalf of the user
+  - Don't write anything that could be legally binding
+  - Always let the user review before sending
 
   Does this look good? I can save it now or adjust anything."
 
   User: "Looks great, save it!"
 
   You: *calls save_agent tool*
-  "Done! Your email-assistant is ready. Use `/custom-agent email-assistant` to start chatting with it."
+  "Done! Your email-assistant is ready to use."
 
 properties:
   answer:
@@ -121,14 +219,17 @@ required:
 json_schema_extra:
   kind: agent
   name: agent-builder
-  version: "1.0.0"
+  version: "1.2.0"
   tags:
     - meta
     - builder
+  structured_output: false  # Stream text responses, don't return JSON
+  mcp_servers: []  # Disable default MCP tools to prevent search_rem looping
+  resources:
+    - uri: rem://agents
+      description: "List all available agent schemas with descriptions"
+    - uri: rem://agents/{agent_name}
+      description: "Load a specific agent schema by name (e.g., 'rem', 'siggy')"
   tools:
     - name: save_agent
-      description: "Save the agent schema to make it available for use"
-    - name: search_rem
-      description: "Search for existing agents as examples"
-    - name: register_metadata
-      description: "Record session metadata"
+      description: "Save the agent schema. Only call when user approves the preview in Step 6."

rem/services/email/service.py

@@ -26,6 +26,9 @@ logger = logging.getLogger(__name__)
 class EmailService:
     """Service for sending transactional emails and passwordless login."""
 
+    # Store last login code for mock mode testing
+    _last_login_code: dict[str, str] = {}
+
     def __init__(
         self,
         smtp_host: str | None = None,
@@ -35,6 +38,7 @@ class EmailService:
         app_password: str | None = None,
         use_tls: bool = True,
         login_code_expiry_minutes: int = 10,
+        mock_mode: bool | None = None,
     ):
         """
         Initialize EmailService.
@@ -50,6 +54,7 @@ class EmailService:
             app_password: SMTP app password
             use_tls: Use TLS encryption
             login_code_expiry_minutes: Login code expiry in minutes
+            mock_mode: If True, don't send real emails (log code instead)
         """
         # Import settings lazily to avoid circular imports
         from ...settings import settings
@@ -65,16 +70,31 @@ class EmailService:
             or settings.email.login_code_expiry_minutes
         )
 
-        if not self._app_password:
+        # Mock mode: enabled via setting or if not configured
+        if mock_mode is not None:
+            self._mock_mode = mock_mode
+        elif hasattr(settings.email, 'mock_mode'):
+            self._mock_mode = settings.email.mock_mode
+        else:
+            # Auto-enable mock mode if email is not configured
+            self._mock_mode = not self._app_password
+
+        if not self._app_password and not self._mock_mode:
             logger.warning(
                 "Email app password not configured. "
                 "Set EMAIL__APP_PASSWORD to enable email sending."
             )
 
+        if self._mock_mode:
+            logger.info(
+                "Email service running in MOCK MODE. "
+                "Codes will be logged but not emailed."
+            )
+
     @property
     def is_configured(self) -> bool:
-        """Check if email service is properly configured."""
-        return bool(self._sender_email and self._app_password)
+        """Check if email service is properly configured (or in mock mode)."""
+        return self._mock_mode or bool(self._sender_email and self._app_password)
 
     def _create_smtp_connection(self) -> smtplib.SMTP:
         """Create and authenticate SMTP connection."""
@@ -107,6 +127,13 @@ class EmailService:
             logger.error("Email service not configured. Cannot send email.")
             return False
 
+        # Mock mode - log but don't send
+        if self._mock_mode:
+            logger.info(
+                f"[MOCK EMAIL] To: {to_email}, Subject: {template.subject}"
+            )
+            return True
+
         try:
             # Create message
             msg = MIMEMultipart("alternative")
@@ -152,13 +179,29 @@ class EmailService:
         """
         return "".join(random.choices(string.digits, k=6))
 
+    @classmethod
+    def get_mock_code(cls, email: str) -> str | None:
+        """
+        Get the last login code sent to an email (mock mode only).
+
+        For testing purposes - retrieves the code that would have been
+        sent in mock mode.
+
+        Args:
+            email: Email address to look up
+
+        Returns:
+            The login code or None if not found
+        """
+        return cls._last_login_code.get(email.lower().strip())
+
     @staticmethod
     def generate_user_id_from_email(email: str) -> str:
         """
         Generate a deterministic UUID from email address.
 
-        Uses UUID v5 with DNS namespace for consistency.
-        Same email always produces same UUID.
+        Uses the centralized email_to_user_id() for consistency.
+        Same email always produces same UUID (bijection).
 
         Args:
             email: Email address
@@ -166,7 +209,8 @@ class EmailService:
         Returns:
             UUID string
         """
-        return str(uuid.uuid5(uuid.NAMESPACE_DNS, email.lower().strip()))
+        from rem.utils.user_id import email_to_user_id
+        return email_to_user_id(email)
 
     async def send_login_code(
         self,
@@ -254,6 +298,15 @@ class EmailService:
         if sent:
             result["success"] = True
             result["code_sent"] = True
+
+            # Store code for mock mode retrieval
+            if self._mock_mode:
+                EmailService._last_login_code[email] = code
+                logger.info(
+                    f"[MOCK MODE] Login code for {email}: {code} "
+                    f"(expires at {expires_at.isoformat()})"
+                )
+
             logger.info(
                 f"Login code sent to {email}, "
                 f"user_id={user_id}, expires at {expires_at.isoformat()}"
@@ -323,8 +376,17 @@ class EmailService:
             await user_repo.upsert(existing_user)
             return {"allowed": True, "error": None}
         else:
-            # New user - check if domain is trusted
-            if settings and hasattr(settings, 'email') and settings.email.trusted_domain_list:
+            # New user - first check if they're a subscriber (by email lookup)
+            from ...models.entities import Subscriber
+            subscriber_repo = Repository(Subscriber, db=db)
+            existing_subscriber = await subscriber_repo.find_one({"email": email})
+
+            if existing_subscriber:
+                # Subscriber exists - allow them to create account
+                # (approved field may not exist in older schemas, so just check existence)
+                logger.info(f"Subscriber {email} creating user account")
+            elif settings and hasattr(settings, 'email') and settings.email.trusted_domain_list:
+                # Not an approved subscriber - check if domain is trusted
                 if not settings.email.is_domain_trusted(email):
                     email_domain = email.split("@")[-1]
                     logger.warning(f"Untrusted domain attempted signup: {email_domain}")
@@ -341,7 +403,8 @@ class EmailService:
             new_user = User(
                 id=uuid.UUID(user_id),
                 tenant_id=tenant_id,
-                name=email.split("@")[0],  # Default name from email
+                user_id=user_id,  # UUID5 hash of email (same as id)
+                name=email,  # Full email as entity_key for LOOKUP
                 email=email,
                 role=user_role,
                 metadata=login_metadata,

rem/services/postgres/register_type.py

@@ -268,7 +268,7 @@ BEGIN
             graph_edges,
             updated_at
         ) VALUES (
-            NEW.{entity_key_field}::VARCHAR,
+            normalize_key(NEW.{entity_key_field}::VARCHAR),
             '{table_name}',
             NEW.id,
             NEW.tenant_id,

rem/services/postgres/repository.py

@@ -141,13 +141,13 @@ class Repository(Generic[T]):
         # Return single item or list to match input type
         return records_list[0] if is_single else records_list
 
-    async def get_by_id(self, record_id: str, tenant_id: str) -> T | None:
+    async def get_by_id(self, record_id: str, tenant_id: str | None = None) -> T | None:
         """
         Get a single record by ID.
 
         Args:
             record_id: Record identifier
-            tenant_id: Tenant identifier for multi-tenancy isolation
+            tenant_id: Optional tenant identifier (deprecated, not used for filtering)
 
         Returns:
             Model instance or None if not found
@@ -164,13 +164,14 @@ class Repository(Generic[T]):
         if not self.db.pool:
             raise RuntimeError("Failed to establish database connection")
 
+        # Note: tenant_id filtering removed - use user_id for access control instead
         query = f"""
             SELECT * FROM {self.table_name}
-            WHERE id = $1 AND tenant_id = $2 AND deleted_at IS NULL
+            WHERE id = $1 AND deleted_at IS NULL
         """
 
         async with self.db.pool.acquire() as conn:
-            row = await conn.fetchrow(query, record_id, tenant_id)
+            row = await conn.fetchrow(query, record_id)
 
         if not row:
             return None

rem/services/user_service.py

@@ -4,7 +4,8 @@ User Service - User account management.
 Handles user creation, profile updates, and session linking.
 """
 
-from datetime import datetime
+from rem.utils.date_utils import utc_now
+from rem.utils.user_id import email_to_user_id
 from typing import Optional
 
 from loguru import logger
@@ -51,28 +52,59 @@ class UserService:
                     updated = True
             
             if updated:
-                user.updated_at = datetime.utcnow()
+                user.updated_at = utc_now()
                 await self.repo.upsert(user)
             
             return user
         
         # Create new user
+        # id and user_id = UUID5 hash of email (deterministic bijection)
+        # name = email (entity_key for LOOKUP by email in KV store)
+        hashed_id = email_to_user_id(email)
         user = User(
+            id=hashed_id,  # Database id = hash of email
             tenant_id=tenant_id,
-            user_id=email, # Use email as user_id for now? Or UUID?
-            # The User model has 'user_id' field but also 'id' UUID.
-            # Usually user_id is the external ID or email.
-            name=name,
+            user_id=hashed_id,  # user_id = hash of email (same as id)
+            name=email,  # Email as entity_key for REM LOOKUP
             email=email,
             tier=UserTier.FREE,
-            created_at=datetime.utcnow(),
-            updated_at=datetime.utcnow(),
+            created_at=utc_now(),
+            updated_at=utc_now(),
             metadata={"avatar_url": avatar_url} if avatar_url else {},
         )
         await self.repo.upsert(user)
         logger.info(f"Created new user: {email}")
         return user
 
+    async def get_user_by_id(self, user_id: str) -> Optional[User]:
+        """
+        Get a user by their UUID.
+
+        Args:
+            user_id: The user's UUID
+
+        Returns:
+            User if found, None otherwise
+        """
+        try:
+            return await self.repo.get_by_id(user_id)
+        except Exception as e:
+            logger.warning(f"Could not find user by id {user_id}: {e}")
+            return None
+
+    async def get_user_by_email(self, email: str) -> Optional[User]:
+        """
+        Get a user by their email address.
+
+        Args:
+            email: The user's email
+
+        Returns:
+            User if found, None otherwise
+        """
+        users = await self.repo.find(filters={"email": email}, limit=1)
+        return users[0] if users else None
+
     async def link_anonymous_session(self, user: User, anon_id: str) -> None:
         """
         Link an anonymous session ID to a user account.
@@ -88,7 +120,7 @@ class UserService:
 
         # Add to list
         user.anonymous_ids.append(anon_id)
-        user.updated_at = datetime.utcnow()
+        user.updated_at = utc_now()
         
         # Save
         await self.repo.upsert(user)

rem/settings.py

@@ -77,6 +77,7 @@ class LLMSettings(BaseSettings):
         LLM__ANTHROPIC_API_KEY or ANTHROPIC_API_KEY - Anthropic API key
         LLM__EMBEDDING_PROVIDER or EMBEDDING_PROVIDER - Default embedding provider (openai)
         LLM__EMBEDDING_MODEL or EMBEDDING_MODEL - Default embedding model name
+        LLM__DEFAULT_STRUCTURED_OUTPUT - Default structured output mode (False = streaming text)
     """
 
     model_config = SettingsConfigDict(
@@ -138,6 +139,11 @@ class LLMSettings(BaseSettings):
         description="Default embedding model (provider-specific model name)",
     )
 
+    default_structured_output: bool = Field(
+        default=False,
+        description="Default structured output mode for agents. False = streaming text (easier), True = JSON schema validation",
+    )
+
     @field_validator("openai_api_key", mode="before")
     @classmethod
     def validate_openai_api_key(cls, v):
@@ -1028,7 +1034,7 @@ class ChatSettings(BaseSettings):
     - Prevents context window bloat while maintaining conversation continuity
 
     User Context (on-demand by default):
-    - Agent system prompt includes: "User ID: {user_id}. To load user profile: Use REM LOOKUP users/{user_id}"
+    - Agent system prompt includes: "User: {email}. To load user profile: Use REM LOOKUP \"{email}\""
     - Agent decides whether to load profile based on query
     - More efficient for queries that don't need personalization
 
@@ -1114,6 +1120,14 @@ class APISettings(BaseSettings):
         ),
     )
 
+    rate_limit_enabled: bool = Field(
+        default=True,
+        description=(
+            "Enable rate limiting for API endpoints. "
+            "Set to false to disable rate limiting entirely (useful for development)."
+        ),
+    )
+
 
 class ModelsSettings(BaseSettings):
     """

rem/sql/background_indexes.sql

@@ -21,6 +21,11 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_moments_vector_hnsw
 ON embeddings_moments
 USING hnsw (embedding vector_cosine_ops);
 
+-- HNSW vector index for embeddings_ontologies
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_ontologies_vector_hnsw
+ON embeddings_ontologies
+USING hnsw (embedding vector_cosine_ops);
+
 -- HNSW vector index for embeddings_ontology_configs
 CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_embeddings_ontology_configs_vector_hnsw
 ON embeddings_ontology_configs