remdb 0.3.141__py3-none-any.whl → 0.3.163__py3-none-any.whl

rem/services/session/compression.py

@@ -1,13 +1,49 @@
 """Session message compression and rehydration for efficient context loading.
 
-This module implements message compression to keep conversation history within
-context windows while preserving full content via REM LOOKUP.
-
-Design Pattern:
-- Long assistant messages (>400 chars) are stored as separate Message entities
-- In-memory conversation uses truncated versions with REM lookup hints
-- Full content retrieved on-demand via LOOKUP queries
-- Compression disabled when Postgres is disabled
+This module implements message storage and compression to keep conversation history
+within context windows while preserving full content via REM LOOKUP.
+
+Message Types and Storage Strategy
+===================================
+
+All messages are stored UNCOMPRESSED in the database for full audit/analysis.
+Compression happens only on RELOAD when reconstructing context for the LLM.
+
+Message Types:
+- `user`: User messages - stored and reloaded as-is
+- `tool`: Tool call messages (e.g., register_metadata) - stored and reloaded as-is
+         NEVER compressed - contains important structured metadata
+- `assistant`: Assistant text responses - stored uncompressed, but MAY BE
+              compressed on reload if long (>400 chars) with REM LOOKUP hints
+
+Example Session Flow:
+```
+Turn 1 (stored uncompressed):
+  - user: "I have a headache"
+  - tool: register_metadata({confidence: 0.3, collected_fields: {...}})
+  - assistant: "I'm sorry to hear that. How long has this been going on?"
+
+Turn 2 (stored uncompressed):
+  - user: "About 3 days, really bad"
+  - tool: register_metadata({confidence: 0.6, collected_fields: {...}})
+  - assistant: "Got it - 3 days. On a scale of 1-10..."
+
+On reload (for LLM context):
+  - user messages: returned as-is
+  - tool messages: returned as-is (never compressed)
+  - assistant messages: compressed if long, with REM LOOKUP hint for full retrieval
+```
+
+REM LOOKUP Pattern:
+- Long assistant messages get truncated with hint: "... [REM LOOKUP session-{id}-msg-{idx}] ..."
+- Agent can retrieve full content on-demand using the LOOKUP key
+- Keeps context window efficient while preserving data integrity
+
+Key Design Decisions:
+1. Store everything uncompressed - full audit trail in database
+2. Compress only on reload - optimize for LLM context window
+3. Never compress tool messages - structured metadata must stay intact
+4. REM LOOKUP enables on-demand retrieval of full assistant responses
 """
 
 from typing import Any
@@ -285,8 +321,23 @@ class SessionMessageStore:
                     msg_copy["_entity_key"] = entity_key
                     compressed_messages.append(msg_copy)
             else:
-                # Short assistant messages, user messages, and system messages stored as-is
+                # Short assistant messages, user messages, tool messages, and system messages stored as-is
                 # Store ALL messages in database for full audit trail
+                # Build metadata dict with standard fields
+                msg_metadata = {
+                    "message_index": idx,
+                    "timestamp": message.get("timestamp"),
+                }
+
+                # For tool messages, include tool call details in metadata
+                if message.get("role") == "tool":
+                    if message.get("tool_call_id"):
+                        msg_metadata["tool_call_id"] = message.get("tool_call_id")
+                    if message.get("tool_name"):
+                        msg_metadata["tool_name"] = message.get("tool_name")
+                    if message.get("tool_arguments"):
+                        msg_metadata["tool_arguments"] = message.get("tool_arguments")
+
                 msg = Message(
                     id=message.get("id"),  # Use pre-generated ID if provided
                     content=content,
@@ -296,10 +347,7 @@ class SessionMessageStore:
                     user_id=user_id or self.user_id,
                     trace_id=message.get("trace_id"),
                     span_id=message.get("span_id"),
-                    metadata={
-                        "message_index": idx,
-                        "timestamp": message.get("timestamp"),
-                    },
+                    metadata=msg_metadata,
                 )
                 await self.repo.upsert(msg)
                 compressed_messages.append(message.copy())
@@ -307,18 +355,24 @@ class SessionMessageStore:
         return compressed_messages
 
     async def load_session_messages(
-        self, session_id: str, user_id: str | None = None, decompress: bool = False
+        self, session_id: str, user_id: str | None = None, compress_on_load: bool = True
     ) -> list[dict[str, Any]]:
         """
-        Load session messages from database.
+        Load session messages from database, optionally compressing long assistant messages.
+
+        Compression on Load:
+        - Tool messages (role: "tool") are NEVER compressed - they contain structured metadata
+        - User messages are returned as-is
+        - Assistant messages MAY be compressed if long (>400 chars) with REM LOOKUP hints
 
         Args:
             session_id: Session identifier
             user_id: Optional user identifier for filtering
-            decompress: Whether to decompress messages (default: False)
+            compress_on_load: Whether to compress long assistant messages (default: True)
 
         Returns:
-            List of session messages in chronological order
+            List of session messages in chronological order, with long assistant
+            messages optionally compressed with REM LOOKUP hints
         """
         if not settings.postgres.enabled:
             logger.debug("Postgres disabled, returning empty message list")
@@ -335,49 +389,58 @@ class SessionMessageStore:
 
             # Convert Message entities to dict format
             message_dicts = []
-            for msg in messages:
+            for idx, msg in enumerate(messages):
+                role = msg.message_type or "assistant"
                 msg_dict = {
-                    "role": msg.message_type or "assistant",
+                    "role": role,
                     "content": msg.content,
                     "timestamp": msg.created_at.isoformat() if msg.created_at else None,
                 }
 
-                # Check if message was compressed
-                entity_key: str | None = msg.metadata.get("entity_key") if msg.metadata else None
-                if entity_key and len(msg.content) <= self.compressor.min_length_for_compression:
-                    # This is a compressed reference, mark it
-                    msg_dict["_compressed"] = True
-                    msg_dict["_entity_key"] = entity_key
-                    msg_dict["_original_length"] = msg.metadata.get("original_length", 0)
+                # For tool messages, reconstruct tool call metadata
+                if role == "tool" and msg.metadata:
+                    if msg.metadata.get("tool_call_id"):
+                        msg_dict["tool_call_id"] = msg.metadata["tool_call_id"]
+                    if msg.metadata.get("tool_name"):
+                        msg_dict["tool_name"] = msg.metadata["tool_name"]
+                    if msg.metadata.get("tool_arguments"):
+                        msg_dict["tool_arguments"] = msg.metadata["tool_arguments"]
+
+                # Compress long ASSISTANT messages on load (never tool messages)
+                if (
+                    compress_on_load
+                    and role == "assistant"
+                    and len(msg.content) > self.compressor.min_length_for_compression
+                ):
+                    # Generate entity key for REM LOOKUP
+                    entity_key = truncate_key(f"session-{session_id}-msg-{idx}")
+                    msg_dict = self.compressor.compress_message(msg_dict, entity_key)
 
                 message_dicts.append(msg_dict)
 
-            # Decompress if requested
-            if decompress:
-                decompressed_messages = []
-                for message in message_dicts:
-                    if self.compressor.is_compressed(message):
-                        entity_key = self.compressor.get_entity_key(message)
-                        if entity_key:
-                            full_content = await self.retrieve_message(entity_key)
-                            if full_content:
-                                decompressed_messages.append(
-                                    self.compressor.decompress_message(
-                                        message, full_content
-                                    )
-                                )
-                            else:
-                                # Fallback to compressed version if retrieval fails
-                                decompressed_messages.append(message)
-                        else:
-                            decompressed_messages.append(message)
-                    else:
-                        decompressed_messages.append(message)
-
-                return decompressed_messages
-
+            logger.debug(
+                f"Loaded {len(message_dicts)} messages for session {session_id} "
+                f"(compress_on_load={compress_on_load})"
+            )
             return message_dicts
 
         except Exception as e:
             logger.error(f"Failed to load session messages: {e}")
             return []
+
+    async def retrieve_full_message(self, session_id: str, message_index: int) -> str | None:
+        """
+        Retrieve full message content by session and message index (for REM LOOKUP).
+
+        This is used when an agent needs to recover full content from a compressed
+        message that has a REM LOOKUP hint.
+
+        Args:
+            session_id: Session identifier
+            message_index: Index of message in session (from REM LOOKUP key)
+
+        Returns:
+            Full message content or None if not found
+        """
+        entity_key = truncate_key(f"session-{session_id}-msg-{message_index}")
+        return await self.retrieve_message(entity_key)

rem/services/session/reload.py

@@ -6,8 +6,14 @@ allowing conversations to be resumed across multiple API calls.
 Design Pattern:
 - Session identified by session_id from X-Session-Id header
 - All messages for session loaded in chronological order
-- Optional decompression of long assistant messages via REM LOOKUP
+- Long assistant messages compressed on load with REM LOOKUP hints
+- Tool messages (register_metadata, etc.) are NEVER compressed
 - Gracefully handles missing database (returns empty history)
+
+Message Types on Reload:
+- user: Returned as-is
+- tool: Returned as-is with metadata (tool_call_id, tool_name, tool_arguments)
+- assistant: Compressed on load if long (>400 chars), with REM LOOKUP for recovery
 """
 
 from loguru import logger
@@ -19,7 +25,7 @@ from rem.settings import settings
 async def reload_session(
     session_id: str,
     user_id: str,
-    decompress_messages: bool = False,
+    compress_on_load: bool = True,
 ) -> list[dict]:
     """
     Reload all messages for a session from the database.
@@ -27,7 +33,8 @@ async def reload_session(
     Args:
         session_id: Session/conversation identifier
         user_id: User identifier for data isolation
-        decompress_messages: Whether to decompress long messages via REM LOOKUP
+        compress_on_load: Whether to compress long assistant messages (default: True)
+                         Tool messages are NEVER compressed.
 
     Returns:
         List of message dicts in chronological order (oldest first)
@@ -41,7 +48,7 @@ async def reload_session(
         history = await reload_session(
             session_id=context.session_id,
             user_id=context.user_id,
-            decompress_messages=False,  # Use compressed versions for efficiency
+            compress_on_load=True,  # Compress long assistant messages
         )
 
         # Combine with new user message
@@ -60,14 +67,14 @@ async def reload_session(
         # Create message store for this session
         store = SessionMessageStore(user_id=user_id)
 
-        # Load messages (optionally decompressed)
+        # Load messages (assistant messages compressed on load, tool messages never compressed)
         messages = await store.load_session_messages(
-            session_id=session_id, user_id=user_id, decompress=decompress_messages
+            session_id=session_id, user_id=user_id, compress_on_load=compress_on_load
         )
 
         logger.debug(
             f"Reloaded {len(messages)} messages for session {session_id} "
-            f"(decompressed={decompress_messages})"
+            f"(compress_on_load={compress_on_load})"
         )
 
         return messages

rem/services/user_service.py

@@ -73,6 +73,35 @@ class UserService:
         logger.info(f"Created new user: {email}")
         return user
 
+    async def get_user_by_id(self, user_id: str) -> Optional[User]:
+        """
+        Get a user by their UUID.
+
+        Args:
+            user_id: The user's UUID
+
+        Returns:
+            User if found, None otherwise
+        """
+        try:
+            return await self.repo.get_by_id(user_id)
+        except Exception as e:
+            logger.warning(f"Could not find user by id {user_id}: {e}")
+            return None
+
+    async def get_user_by_email(self, email: str) -> Optional[User]:
+        """
+        Get a user by their email address.
+
+        Args:
+            email: The user's email
+
+        Returns:
+            User if found, None otherwise
+        """
+        users = await self.repo.find(filters={"email": email}, limit=1)
+        return users[0] if users else None
+
     async def link_anonymous_session(self, user: User, anon_id: str) -> None:
         """
         Link an anonymous session ID to a user account.

rem/settings.py

@@ -21,8 +21,8 @@ Example .env file:
     LLM__OPENAI_API_KEY=sk-...
     LLM__ANTHROPIC_API_KEY=sk-ant-...
 
-    # Database (port 5050 for Docker Compose)
-    POSTGRES__CONNECTION_STRING=postgresql://rem:rem@localhost:5050/rem
+    # Database (port 5051 for Docker Compose prebuilt, 5050 for local dev)
+    POSTGRES__CONNECTION_STRING=postgresql://rem:rem@localhost:5051/rem
     POSTGRES__POOL_MIN_SIZE=5
     POSTGRES__POOL_MAX_SIZE=20
     POSTGRES__STATEMENT_TIMEOUT=30000
@@ -464,10 +464,11 @@ class PostgresSettings(BaseSettings):
     )
 
     connection_string: str = Field(
-        default="postgresql://rem:rem@localhost:5050/rem",
-        description="PostgreSQL connection string (default uses Docker Compose port 5050)",
+        default="postgresql://rem:rem@localhost:5051/rem",
+        description="PostgreSQL connection string (default uses Docker Compose prebuilt port 5051)",
     )
 
+
     pool_size: int = Field(
         default=10,
         description="Connection pool size (deprecated, use pool_min_size/pool_max_size)",
@@ -1060,6 +1061,8 @@ class APISettings(BaseSettings):
         API__RELOAD - Enable auto-reload for development
         API__WORKERS - Number of worker processes (production)
         API__LOG_LEVEL - Logging level (debug, info, warning, error)
+        API__API_KEY_ENABLED - Enable X-API-Key header authentication
+        API__API_KEY - API key for X-API-Key authentication
     """
 
     model_config = SettingsConfigDict(
@@ -1094,6 +1097,31 @@ class APISettings(BaseSettings):
         description="Logging level (debug, info, warning, error, critical)",
     )
 
+    api_key_enabled: bool = Field(
+        default=False,
+        description=(
+            "Enable X-API-Key header authentication for API endpoints. "
+            "When enabled, requests must include X-API-Key header with valid key. "
+            "This provides simple API key auth independent of OAuth."
+        ),
+    )
+
+    api_key: str | None = Field(
+        default=None,
+        description=(
+            "API key for X-API-Key authentication. Required when api_key_enabled=true. "
+            "Generate with: python -c \"import secrets; print(secrets.token_urlsafe(32))\""
+        ),
+    )
+
+    rate_limit_enabled: bool = Field(
+        default=True,
+        description=(
+            "Enable rate limiting for API endpoints. "
+            "Set to false to disable rate limiting entirely (useful for development)."
+        ),
+    )
+
 
 class ModelsSettings(BaseSettings):
     """
@@ -1451,6 +1479,172 @@ class DBListenerSettings(BaseSettings):
         return [c.strip() for c in self.channels.split(",") if c.strip()]
 
 
+class EmailSettings(BaseSettings):
+    """
+    Email service settings for SMTP.
+
+    Supports passwordless login via email codes and transactional emails.
+    Uses Gmail SMTP with App Passwords by default.
+
+    Generate app password at: https://myaccount.google.com/apppasswords
+
+    Environment variables:
+        EMAIL__ENABLED - Enable email service (default: false)
+        EMAIL__SMTP_HOST - SMTP server host (default: smtp.gmail.com)
+        EMAIL__SMTP_PORT - SMTP server port (default: 587 for TLS)
+        EMAIL__SENDER_EMAIL - Sender email address
+        EMAIL__SENDER_NAME - Sender display name
+        EMAIL__APP_PASSWORD - Gmail app password (from secrets)
+        EMAIL__USE_TLS - Use TLS encryption (default: true)
+        EMAIL__LOGIN_CODE_EXPIRY_MINUTES - Login code expiry (default: 10)
+
+    Branding environment variables (for email templates):
+        EMAIL__APP_NAME - Application name in emails (default: REM)
+        EMAIL__LOGO_URL - Logo URL for email templates (40x40 recommended)
+        EMAIL__TAGLINE - Tagline shown in email footer
+        EMAIL__WEBSITE_URL - Main website URL for email links
+        EMAIL__PRIVACY_URL - Privacy policy URL for email footer
+        EMAIL__TERMS_URL - Terms of service URL for email footer
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="EMAIL__",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    enabled: bool = Field(
+        default=False,
+        description="Enable email service (requires app_password to be set)",
+    )
+
+    smtp_host: str = Field(
+        default="smtp.gmail.com",
+        description="SMTP server host",
+    )
+
+    smtp_port: int = Field(
+        default=587,
+        description="SMTP server port (587 for TLS, 465 for SSL)",
+    )
+
+    sender_email: str = Field(
+        default="",
+        description="Sender email address",
+    )
+
+    sender_name: str = Field(
+        default="REM",
+        description="Sender display name",
+    )
+
+    # Branding settings for email templates
+    app_name: str = Field(
+        default="REM",
+        description="Application name shown in email templates",
+    )
+
+    logo_url: str | None = Field(
+        default=None,
+        description="Logo URL for email templates (40x40 recommended)",
+    )
+
+    tagline: str = Field(
+        default="Your AI-powered platform",
+        description="Tagline shown in email footer",
+    )
+
+    website_url: str = Field(
+        default="https://rem.ai",
+        description="Main website URL for email links",
+    )
+
+    privacy_url: str = Field(
+        default="https://rem.ai/privacy",
+        description="Privacy policy URL for email footer",
+    )
+
+    terms_url: str = Field(
+        default="https://rem.ai/terms",
+        description="Terms of service URL for email footer",
+    )
+
+    app_password: str | None = Field(
+        default=None,
+        description="Gmail app password for SMTP authentication",
+    )
+
+    use_tls: bool = Field(
+        default=True,
+        description="Use TLS encryption for SMTP",
+    )
+
+    login_code_expiry_minutes: int = Field(
+        default=10,
+        description="Login code expiry in minutes",
+    )
+
+    trusted_email_domains: str = Field(
+        default="",
+        description=(
+            "Comma-separated list of trusted email domains for new user registration. "
+            "Existing users can always login regardless of domain. "
+            "New users must have an email from a trusted domain. "
+            "Empty string means all domains are allowed. "
+            "Example: 'siggymd.ai,example.com'"
+        ),
+    )
+
+    @property
+    def trusted_domain_list(self) -> list[str]:
+        """Get trusted domains as a list, filtering empty strings."""
+        if not self.trusted_email_domains:
+            return []
+        return [d.strip().lower() for d in self.trusted_email_domains.split(",") if d.strip()]
+
+    def is_domain_trusted(self, email: str) -> bool:
+        """Check if an email's domain is in the trusted list.
+
+        Args:
+            email: Email address to check
+
+        Returns:
+            True if domain is trusted (or if no trusted domains configured)
+        """
+        domains = self.trusted_domain_list
+        if not domains:
+            # No restrictions configured
+            return True
+
+        email_domain = email.lower().split("@")[-1].strip()
+        return email_domain in domains
+
+    @property
+    def is_configured(self) -> bool:
+        """Check if email service is properly configured."""
+        return bool(self.sender_email and self.app_password)
+
+    @property
+    def template_kwargs(self) -> dict:
+        """
+        Get branding kwargs for email templates.
+
+        Returns a dict that can be passed to template functions:
+            login_code_template(..., **settings.email.template_kwargs)
+        """
+        kwargs = {
+            "app_name": self.app_name,
+            "tagline": self.tagline,
+            "website_url": self.website_url,
+            "privacy_url": self.privacy_url,
+            "terms_url": self.terms_url,
+        }
+        if self.logo_url:
+            kwargs["logo_url"] = self.logo_url
+        return kwargs
+
+
 class TestSettings(BaseSettings):
     """
     Test environment settings.
@@ -1565,6 +1759,7 @@ class Settings(BaseSettings):
     chunking: ChunkingSettings = Field(default_factory=ChunkingSettings)
     content: ContentSettings = Field(default_factory=ContentSettings)
     schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
+    email: EmailSettings = Field(default_factory=EmailSettings)
     test: TestSettings = Field(default_factory=TestSettings)