remdb 0.3.202__py3-none-any.whl → 0.3.245__py3-none-any.whl

rem/cli/commands/session.py

@@ -331,6 +331,123 @@ async def _show_async(
             raise
 
 
+@session.command("clone")
+@click.argument("session_id")
+@click.option("--to-turn", "-t", type=int, help="Clone up to turn N (counting user messages only)")
+@click.option("--name", "-n", help="Name/description for the cloned session")
+def clone(session_id: str, to_turn: int | None, name: str | None):
+    """
+    Clone a session for exploring alternate conversation paths.
+
+    SESSION_ID: The session ID to clone.
+
+    Examples:
+
+        # Clone entire session
+        rem session clone 810f1f2d-d5a1-4c02-83b6-67040b47f7c0
+
+        # Clone up to turn 3 (first 3 user messages and their responses)
+        rem session clone 810f1f2d-d5a1-4c02-83b6-67040b47f7c0 --to-turn 3
+
+        # Clone with a descriptive name
+        rem session clone 810f1f2d-d5a1-4c02-83b6-67040b47f7c0 -n "Alternate anxiety path"
+    """
+    asyncio.run(_clone_async(session_id, to_turn, name))
+
+
+async def _clone_async(
+    session_id: str,
+    to_turn: int | None,
+    name: str | None,
+):
+    """Async implementation of clone command."""
+    from uuid import uuid4
+    from ...models.entities.session import Session, SessionMode
+
+    pg = get_postgres_service()
+    if not pg:
+        logger.error("PostgreSQL not available")
+        return
+
+    await pg.connect()
+
+    try:
+        # Load original session messages
+        message_repo = Repository(Message, "messages", db=pg)
+        messages = await message_repo.find(
+            filters={"session_id": session_id},
+            order_by="created_at ASC",
+            limit=1000,
+        )
+
+        if not messages:
+            logger.error(f"No messages found for session {session_id}")
+            return
+
+        # If --to-turn specified, filter messages up to that turn (user messages)
+        if to_turn is not None:
+            user_count = 0
+            cutoff_idx = len(messages)
+            for idx, msg in enumerate(messages):
+                if msg.message_type == "user":
+                    user_count += 1
+                    if user_count > to_turn:
+                        cutoff_idx = idx
+                        break
+            messages = messages[:cutoff_idx]
+            logger.info(f"Cloning {len(messages)} messages (up to turn {to_turn})")
+        else:
+            logger.info(f"Cloning all {len(messages)} messages")
+
+        # Generate new session ID
+        new_session_id = str(uuid4())
+
+        # Get user_id and tenant_id from first message
+        first_msg = messages[0]
+        user_id = first_msg.user_id
+        tenant_id = first_msg.tenant_id or "default"
+
+        # Create Session record with CLONE mode and lineage
+        session_repo = Repository(Session, "sessions", db=pg)
+        new_session = Session(
+            id=uuid4(),
+            name=name or f"Clone of {session_id[:8]}",
+            mode=SessionMode.CLONE,
+            original_trace_id=session_id,
+            description=f"Cloned from session {session_id}" + (f" at turn {to_turn}" if to_turn else ""),
+            user_id=user_id,
+            tenant_id=tenant_id,
+            message_count=len(messages),
+        )
+        await session_repo.upsert(new_session)
+        logger.info(f"Created session record: {new_session.id}")
+
+        # Copy messages with new session_id
+        for msg in messages:
+            new_msg = Message(
+                id=uuid4(),
+                user_id=msg.user_id,
+                tenant_id=msg.tenant_id,
+                session_id=str(new_session.id),
+                content=msg.content,
+                message_type=msg.message_type,
+                metadata=msg.metadata,
+            )
+            await message_repo.upsert(new_msg)
+
+        click.echo(f"\n✅ Cloned session successfully!")
+        click.echo(f"   Original: {session_id}")
+        click.echo(f"   New:      {new_session.id}")
+        click.echo(f"   Messages: {len(messages)}")
+        if to_turn:
+            click.echo(f"   Turns:    {to_turn}")
+        click.echo(f"\nContinue this session with:")
+        click.echo(f"   rem ask <agent> \"your message\" --session-id {new_session.id}")
+
+    finally:
+        await pg.disconnect()
+
+
 def register_command(cli_group):
     """Register the session command group."""
     cli_group.add_command(session)

rem/cli/main.py

@@ -97,6 +97,7 @@ from .commands.mcp import register_command as register_mcp_command
 from .commands.scaffold import scaffold as scaffold_command
 from .commands.cluster import register_commands as register_cluster_commands
 from .commands.session import register_command as register_session_command
+from .commands.query import register_command as register_query_command
 
 register_schema_commands(schema)
 register_db_commands(db)
@@ -107,6 +108,7 @@ register_ask_command(cli)
 register_configure_command(cli)
 register_serve_command(cli)
 register_mcp_command(cli)
+register_query_command(cli)
 cli.add_command(experiments_group)
 cli.add_command(scaffold_command)
 register_session_command(cli)

rem/models/entities/session.py

@@ -21,6 +21,7 @@ class SessionMode(str, Enum):
 
     NORMAL = "normal"
     EVALUATION = "evaluation"
+    CLONE = "clone"
 
 
 class Session(CoreModel):

rem/schemas/agents/rem.yaml

@@ -124,7 +124,7 @@ json_schema_extra:
 
   # Explicit resource declarations for reference data
   resources:
-    - uri: rem://schemas
+    - uri: rem://agents
       name: Agent Schemas List
       description: List all available agent schemas in the system
     - uri: rem://status

rem/services/postgres/repository.py

@@ -33,15 +33,15 @@ if TYPE_CHECKING:
 
 def get_postgres_service() -> "PostgresService | None":
     """
-    Get PostgresService instance with connection string from settings.
+    Get PostgresService singleton from parent module.
 
-    Returns None if Postgres is disabled.
+    Uses late import to avoid circular import issues.
+    Previously had a separate _postgres_instance here which caused
+    "pool not connected" errors due to duplicate connection pools.
     """
-    if not settings.postgres.enabled:
-        return None
-    
-    from .service import PostgresService
-    return PostgresService()
+    # Late import to avoid circular import (repository.py imported by __init__.py)
+    from rem.services.postgres import get_postgres_service as _get_singleton
+    return _get_singleton()
 
 T = TypeVar("T", bound=BaseModel)
 

rem/services/rem/service.py

@@ -478,6 +478,53 @@ class RemService:
         parser = RemQueryParser()
         return parser.parse(query_string)
 
+    async def execute_query_string(
+        self, query_string: str, user_id: str | None = None
+    ) -> dict[str, Any]:
+        """
+        Execute a REM dialect query string directly.
+
+        This is the unified entry point for executing REM queries from both
+        the CLI and API. It handles parsing the query string, creating the
+        RemQuery model, and executing it.
+
+        Args:
+            query_string: REM dialect query (e.g., 'LOOKUP "Sarah Chen"',
+                         'SEARCH resources "API design"', 'SELECT * FROM users')
+            user_id: Optional user ID for query isolation
+
+        Returns:
+            Dict with query results and metadata:
+            - query_type: The type of query executed
+            - results: List of result rows
+            - count: Number of results
+            - Additional fields depending on query type
+
+        Raises:
+            ValueError: If the query string is invalid
+            QueryExecutionError: If query execution fails
+
+        Example:
+            >>> result = await rem_service.execute_query_string(
+            ...     'LOOKUP "Sarah Chen"',
+            ...     user_id="user-123"
+            ... )
+            >>> print(result["count"])
+            1
+        """
+        # Parse the query string into type and parameters
+        query_type, parameters = self._parse_query_string(query_string)
+
+        # Create and validate the RemQuery model
+        rem_query = RemQuery.model_validate({
+            "query_type": query_type,
+            "parameters": parameters,
+            "user_id": user_id,
+        })
+
+        # Execute and return results
+        return await self.execute_query(rem_query)
+
     async def ask_rem(
         self, natural_query: str, tenant_id: str, llm_model: str | None = None, plan_mode: bool = False
     ) -> dict[str, Any]:

rem/services/session/__init__.py

@@ -1,12 +1,13 @@
 """Session management services for conversation persistence and compression."""
 
 from .compression import MessageCompressor, SessionMessageStore
-from .pydantic_messages import session_to_pydantic_messages
+from .pydantic_messages import audit_session_history, session_to_pydantic_messages
 from .reload import reload_session
 
 __all__ = [
     "MessageCompressor",
     "SessionMessageStore",
+    "audit_session_history",
     "reload_session",
     "session_to_pydantic_messages",
 ]

rem/services/session/compression.py

@@ -96,7 +96,7 @@ class MessageCompressor:
         Returns:
             Compressed message dict
         """
-        content = message.get("content", "")
+        content = message.get("content") or ""
 
         # Don't compress short messages or system messages
         if (
@@ -188,21 +188,19 @@ class SessionMessageStore:
         Ensure session exists, creating it if necessary.
 
         Args:
-            session_id: Session identifier (maps to Session.name)
+            session_id: Session UUID from X-Session-Id header
             user_id: Optional user identifier
         """
         try:
-            # Check if session already exists by name
-            existing = await self._session_repo.find(
-                filters={"name": session_id},
-                limit=1,
-            )
+            # Check if session already exists by UUID
+            existing = await self._session_repo.get_by_id(session_id)
             if existing:
                 return  # Session already exists
 
-            # Create new session
+            # Create new session with the provided UUID as id
             session = Session(
-                name=session_id,
+                id=session_id,  # Use the provided UUID as session id
+                name=session_id,  # Default name to UUID, can be updated later
                 user_id=user_id or self.user_id,
                 tenant_id=self.user_id,  # tenant_id set to user_id for scoping
             )
@@ -244,7 +242,7 @@ class SessionMessageStore:
         # Use pre-generated id from message dict if available (for frontend feedback)
         msg = Message(
             id=message.get("id"),  # Use pre-generated ID if provided
-            content=message.get("content", ""),
+            content=message.get("content") or "",
             message_type=message.get("role", "assistant"),
             session_id=session_id,
             tenant_id=self.user_id,  # Set tenant_id to user_id (application scoped to user)
@@ -321,7 +319,7 @@ class SessionMessageStore:
         Ensures session exists before storing messages.
 
         Args:
-            session_id: Session identifier (maps to Session.name)
+            session_id: Session UUID
             messages: List of messages to store
             user_id: Optional user identifier
             compress: Whether to compress messages (default: True)
@@ -339,7 +337,7 @@ class SessionMessageStore:
         compressed_messages = []
 
         for idx, message in enumerate(messages):
-            content = message.get("content", "")
+            content = message.get("content") or ""
 
             # Only store and compress long assistant responses
             if (
@@ -370,6 +368,8 @@ class SessionMessageStore:
                 }
 
                 # For tool messages, include tool call details in metadata
+                # Note: tool_arguments is stored only when provided (parent tool calls)
+                # For child tool calls (e.g., register_metadata), args are in content as JSON
                 if message.get("role") == "tool":
                     if message.get("tool_call_id"):
                         msg_metadata["tool_call_id"] = message.get("tool_call_id")
@@ -438,6 +438,8 @@ class SessionMessageStore:
                 }
 
                 # For tool messages, reconstruct tool call metadata
+                # Note: tool_arguments may be in metadata (parent calls) or parsed from
+                # content (child calls like register_metadata) by pydantic_messages.py
                 if role == "tool" and msg.metadata:
                     if msg.metadata.get("tool_call_id"):
                         msg_dict["tool_call_id"] = msg.metadata["tool_call_id"]

rem/services/session/pydantic_messages.py

@@ -5,12 +5,16 @@ storage format into pydantic-ai's native ModelRequest/ModelResponse types.
 
 Key insight: When we store tool results, we only store the result (ToolReturnPart).
 But LLM APIs require matching ToolCallPart for each ToolReturnPart. So we synthesize
-the ToolCallPart from stored metadata (tool_name, tool_call_id, tool_arguments).
+the ToolCallPart from stored metadata (tool_name, tool_call_id) and arguments.
+
+Tool arguments can come from two places:
+- Parent tool calls (ask_agent): tool_arguments stored in metadata (content = result)
+- Child tool calls (register_metadata): arguments parsed from content (content = args as JSON)
 
 Storage format (our simplified format):
     {"role": "user", "content": "..."}
     {"role": "assistant", "content": "..."}
-    {"role": "tool", "content": "{...}", "tool_name": "...", "tool_call_id": "...", "tool_arguments": {...}}
+    {"role": "tool", "content": "{...}", "tool_name": "...", "tool_call_id": "...", "tool_arguments": {...}}  # optional
 
 Pydantic-ai format (what the LLM expects):
     ModelRequest(parts=[UserPromptPart(content="...")])
@@ -31,6 +35,7 @@ Example usage:
 """
 
 import json
+import re
 from typing import Any
 
 from loguru import logger
@@ -46,6 +51,15 @@ from pydantic_ai.messages import (
 )
 
 
+def _sanitize_tool_name(tool_name: str) -> str:
+    """Sanitize tool name for OpenAI API compatibility.
+
+    OpenAI requires tool names to match pattern: ^[a-zA-Z0-9_-]+$
+    This replaces invalid characters (like colons) with underscores.
+    """
+    return re.sub(r'[^a-zA-Z0-9_-]', '_', tool_name)
+
+
 def session_to_pydantic_messages(
     session_history: list[dict[str, Any]],
     system_prompt: str | None = None,
@@ -92,7 +106,7 @@ def session_to_pydantic_messages(
     while i < len(session_history):
         msg = session_history[i]
         role = msg.get("role", "")
-        content = msg.get("content", "")
+        content = msg.get("content") or ""
 
         if role == "user":
             # User messages become ModelRequest with UserPromptPart
@@ -110,8 +124,15 @@ def session_to_pydantic_messages(
                 tool_msg = session_history[j]
                 tool_name = tool_msg.get("tool_name", "unknown_tool")
                 tool_call_id = tool_msg.get("tool_call_id", f"call_{j}")
-                tool_arguments = tool_msg.get("tool_arguments", {})
-                tool_content = tool_msg.get("content", "{}")
+                tool_content = tool_msg.get("content") or "{}"
+
+                # tool_arguments: prefer explicit field, fallback to parsing content
+                tool_arguments = tool_msg.get("tool_arguments")
+                if tool_arguments is None and isinstance(tool_content, str) and tool_content:
+                    try:
+                        tool_arguments = json.loads(tool_content)
+                    except json.JSONDecodeError:
+                        tool_arguments = {}
 
                 # Parse tool content if it's a JSON string
                 if isinstance(tool_content, str):
@@ -122,16 +143,19 @@ def session_to_pydantic_messages(
                 else:
                     tool_result = tool_content
 
+                # Sanitize tool name for OpenAI API compatibility
+                safe_tool_name = _sanitize_tool_name(tool_name)
+
                 # Synthesize ToolCallPart (what the model "called")
                 tool_calls.append(ToolCallPart(
-                    tool_name=tool_name,
+                    tool_name=safe_tool_name,
                     args=tool_arguments if tool_arguments else {},
                     tool_call_id=tool_call_id,
                 ))
 
                 # Create ToolReturnPart (the actual result)
                 tool_returns.append(ToolReturnPart(
-                    tool_name=tool_name,
+                    tool_name=safe_tool_name,
                     content=tool_result,
                     tool_call_id=tool_call_id,
                 ))
@@ -166,8 +190,15 @@ def session_to_pydantic_messages(
             # Orphan tool message (no preceding assistant) - synthesize both parts
             tool_name = msg.get("tool_name", "unknown_tool")
             tool_call_id = msg.get("tool_call_id", f"call_{i}")
-            tool_arguments = msg.get("tool_arguments", {})
-            tool_content = msg.get("content", "{}")
+            tool_content = msg.get("content") or "{}"
+
+            # tool_arguments: prefer explicit field, fallback to parsing content
+            tool_arguments = msg.get("tool_arguments")
+            if tool_arguments is None and isinstance(tool_content, str) and tool_content:
+                try:
+                    tool_arguments = json.loads(tool_content)
+                except json.JSONDecodeError:
+                    tool_arguments = {}
 
             # Parse tool content
             if isinstance(tool_content, str):
@@ -178,10 +209,13 @@ def session_to_pydantic_messages(
             else:
                 tool_result = tool_content
 
+            # Sanitize tool name for OpenAI API compatibility
+            safe_tool_name = _sanitize_tool_name(tool_name)
+
             # Synthesize the tool call (ModelResponse with ToolCallPart)
             messages.append(ModelResponse(
                 parts=[ToolCallPart(
-                    tool_name=tool_name,
+                    tool_name=safe_tool_name,
                     args=tool_arguments if tool_arguments else {},
                     tool_call_id=tool_call_id,
                 )],
@@ -191,7 +225,7 @@ def session_to_pydantic_messages(
             # Add the tool return (ModelRequest with ToolReturnPart)
             messages.append(ModelRequest(
                 parts=[ToolReturnPart(
-                    tool_name=tool_name,
+                    tool_name=safe_tool_name,
                     content=tool_result,
                     tool_call_id=tool_call_id,
                 )]
@@ -208,3 +242,69 @@ def session_to_pydantic_messages(
 
     logger.debug(f"Converted {len(session_history)} stored messages to {len(messages)} pydantic-ai messages")
     return messages
+
+
+def audit_session_history(
+    session_id: str,
+    agent_name: str,
+    prompt: str,
+    raw_session_history: list[dict[str, Any]],
+    pydantic_messages_count: int,
+) -> None:
+    """
+    Dump session history to a YAML file for debugging.
+
+    Only runs when DEBUG__AUDIT_SESSION=true. Writes to DEBUG__AUDIT_DIR (default /tmp).
+    Appends to the same file for a session, so all agent invocations are in one place.
+
+    Args:
+        session_id: The session identifier
+        agent_name: Name of the agent being invoked
+        prompt: The prompt being sent to the agent
+        raw_session_history: The raw session messages from the database
+        pydantic_messages_count: Count of converted pydantic-ai messages
+    """
+    from ...settings import settings
+
+    if not settings.debug.audit_session:
+        return
+
+    try:
+        import yaml
+        from pathlib import Path
+        from ...utils.date_utils import utc_now, to_iso
+
+        audit_dir = Path(settings.debug.audit_dir)
+        audit_dir.mkdir(parents=True, exist_ok=True)
+        audit_file = audit_dir / f"{session_id}.yaml"
+
+        # Create entry for this agent invocation
+        entry = {
+            "timestamp": to_iso(utc_now()),
+            "agent_name": agent_name,
+            "prompt": prompt,
+            "raw_history_count": len(raw_session_history),
+            "pydantic_messages_count": pydantic_messages_count,
+            "raw_session_history": raw_session_history,
+        }
+
+        # Load existing data or create new
+        existing_data: dict[str, Any] = {"session_id": session_id, "invocations": []}
+        if audit_file.exists():
+            with open(audit_file) as f:
+                loaded = yaml.safe_load(f)
+                if loaded:
+                    # Ensure session_id is always present (backfill if missing)
+                    existing_data = {
+                        "session_id": loaded.get("session_id", session_id),
+                        "invocations": loaded.get("invocations", []),
+                    }
+
+        # Append this invocation
+        existing_data["invocations"].append(entry)
+
+        with open(audit_file, "w") as f:
+            yaml.dump(existing_data, f, default_flow_style=False, allow_unicode=True)
+        logger.info(f"DEBUG: Session audit updated: {audit_file}")
+    except Exception as e:
+        logger.warning(f"DEBUG: Failed to dump session audit: {e}")

rem/services/session/reload.py

@@ -12,7 +12,8 @@ Design Pattern:
 
 Message Types on Reload:
 - user: Returned as-is
-- tool: Returned as-is with metadata (tool_call_id, tool_name, tool_arguments)
+- tool: Returned with metadata (tool_call_id, tool_name). tool_arguments may be in
+  metadata (parent calls) or parsed from content (child calls) by pydantic_messages.py
 - assistant: Compressed on load if long (>400 chars), with REM LOOKUP for recovery
 """
 

rem/settings.py

@@ -424,6 +424,49 @@ class AuthSettings(BaseSettings):
     google: GoogleOAuthSettings = Field(default_factory=GoogleOAuthSettings)
     microsoft: MicrosoftOAuthSettings = Field(default_factory=MicrosoftOAuthSettings)
 
+    # Pre-approved login codes (bypass email verification)
+    # Format: comma-separated codes with prefix A=admin, B=normal user
+    # Example: "A12345,A67890,B11111,B22222"
+    preapproved_codes: str = Field(
+        default="",
+        description=(
+            "Comma-separated list of pre-approved login codes. "
+            "Prefix A = admin user, B = normal user. "
+            "Example: 'A12345,A67890,B11111'. "
+            "Users can login with these codes without email verification."
+        ),
+    )
+
+    def check_preapproved_code(self, code: str) -> dict | None:
+        """
+        Check if a code is in the pre-approved list.
+
+        Args:
+            code: The code to check (including prefix)
+
+        Returns:
+            Dict with 'role' key if valid, None if not found.
+            - A prefix -> role='admin'
+            - B prefix -> role='user'
+        """
+        if not self.preapproved_codes:
+            return None
+
+        codes = [c.strip().upper() for c in self.preapproved_codes.split(",") if c.strip()]
+        code_upper = code.strip().upper()
+
+        if code_upper not in codes:
+            return None
+
+        # Parse prefix to determine role
+        if code_upper.startswith("A"):
+            return {"role": "admin", "code": code_upper}
+        elif code_upper.startswith("B"):
+            return {"role": "user", "code": code_upper}
+        else:
+            # Unknown prefix, treat as user
+            return {"role": "user", "code": code_upper}
+
     @field_validator("session_secret", mode="before")
     @classmethod
     def generate_dev_secret(cls, v: str | None, info: ValidationInfo) -> str:
@@ -1651,6 +1694,33 @@ class EmailSettings(BaseSettings):
         return kwargs
 
 
+class DebugSettings(BaseSettings):
+    """
+    Debug settings for development and troubleshooting.
+
+    Environment variables:
+        DEBUG__AUDIT_SESSION - Dump session history to /tmp/{session_id}.yaml
+        DEBUG__AUDIT_DIR - Directory for session audit files (default: /tmp)
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="DEBUG__",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    audit_session: bool = Field(
+        default=False,
+        description="When true, dump full session history to audit files for debugging",
+    )
+
+    audit_dir: str = Field(
+        default="/tmp",
+        description="Directory for session audit files",
+    )
+
+
 class TestSettings(BaseSettings):
     """
     Test environment settings.
@@ -1767,6 +1837,7 @@ class Settings(BaseSettings):
     schema_search: SchemaSettings = Field(default_factory=SchemaSettings)
     email: EmailSettings = Field(default_factory=EmailSettings)
     test: TestSettings = Field(default_factory=TestSettings)
+    debug: DebugSettings = Field(default_factory=DebugSettings)
 
 
 # Auto-load .env file from current directory if it exists

rem/sql/migrations/001_install.sql

@@ -822,7 +822,7 @@ COMMENT ON FUNCTION fn_get_shared_messages IS
 -- Function to list sessions with user details (name, email) for admin views
 
 -- List sessions with user info, CTE pagination
--- Note: messages.session_id stores the session name (not UUID), so we join on sessions.name
+-- Note: messages.session_id stores the session UUID (sessions.id)
 CREATE OR REPLACE FUNCTION fn_list_sessions_with_user(
     p_user_id VARCHAR(256) DEFAULT NULL,  -- Filter by user_id (NULL = all users, admin only)
     p_user_name VARCHAR(256) DEFAULT NULL,  -- Filter by user name (partial match, admin only)
@@ -849,9 +849,9 @@ RETURNS TABLE(
 BEGIN
     RETURN QUERY
     WITH session_msg_counts AS (
-        -- Count messages per session (joining on session name since messages.session_id = sessions.name)
+        -- Count messages per session (joining on session UUID)
         SELECT
-            m.session_id as session_name,
+            m.session_id,
             COUNT(*)::INTEGER as actual_message_count
         FROM messages m
         GROUP BY m.session_id
@@ -872,7 +872,7 @@ BEGIN
             s.metadata
         FROM sessions s
         LEFT JOIN users u ON u.id::text = s.user_id
-        LEFT JOIN session_msg_counts mc ON mc.session_name = s.name
+        LEFT JOIN session_msg_counts mc ON mc.session_id = s.id::text
         WHERE s.deleted_at IS NULL
           AND (p_user_id IS NULL OR s.user_id = p_user_id)
           AND (p_user_name IS NULL OR u.name ILIKE '%' || p_user_name || '%')