remdb 0.3.230__py3-none-any.whl → 0.3.258__py3-none-any.whl

rem/cli/commands/session.py

@@ -331,6 +331,123 @@ async def _show_async(
             raise
 
 
+@session.command("clone")
+@click.argument("session_id")
+@click.option("--to-turn", "-t", type=int, help="Clone up to turn N (counting user messages only)")
+@click.option("--name", "-n", help="Name/description for the cloned session")
+def clone(session_id: str, to_turn: int | None, name: str | None):
+    """
+    Clone a session for exploring alternate conversation paths.
+
+    SESSION_ID: The session ID to clone.
+
+    Examples:
+
+        # Clone entire session
+        rem session clone 810f1f2d-d5a1-4c02-83b6-67040b47f7c0
+
+        # Clone up to turn 3 (first 3 user messages and their responses)
+        rem session clone 810f1f2d-d5a1-4c02-83b6-67040b47f7c0 --to-turn 3
+
+        # Clone with a descriptive name
+        rem session clone 810f1f2d-d5a1-4c02-83b6-67040b47f7c0 -n "Alternate anxiety path"
+    """
+    asyncio.run(_clone_async(session_id, to_turn, name))
+
+
+async def _clone_async(
+    session_id: str,
+    to_turn: int | None,
+    name: str | None,
+):
+    """Async implementation of clone command."""
+    from uuid import uuid4
+    from ...models.entities.session import Session, SessionMode
+
+    pg = get_postgres_service()
+    if not pg:
+        logger.error("PostgreSQL not available")
+        return
+
+    await pg.connect()
+
+    try:
+        # Load original session messages
+        message_repo = Repository(Message, "messages", db=pg)
+        messages = await message_repo.find(
+            filters={"session_id": session_id},
+            order_by="created_at ASC",
+            limit=1000,
+        )
+
+        if not messages:
+            logger.error(f"No messages found for session {session_id}")
+            return
+
+        # If --to-turn specified, filter messages up to that turn (user messages)
+        if to_turn is not None:
+            user_count = 0
+            cutoff_idx = len(messages)
+            for idx, msg in enumerate(messages):
+                if msg.message_type == "user":
+                    user_count += 1
+                    if user_count > to_turn:
+                        cutoff_idx = idx
+                        break
+            messages = messages[:cutoff_idx]
+            logger.info(f"Cloning {len(messages)} messages (up to turn {to_turn})")
+        else:
+            logger.info(f"Cloning all {len(messages)} messages")
+
+        # Generate new session ID
+        new_session_id = str(uuid4())
+
+        # Get user_id and tenant_id from first message
+        first_msg = messages[0]
+        user_id = first_msg.user_id
+        tenant_id = first_msg.tenant_id or "default"
+
+        # Create Session record with CLONE mode and lineage
+        session_repo = Repository(Session, "sessions", db=pg)
+        new_session = Session(
+            id=uuid4(),
+            name=name or f"Clone of {session_id[:8]}",
+            mode=SessionMode.CLONE,
+            original_trace_id=session_id,
+            description=f"Cloned from session {session_id}" + (f" at turn {to_turn}" if to_turn else ""),
+            user_id=user_id,
+            tenant_id=tenant_id,
+            message_count=len(messages),
+        )
+        await session_repo.upsert(new_session)
+        logger.info(f"Created session record: {new_session.id}")
+
+        # Copy messages with new session_id
+        for msg in messages:
+            new_msg = Message(
+                id=uuid4(),
+                user_id=msg.user_id,
+                tenant_id=msg.tenant_id,
+                session_id=str(new_session.id),
+                content=msg.content,
+                message_type=msg.message_type,
+                metadata=msg.metadata,
+            )
+            await message_repo.upsert(new_msg)
+
+        click.echo(f"\n✅ Cloned session successfully!")
+        click.echo(f"   Original: {session_id}")
+        click.echo(f"   New:      {new_session.id}")
+        click.echo(f"   Messages: {len(messages)}")
+        if to_turn:
+            click.echo(f"   Turns:    {to_turn}")
+        click.echo(f"\nContinue this session with:")
+        click.echo(f"   rem ask <agent> \"your message\" --session-id {new_session.id}")
+
+    finally:
+        await pg.disconnect()
+
+
 def register_command(cli_group):
     """Register the session command group."""
     cli_group.add_command(session)

rem/cli/main.py

@@ -97,6 +97,7 @@ from .commands.mcp import register_command as register_mcp_command
 from .commands.scaffold import scaffold as scaffold_command
 from .commands.cluster import register_commands as register_cluster_commands
 from .commands.session import register_command as register_session_command
+from .commands.query import register_command as register_query_command
 
 register_schema_commands(schema)
 register_db_commands(db)
@@ -107,6 +108,7 @@ register_ask_command(cli)
 register_configure_command(cli)
 register_serve_command(cli)
 register_mcp_command(cli)
+register_query_command(cli)
 cli.add_command(experiments_group)
 cli.add_command(scaffold_command)
 register_session_command(cli)

rem/models/core/experiment.py

@@ -461,7 +461,7 @@ class ExperimentConfig(BaseModel):
         """
         Get the evaluator filename with task prefix.
 
-        Returns: {agent_name}-{task}.yaml (e.g., siggy-risk-assessment.yaml)
+        Returns: {agent_name}-{task}.yaml (e.g., rem-risk-assessment.yaml)
         """
         return f"{self.agent_schema_ref.name}-{self.task}.yaml"
 

rem/models/entities/session.py

@@ -21,6 +21,7 @@ class SessionMode(str, Enum):
 
     NORMAL = "normal"
     EVALUATION = "evaluation"
+    CLONE = "clone"
 
 
 class Session(CoreModel):

rem/schemas/agents/core/agent-builder.yaml

@@ -229,7 +229,7 @@ json_schema_extra:
     - uri: rem://agents
       description: "List all available agent schemas with descriptions"
     - uri: rem://agents/{agent_name}
-      description: "Load a specific agent schema by name (e.g., 'rem', 'siggy')"
+      description: "Load a specific agent schema by name (e.g., 'rem', 'intake')"
   tools:
     - name: save_agent
       description: "Save the agent schema. Only call when user approves the preview in Step 6."

rem/schemas/agents/test_orchestrator.yaml

@@ -0,0 +1,42 @@
+# =============================================================================
+# TEST ORCHESTRATOR AGENT
+# =============================================================================
+# Parent agent that delegates to test_structured_output for testing
+# the structured output persistence feature.
+# =============================================================================
+
+name: test_orchestrator
+version: "1.0"
+description: |
+  You are an orchestrator that helps analyze user messages.
+
+  When the user provides a message to analyze, you MUST:
+  1. Call the ask_agent tool to delegate to "test_structured_output"
+  2. Return the structured result to the user
+
+  ## CRITICAL RULES
+
+  - ALWAYS call ask_agent with agent_name="test_structured_output"
+  - Pass the user's message as input_text
+  - Report back the structured result you receive
+
+type: object
+properties:
+  answer:
+    type: string
+    description: Response to the user
+
+required:
+  - answer
+
+json_schema_extra:
+  kind: agent
+  name: test_orchestrator
+  version: "1.0.0"
+  tags: [test, orchestrator]
+  tools:
+    - name: ask_agent
+      description: |
+        Delegate to the test_structured_output agent to analyze the message.
+        Always use agent_name="test_structured_output".
+  resources: []

rem/schemas/agents/test_structured_output.yaml

@@ -0,0 +1,52 @@
+# =============================================================================
+# TEST STRUCTURED OUTPUT AGENT
+# =============================================================================
+# Simple agent for testing structured_output: true functionality
+# =============================================================================
+
+name: test_structured_output
+version: "1.0"
+description: |
+  You are a test agent that produces structured output.
+
+  Your ONLY job is to return a structured response matching the schema below.
+
+  Based on the user's input, extract:
+  - summary: A brief summary of what they said
+  - sentiment: positive, negative, or neutral
+  - keywords: List of key words from their message
+
+  DO NOT ask questions. Just produce the structured output.
+
+type: object
+properties:
+  result:
+    type: object
+    description: Structured analysis result
+    properties:
+      summary:
+        type: string
+        description: Brief summary of the input
+      sentiment:
+        type: string
+        enum: [positive, negative, neutral]
+        description: Overall sentiment
+      keywords:
+        type: array
+        items:
+          type: string
+        description: Key words extracted from input
+    required: [summary, sentiment, keywords]
+    additionalProperties: false
+
+required:
+  - result
+
+json_schema_extra:
+  kind: agent
+  name: test_structured_output
+  version: "1.0.0"
+  tags: [test, structured-output]
+  structured_output: true
+  tools: []
+  resources: []

rem/services/content/providers.py

@@ -118,15 +118,40 @@ class DocProvider(ContentProvider):
     - Images (.png, .jpg) - OCR text extraction
 
     Handles:
-    - Text extraction with OCR fallback
+    - Text extraction with automatic OCR fallback for scanned documents
     - Table detection and extraction
     - Daemon process workaround for multiprocessing restrictions
+
+    Environment Variables:
+        EXTRACTION_OCR_FALLBACK: Enable OCR fallback (default: true)
+        EXTRACTION_OCR_THRESHOLD: Min chars before triggering OCR fallback (default: 100)
+        EXTRACTION_FORCE_OCR: Always use OCR, skip native extraction (default: false)
+        EXTRACTION_OCR_LANGUAGE: Tesseract language codes (default: eng)
     """
 
     @property
     def name(self) -> str:
         return "doc"
 
+    def _get_env_bool(self, key: str, default: bool) -> bool:
+        """Get boolean from environment variable."""
+        import os
+        val = os.environ.get(key, "").lower()
+        if val in ("true", "1", "yes"):
+            return True
+        elif val in ("false", "0", "no"):
+            return False
+        return default
+
+    def _get_env_int(self, key: str, default: int) -> int:
+        """Get integer from environment variable."""
+        import os
+        val = os.environ.get(key, "")
+        try:
+            return int(val) if val else default
+        except ValueError:
+            return default
+
     def _is_daemon_process(self) -> bool:
         """Check if running in a daemon process."""
         try:
@@ -134,29 +159,34 @@ class DocProvider(ContentProvider):
         except Exception:
             return False
 
-    def _parse_in_subprocess(self, file_path: Path) -> dict:
+    def _parse_in_subprocess(self, file_path: Path, force_ocr: bool = False) -> dict:
         """Run kreuzberg in a separate subprocess to bypass daemon restrictions."""
-        script = """
+        import os
+        ocr_language = os.environ.get("EXTRACTION_OCR_LANGUAGE", "eng")
+
+        script = f"""
 import json
 import sys
 from pathlib import Path
-from kreuzberg import ExtractionConfig, extract_file_sync
+from kreuzberg import ExtractionConfig, OcrConfig, extract_file_sync
 
-# Parse document with kreuzberg 3.x
-config = ExtractionConfig(
-    extract_tables=True,
-    chunk_content=False,
-    extract_keywords=False,
-)
+force_ocr = {force_ocr}
+
+if force_ocr:
+    config = ExtractionConfig(
+        force_ocr=True,
+        ocr=OcrConfig(backend="tesseract", language="{ocr_language}")
+    )
+else:
+    config = ExtractionConfig()
 
 result = extract_file_sync(Path(sys.argv[1]), config=config)
 
-# Serialize result to JSON
-output = {
+output = {{
     'content': result.content,
-    'tables': [t.model_dump() for t in result.tables] if result.tables else [],
-    'metadata': result.metadata
-}
+    'tables': [],
+    'metadata': {{}}
+}}
 print(json.dumps(output))
 """
 
@@ -173,9 +203,41 @@ print(json.dumps(output))
 
         return json.loads(result.stdout)
 
+    def _extract_with_config(self, tmp_path: Path, force_ocr: bool = False) -> tuple[str, dict]:
+        """Extract content with optional OCR config."""
+        import os
+        from kreuzberg import ExtractionConfig, OcrConfig, extract_file_sync
+
+        ocr_language = os.environ.get("EXTRACTION_OCR_LANGUAGE", "eng")
+
+        if force_ocr:
+            config = ExtractionConfig(
+                force_ocr=True,
+                ocr=OcrConfig(backend="tesseract", language=ocr_language)
+            )
+            parser_name = "kreuzberg_ocr"
+        else:
+            config = ExtractionConfig()
+            parser_name = "kreuzberg"
+
+        result = extract_file_sync(tmp_path, config=config)
+        text = result.content
+
+        extraction_metadata = {
+            "parser": parser_name,
+            "file_extension": tmp_path.suffix,
+        }
+
+        return text, extraction_metadata
+
     def extract(self, content: bytes, metadata: dict[str, Any]) -> dict[str, Any]:
         """
-        Extract document content using Kreuzberg.
+        Extract document content using Kreuzberg with intelligent OCR fallback.
+
+        Process:
+        1. Try native text extraction first (fast, preserves structure)
+        2. If content is minimal (< threshold chars), retry with OCR
+        3. Use OCR result if it's better than native result
 
         Args:
             content: Document file bytes
@@ -184,49 +246,89 @@ print(json.dumps(output))
         Returns:
             dict with text and extraction metadata
         """
+        # Get OCR settings from environment
+        force_ocr = self._get_env_bool("EXTRACTION_FORCE_OCR", False)
+        ocr_fallback = self._get_env_bool("EXTRACTION_OCR_FALLBACK", True)
+        ocr_threshold = self._get_env_int("EXTRACTION_OCR_THRESHOLD", 100)
+
         # Write bytes to temp file for kreuzberg
-        # Detect extension from metadata
         content_type = metadata.get("content_type", "")
         suffix = get_extension(content_type, default=".pdf")
 
         with temp_file_from_bytes(content, suffix=suffix) as tmp_path:
+            ocr_used = False
+            ocr_fallback_triggered = False
+            native_char_count = 0
+
             # Check if running in daemon process
             if self._is_daemon_process():
-                logger.info("Daemon process detected - using subprocess workaround for document parsing")
+                logger.info("Daemon process detected - using subprocess workaround")
                 try:
-                    result_dict = self._parse_in_subprocess(tmp_path)
-                    text = result_dict["content"]
-                    extraction_metadata = {
-                        "table_count": len(result_dict["tables"]),
-                        "parser": "kreuzberg_subprocess",
-                        "file_extension": tmp_path.suffix,
-                    }
+                    if force_ocr:
+                        result_dict = self._parse_in_subprocess(tmp_path, force_ocr=True)
+                        text = result_dict["content"]
+                        ocr_used = True
+                        extraction_metadata = {
+                            "parser": "kreuzberg_subprocess_ocr",
+                            "file_extension": tmp_path.suffix,
+                        }
+                    else:
+                        # Try native first
+                        result_dict = self._parse_in_subprocess(tmp_path, force_ocr=False)
+                        text = result_dict["content"]
+                        native_char_count = len(text)
+
+                        # OCR fallback if content is minimal
+                        if ocr_fallback and len(text.strip()) < ocr_threshold:
+                            logger.warning(f"Content below threshold ({len(text.strip())} < {ocr_threshold}) - trying OCR fallback")
+                            try:
+                                ocr_result = self._parse_in_subprocess(tmp_path, force_ocr=True)
+                                ocr_text = ocr_result["content"]
+                                if len(ocr_text.strip()) > len(text.strip()):
+                                    logger.info(f"OCR fallback improved result: {len(ocr_text)} chars (was {native_char_count})")
+                                    text = ocr_text
+                                    ocr_used = True
+                                    ocr_fallback_triggered = True
+                            except Exception as e:
+                                logger.warning(f"OCR fallback failed in subprocess: {e}")
+
+                        extraction_metadata = {
+                            "parser": "kreuzberg_subprocess" if not ocr_used else "kreuzberg_subprocess_ocr_fallback",
+                            "file_extension": tmp_path.suffix,
+                        }
                 except Exception as e:
-                    logger.error(f"Subprocess parsing failed: {e}. Falling back to text-only.")
-                    # Fallback to simple text extraction (kreuzberg 3.x API)
-                    from kreuzberg import ExtractionConfig, extract_file_sync
-                    config = ExtractionConfig(extract_tables=False)
-                    result = extract_file_sync(tmp_path, config=config)
-                    text = result.content
-                    extraction_metadata = {
-                        "parser": "kreuzberg_fallback",
-                        "file_extension": tmp_path.suffix,
-                    }
+                    logger.error(f"Subprocess parsing failed: {e}. Falling back to direct call.")
+                    text, extraction_metadata = self._extract_with_config(tmp_path, force_ocr=force_ocr)
+                    ocr_used = force_ocr
             else:
-                # Normal execution (not in daemon) - kreuzberg 4.x with native ONNX/Rust
-                from kreuzberg import ExtractionConfig, extract_file_sync
-                config = ExtractionConfig(
-                    enable_quality_processing=True,  # Enables table extraction with native ONNX
-                    chunk_content=False,  # We handle chunking ourselves
-                    extract_tables=False,  # Disable table extraction to avoid PyTorch dependency
-                )
-                result = extract_file_sync(tmp_path, config=config)
-                text = result.content
-                extraction_metadata = {
-                    "table_count": len(result.tables) if result.tables else 0,
-                    "parser": "kreuzberg",
-                    "file_extension": tmp_path.suffix,
-                }
+                # Normal execution (not in daemon)
+                if force_ocr:
+                    text, extraction_metadata = self._extract_with_config(tmp_path, force_ocr=True)
+                    ocr_used = True
+                else:
+                    # Try native first
+                    text, extraction_metadata = self._extract_with_config(tmp_path, force_ocr=False)
+                    native_char_count = len(text)
+
+                    # OCR fallback if content is minimal
+                    if ocr_fallback and len(text.strip()) < ocr_threshold:
+                        logger.warning(f"Content below threshold ({len(text.strip())} < {ocr_threshold}) - trying OCR fallback")
+                        try:
+                            ocr_text, _ = self._extract_with_config(tmp_path, force_ocr=True)
+                            if len(ocr_text.strip()) > len(text.strip()):
+                                logger.info(f"OCR fallback improved result: {len(ocr_text)} chars (was {native_char_count})")
+                                text = ocr_text
+                                ocr_used = True
+                                ocr_fallback_triggered = True
+                                extraction_metadata["parser"] = "kreuzberg_ocr_fallback"
+                        except Exception as e:
+                            logger.warning(f"OCR fallback failed: {e}")
+
+            # Add OCR metadata
+            extraction_metadata["ocr_used"] = ocr_used
+            extraction_metadata["ocr_fallback_triggered"] = ocr_fallback_triggered
+            extraction_metadata["native_char_count"] = native_char_count
+            extraction_metadata["final_char_count"] = len(text)
 
             return {
                 "text": text,

rem/services/postgres/repository.py

@@ -25,6 +25,7 @@ from .sql_builder import (
     build_select,
     build_upsert,
 )
+from ...settings import settings
 
 if TYPE_CHECKING:
     from .service import PostgresService

rem/services/rem/README.md

@@ -40,15 +40,16 @@ FuzzyQuery ::= FUZZY <text:string> [THRESHOLD <t:float>] [LIMIT <n:int>]
   available   : Stage 1+
   example     : FUZZY "sara" THRESHOLD 0.5 LIMIT 10
 
-SearchQuery ::= SEARCH <text:string> [TABLE <table:string>] [WHERE <clause:string>] [LIMIT <n:int>]
+SearchQuery ::= SEARCH <text:string> [IN|TABLE <table:string>] [WHERE <clause:string>] [LIMIT <n:int>]
   text        : Semantic query text
-  table       : Target table (default: "resources")
+  table       : Target table (default: "resources"). Use IN or TABLE keyword.
   clause      : Optional PostgreSQL WHERE clause for hybrid filtering (combines vector + structured)
   limit       : Max results (default: 10)
   performance : Indexed (pgvector)
   available   : Stage 3+
   examples    :
-    - SEARCH "database migration" TABLE resources LIMIT 10
+    - SEARCH "database migration" IN resources LIMIT 10
+    - SEARCH "parcel delivery" IN ontologies
     - SEARCH "team discussion" TABLE moments WHERE "moment_type='meeting'" LIMIT 5
     - SEARCH "project updates" WHERE "created_at >= '2024-01-01'" LIMIT 20
     - SEARCH "AI research" WHERE "tags @> ARRAY['machine-learning']" LIMIT 10

rem/services/rem/parser.py

@@ -64,7 +64,7 @@ class RemQueryParser:
             token_upper = token.upper()
 
             # Handle REM keywords that take a value
-            if token_upper in ("LIMIT", "DEPTH", "THRESHOLD", "TYPE", "FROM", "WITH"):
+            if token_upper in ("LIMIT", "DEPTH", "THRESHOLD", "TYPE", "FROM", "WITH", "TABLE", "IN", "WHERE"):
                 if i + 1 < len(tokens):
                     keyword_map = {
                         "LIMIT": "limit",
@@ -73,6 +73,9 @@ class RemQueryParser:
                         "TYPE": "edge_types",
                         "FROM": "initial_query",
                         "WITH": "initial_query",
+                        "TABLE": "table_name",
+                        "IN": "table_name",  # IN is alias for TABLE
+                        "WHERE": "where_clause",
                     }
                     key = keyword_map[token_upper]
                     value = tokens[i + 1]
@@ -161,15 +164,9 @@ class RemQueryParser:
             params["query_text"] = combined_value
 
         elif query_type == QueryType.SEARCH:
-            # SEARCH expects: SEARCH <table> <query_text> [LIMIT n]
-            # First positional arg is table name, rest is query text
-            if len(positional_args) >= 2:
-                params["table_name"] = positional_args[0]
-                params["query_text"] = " ".join(positional_args[1:])
-            elif len(positional_args) == 1:
-                # Could be table name or query text - assume query text if no table
-                params["query_text"] = positional_args[0]
-            # If no positional args, params stays empty
+            # SEARCH expects: SEARCH <text> [TABLE <table>] [WHERE <clause>] [LIMIT n]
+            # All positional args are query_text, TABLE/WHERE/LIMIT are handled as keywords
+            params["query_text"] = combined_value
 
         elif query_type == QueryType.TRAVERSE:
             params["initial_query"] = combined_value

rem/services/rem/service.py

@@ -478,6 +478,53 @@ class RemService:
         parser = RemQueryParser()
         return parser.parse(query_string)
 
+    async def execute_query_string(
+        self, query_string: str, user_id: str | None = None
+    ) -> dict[str, Any]:
+        """
+        Execute a REM dialect query string directly.
+
+        This is the unified entry point for executing REM queries from both
+        the CLI and API. It handles parsing the query string, creating the
+        RemQuery model, and executing it.
+
+        Args:
+            query_string: REM dialect query (e.g., 'LOOKUP "Sarah Chen"',
+                         'SEARCH resources "API design"', 'SELECT * FROM users')
+            user_id: Optional user ID for query isolation
+
+        Returns:
+            Dict with query results and metadata:
+            - query_type: The type of query executed
+            - results: List of result rows
+            - count: Number of results
+            - Additional fields depending on query type
+
+        Raises:
+            ValueError: If the query string is invalid
+            QueryExecutionError: If query execution fails
+
+        Example:
+            >>> result = await rem_service.execute_query_string(
+            ...     'LOOKUP "Sarah Chen"',
+            ...     user_id="user-123"
+            ... )
+            >>> print(result["count"])
+            1
+        """
+        # Parse the query string into type and parameters
+        query_type, parameters = self._parse_query_string(query_string)
+
+        # Create and validate the RemQuery model
+        rem_query = RemQuery.model_validate({
+            "query_type": query_type,
+            "parameters": parameters,
+            "user_id": user_id,
+        })
+
+        # Execute and return results
+        return await self.execute_query(rem_query)
+
     async def ask_rem(
         self, natural_query: str, tenant_id: str, llm_model: str | None = None, plan_mode: bool = False
     ) -> dict[str, Any]: