remdb 0.3.7__py3-none-any.whl → 0.3.14__py3-none-any.whl

rem/services/content/providers.py

@@ -2,17 +2,27 @@
 
 import json
 import multiprocessing
-import os
 import random
 import subprocess
 import sys
-import tempfile
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, Optional
 
 from loguru import logger
 
+from rem.utils.constants import (
+    AUDIO_CHUNK_TARGET_SECONDS,
+    AUDIO_CHUNK_WINDOW_SECONDS,
+    MIN_SILENCE_MS,
+    SILENCE_THRESHOLD_DB,
+    SUBPROCESS_TIMEOUT_SECONDS,
+    WAV_HEADER_MIN_BYTES,
+    WHISPER_COST_PER_MINUTE,
+)
+from rem.utils.files import temp_file_from_bytes
+from rem.utils.mime_types import get_extension
+
 
 class ContentProvider(ABC):
     """Base class for content extraction providers."""
@@ -132,7 +142,7 @@ import sys
 from pathlib import Path
 from kreuzberg import ExtractionConfig, extract_file_sync
 
-# Parse document with table extraction (requires PyTorch - Python <3.13 required)
+# Parse document with kreuzberg 3.x
 config = ExtractionConfig(
     extract_tables=True,
     chunk_content=False,
@@ -155,7 +165,7 @@ print(json.dumps(output))
             [sys.executable, "-c", script, str(file_path)],
             capture_output=True,
             text=True,
-            timeout=300,  # 5 minute timeout
+            timeout=SUBPROCESS_TIMEOUT_SECONDS,
         )
 
         if result.returncode != 0:
@@ -177,21 +187,9 @@ print(json.dumps(output))
         # Write bytes to temp file for kreuzberg
         # Detect extension from metadata
         content_type = metadata.get("content_type", "")
-        extension_map = {
-            "application/pdf": ".pdf",
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
-            "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
-            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
-            "image/png": ".png",
-            "image/jpeg": ".jpg",
-        }
-        suffix = extension_map.get(content_type, ".pdf")  # Default to PDF
+        suffix = get_extension(content_type, default=".pdf")
 
-        with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
-            tmp.write(content)
-            tmp_path = Path(tmp.name)
-
-        try:
+        with temp_file_from_bytes(content, suffix=suffix) as tmp_path:
             # Check if running in daemon process
             if self._is_daemon_process():
                 logger.info("Daemon process detected - using subprocess workaround for document parsing")
@@ -205,7 +203,7 @@ print(json.dumps(output))
                     }
                 except Exception as e:
                     logger.error(f"Subprocess parsing failed: {e}. Falling back to text-only.")
-                    # Fallback to simple text extraction
+                    # Fallback to simple text extraction (kreuzberg 3.x API)
                     from kreuzberg import ExtractionConfig, extract_file_sync
                     config = ExtractionConfig(extract_tables=False)
                     result = extract_file_sync(tmp_path, config=config)
@@ -215,13 +213,12 @@ print(json.dumps(output))
                         "file_extension": tmp_path.suffix,
                     }
             else:
-                # Normal execution (not in daemon)
+                # Normal execution (not in daemon) - kreuzberg 4.x with native ONNX/Rust
                 from kreuzberg import ExtractionConfig, extract_file_sync
-                # Table extraction with gmft (requires PyTorch - Python <3.13 required)
                 config = ExtractionConfig(
-                    extract_tables=True,
-                    chunk_content=False,
-                    extract_keywords=False,
+                    enable_quality_processing=True,  # Enables table extraction with native ONNX
+                    chunk_content=False,  # We handle chunking ourselves
+                    extract_tables=False,  # Disable table extraction to avoid PyTorch dependency
                 )
                 result = extract_file_sync(tmp_path, config=config)
                 text = result.content
@@ -236,10 +233,6 @@ print(json.dumps(output))
                 "metadata": extraction_metadata,
             }
 
-        finally:
-            # Clean up temp file
-            tmp_path.unlink(missing_ok=True)
-
 
 class AudioProvider(ContentProvider):
     """
@@ -284,19 +277,20 @@ class AudioProvider(ContentProvider):
             ValueError: If OpenAI API key missing
         """
         # Handle empty or invalid content
-        if not content or len(content) < 44:  # WAV header is minimum 44 bytes
+        if not content or len(content) < WAV_HEADER_MIN_BYTES:
             logger.warning("Audio content too small to be valid WAV file")
             return {
                 "text": "[Invalid or empty audio file]",
                 "metadata": {"error": "invalid_content", "size": len(content)},
             }
 
-        # Check for OpenAI API key
-        api_key = os.getenv("OPENAI_API_KEY")
+        # Check for OpenAI API key (use settings)
+        from rem.settings import settings
+        api_key = settings.llm.openai_api_key
         if not api_key:
-            logger.warning("No OPENAI_API_KEY found - audio transcription disabled")
+            logger.warning("No OpenAI API key found - audio transcription disabled")
             return {
-                "text": "[Audio transcription requires OPENAI_API_KEY environment variable]",
+                "text": "[Audio transcription requires LLM__OPENAI_API_KEY to be set]",
                 "metadata": {"error": "missing_api_key"},
             }
 
@@ -313,83 +307,74 @@ class AudioProvider(ContentProvider):
         # Write bytes to temp file
         # Detect extension from metadata or use .wav as fallback
         content_type = metadata.get("content_type", "audio/wav")
-        extension_map = {
-            "audio/wav": ".wav",
-            "audio/mpeg": ".mp3",
-            "audio/mp4": ".m4a",
-            "audio/x-m4a": ".m4a",
-            "audio/flac": ".flac",
-            "audio/ogg": ".ogg",
-        }
-        extension = extension_map.get(content_type, ".wav")
+        extension = get_extension(content_type, default=".wav")
 
-        with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as tmp:
-            tmp.write(content)
-            tmp_path = Path(tmp.name)
+        chunker = None
+        chunks = None
 
-        try:
-            logger.info(f"Processing audio file: {tmp_path.name} ({len(content) / 1024 / 1024:.1f} MB)")
-
-            # Step 1: Chunk audio by silence
-            chunker = AudioChunker(
-                target_chunk_seconds=60.0,
-                chunk_window_seconds=2.0,
-                silence_threshold_db=-40.0,
-                min_silence_ms=500,
-            )
-
-            chunks = chunker.chunk_audio(tmp_path)
-            logger.info(f"Created {len(chunks)} audio chunks")
-
-            # Step 2: Transcribe chunks
-            transcriber = AudioTranscriber(api_key=api_key)
-            results = transcriber.transcribe_chunks(chunks)
-            logger.info(f"Transcribed {len(results)} chunks")
-
-            # Step 3: Combine into markdown format
-            # Format: Each chunk becomes a section with timestamp
-            markdown_parts = []
-            for result in results:
-                timestamp = f"{result.start_seconds:.1f}s - {result.end_seconds:.1f}s"
-                markdown_parts.append(f"## [{timestamp}]\n\n{result.text}\n")
-
-            markdown_text = "\n".join(markdown_parts)
-
-            # Calculate metadata
-            total_duration = sum(r.duration_seconds for r in results)
-            estimated_cost = (total_duration / 60) * 0.006  # $0.006 per minute
-            successful_chunks = sum(1 for r in results if r.confidence > 0)
-
-            extraction_metadata = {
-                "chunk_count": len(chunks),
-                "transcribed_chunks": successful_chunks,
-                "duration_seconds": total_duration,
-                "estimated_cost": estimated_cost,
-                "parser": "whisper_api",
-            }
+        with temp_file_from_bytes(content, suffix=extension) as tmp_path:
+            try:
+                logger.info(f"Processing audio file: {tmp_path.name} ({len(content) / 1024 / 1024:.1f} MB)")
+
+                # Step 1: Chunk audio by silence
+                chunker = AudioChunker(
+                    target_chunk_seconds=AUDIO_CHUNK_TARGET_SECONDS,
+                    chunk_window_seconds=AUDIO_CHUNK_WINDOW_SECONDS,
+                    silence_threshold_db=SILENCE_THRESHOLD_DB,
+                    min_silence_ms=MIN_SILENCE_MS,
+                )
 
-            logger.info(
-                f"Transcription complete: {successful_chunks}/{len(chunks)} chunks, "
-                f"${estimated_cost:.3f} cost"
-            )
+                chunks = chunker.chunk_audio(tmp_path)
+                logger.info(f"Created {len(chunks)} audio chunks")
 
-            return {
-                "text": markdown_text,
-                "metadata": extraction_metadata,
-            }
+                # Step 2: Transcribe chunks
+                transcriber = AudioTranscriber(api_key=api_key)
+                results = transcriber.transcribe_chunks(chunks)
+                logger.info(f"Transcribed {len(results)} chunks")
 
-        except Exception as e:
-            logger.error(f"Audio extraction failed: {e}")
-            raise RuntimeError(f"Audio transcription failed: {e}") from e
+                # Step 3: Combine into markdown format
+                # Format: Each chunk becomes a section with timestamp
+                markdown_parts = []
+                for result in results:
+                    timestamp = f"{result.start_seconds:.1f}s - {result.end_seconds:.1f}s"
+                    markdown_parts.append(f"## [{timestamp}]\n\n{result.text}\n")
+
+                markdown_text = "\n".join(markdown_parts)
+
+                # Calculate metadata
+                total_duration = sum(r.duration_seconds for r in results)
+                estimated_cost = (total_duration / 60) * WHISPER_COST_PER_MINUTE
+                successful_chunks = sum(1 for r in results if r.confidence > 0)
+
+                extraction_metadata = {
+                    "chunk_count": len(chunks),
+                    "transcribed_chunks": successful_chunks,
+                    "duration_seconds": total_duration,
+                    "estimated_cost": estimated_cost,
+                    "parser": "whisper_api",
+                }
+
+                logger.info(
+                    f"Transcription complete: {successful_chunks}/{len(chunks)} chunks, "
+                    f"${estimated_cost:.3f} cost"
+                )
+
+                return {
+                    "text": markdown_text,
+                    "metadata": extraction_metadata,
+                }
 
-        finally:
-            # Clean up temp file and chunks
-            try:
-                tmp_path.unlink(missing_ok=True)
-                if 'chunker' in locals() and 'chunks' in locals():
-                    chunker.cleanup_chunks(chunks)
             except Exception as e:
-                logger.warning(f"Cleanup failed: {e}")
+                logger.error(f"Audio extraction failed: {e}")
+                raise RuntimeError(f"Audio transcription failed: {e}") from e
+
+            finally:
+                # Clean up audio chunks (temp file cleanup handled by context manager)
+                if chunker is not None and chunks is not None:
+                    try:
+                        chunker.cleanup_chunks(chunks)
+                    except Exception as e:
+                        logger.warning(f"Chunk cleanup failed: {e}")
 
 
 class SchemaProvider(ContentProvider):
@@ -667,19 +652,9 @@ class ImageProvider(ContentProvider):
 
                 # Write bytes to temp file for analysis
                 content_type = metadata.get("content_type", "image/png")
-                extension_map = {
-                    "image/png": ".png",
-                    "image/jpeg": ".jpg",
-                    "image/gif": ".gif",
-                    "image/webp": ".webp",
-                }
-                extension = extension_map.get(content_type, ".png")
-
-                with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as tmp:
-                    tmp.write(content)
-                    tmp_path = Path(tmp.name)
+                extension = get_extension(content_type, default=".png")
 
-                try:
+                with temp_file_from_bytes(content, suffix=extension) as tmp_path:
                     # Analyze image
                     result = analyzer.analyze_image(tmp_path)
                     vision_description = result.description
@@ -687,9 +662,6 @@ class ImageProvider(ContentProvider):
                     vision_model = result.model
 
                     logger.info(f"Vision analysis complete: {len(vision_description)} chars")
-                finally:
-                    # Clean up temp file
-                    tmp_path.unlink(missing_ok=True)
 
             except ImportError as e:
                 logger.warning(f"Vision analysis not available: {e}")
@@ -732,19 +704,9 @@ class ImageProvider(ContentProvider):
                 if embedder.is_available():
                     # Write bytes to temp file for CLIP embedding
                     content_type = metadata.get("content_type", "image/png")
-                    extension_map = {
-                        "image/png": ".png",
-                        "image/jpeg": ".jpg",
-                        "image/gif": ".gif",
-                        "image/webp": ".webp",
-                    }
-                    extension = extension_map.get(content_type, ".png")
+                    extension = get_extension(content_type, default=".png")
 
-                    with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as tmp:
-                        tmp.write(content)
-                        tmp_path = Path(tmp.name)
-
-                    try:
+                    with temp_file_from_bytes(content, suffix=extension) as tmp_path:
                         # Generate CLIP embedding
                         result = embedder.embed_image(tmp_path)
                         if result:
@@ -754,9 +716,6 @@ class ImageProvider(ContentProvider):
                             logger.info(
                                 f"CLIP embedding generated: {clip_dimensions} dims, {clip_tokens} tokens"
                             )
-                    finally:
-                        # Clean up temp file
-                        tmp_path.unlink(missing_ok=True)
                 else:
                     logger.debug(
                         "CLIP embeddings disabled - set CONTENT__JINA_API_KEY to enable. "

rem/services/dreaming/affinity_service.py

@@ -8,12 +8,11 @@ vector similarity (fast) or LLM analysis (intelligent).
 import json
 from datetime import datetime, timedelta
 from enum import Enum
-from pathlib import Path
 from typing import Any, Optional
 
-import yaml
 from loguru import logger
 
+from ...utils.schema_loader import load_agent_schema
 from ...agentic.providers.pydantic_ai import create_agent
 from ...agentic.serialization import serialize_agent_result
 from ...models.core import QueryType, RemQuery, SearchParameters
@@ -125,20 +124,7 @@ async def build_affinity(
     # Load LLM agent for relationship assessment if needed
     affinity_agent = None
     if mode == AffinityMode.LLM:
-        schema_path = (
-            Path(__file__).parent.parent.parent
-            / "schemas"
-            / "agents"
-            / "resource-affinity-assessor.yaml"
-        )
-
-        if not schema_path.exists():
-            raise FileNotFoundError(
-                f"ResourceAffinityAssessor schema not found: {schema_path}"
-            )
-
-        with open(schema_path) as f:
-            agent_schema = yaml.safe_load(f)
+        agent_schema = load_agent_schema("resource-affinity-assessor")
 
         affinity_agent_runtime = await create_agent(
             agent_schema_override=agent_schema,

rem/services/dreaming/moment_service.py

@@ -8,13 +8,12 @@ with temporal boundaries and metadata.
 
 import json
 from datetime import datetime, timedelta
-from pathlib import Path
 from typing import Any, Optional
 from uuid import uuid4
 
-import yaml
 from loguru import logger
 
+from ...utils.schema_loader import load_agent_schema
 from ...agentic.providers.pydantic_ai import create_agent
 from ...agentic.serialization import serialize_agent_result
 from ...models.entities.moment import Moment, Person
@@ -101,19 +100,7 @@ async def construct_moments(
         }
 
     # Load MomentBuilder agent schema
-    schema_path = (
-        Path(__file__).parent.parent.parent
-        / "schemas"
-        / "agents"
-        / "core"
-        / "moment-builder.yaml"
-    )
-
-    if not schema_path.exists():
-        raise FileNotFoundError(f"MomentBuilder schema not found: {schema_path}")
-
-    with open(schema_path) as f:
-        agent_schema = yaml.safe_load(f)
+    agent_schema = load_agent_schema("moment-builder")
 
     # Prepare input data for agent
     input_data = {

rem/services/embeddings/api.py

@@ -5,13 +5,20 @@ Provides synchronous and async wrappers for embedding generation using
 raw HTTP requests (no OpenAI SDK dependency).
 """
 
-import os
 from typing import Optional, cast
 
 import httpx
 import requests
 from loguru import logger
 
+from rem.utils.constants import DEFAULT_EMBEDDING_DIMS, HTTP_TIMEOUT_DEFAULT
+
+
+def _get_openai_api_key() -> Optional[str]:
+    """Get OpenAI API key from settings."""
+    from rem.settings import settings
+    return settings.llm.openai_api_key
+
 
 def generate_embedding(
     text: str,
@@ -26,16 +33,16 @@ def generate_embedding(
         text: Text to embed
         model: Model name (default: text-embedding-3-small)
         provider: Provider name (default: openai)
-        api_key: API key (defaults to OPENAI_API_KEY env var)
+        api_key: API key (defaults to settings.llm.openai_api_key)
 
     Returns:
         Embedding vector (1536 dimensions for text-embedding-3-small)
     """
     if provider == "openai":
-        api_key = api_key or os.getenv("OPENAI_API_KEY")
+        api_key = api_key or _get_openai_api_key()
         if not api_key:
             logger.warning("No OpenAI API key - returning zero vector")
-            return [0.0] * 1536
+            return [0.0] * DEFAULT_EMBEDDING_DIMS
 
         try:
             logger.info(f"Generating OpenAI embedding for text using {model}")
@@ -47,7 +54,7 @@ def generate_embedding(
                     "Content-Type": "application/json",
                 },
                 json={"input": [text], "model": model},
-                timeout=30,
+                timeout=HTTP_TIMEOUT_DEFAULT,
             )
             response.raise_for_status()
 
@@ -58,11 +65,11 @@ def generate_embedding(
 
         except Exception as e:
             logger.error(f"Failed to generate embedding from OpenAI: {e}", exc_info=True)
-            return [0.0] * 1536
+            return [0.0] * DEFAULT_EMBEDDING_DIMS
 
     else:
         logger.warning(f"Unsupported provider '{provider}' - returning zero vector")
-        return [0.0] * 1536
+        return [0.0] * DEFAULT_EMBEDDING_DIMS
 
 
 async def generate_embedding_async(
@@ -78,16 +85,16 @@ async def generate_embedding_async(
         text: Text to embed
         model: Model name (default: text-embedding-3-small)
         provider: Provider name (default: openai)
-        api_key: API key (defaults to OPENAI_API_KEY env var)
+        api_key: API key (defaults to settings.llm.openai_api_key)
 
     Returns:
         Embedding vector (1536 dimensions for text-embedding-3-small)
     """
     if provider == "openai":
-        api_key = api_key or os.getenv("OPENAI_API_KEY")
+        api_key = api_key or _get_openai_api_key()
         if not api_key:
             logger.warning("No OpenAI API key - returning zero vector")
-            return [0.0] * 1536
+            return [0.0] * DEFAULT_EMBEDDING_DIMS
 
         try:
             logger.info(f"Generating OpenAI embedding for text using {model}")
@@ -100,7 +107,7 @@ async def generate_embedding_async(
                         "Content-Type": "application/json",
                     },
                     json={"input": [text], "model": model},
-                    timeout=30.0,
+                    timeout=HTTP_TIMEOUT_DEFAULT,
                 )
                 response.raise_for_status()
 
@@ -113,8 +120,8 @@ async def generate_embedding_async(
 
         except Exception as e:
             logger.error(f"Failed to generate embedding from OpenAI: {e}", exc_info=True)
-            return [0.0] * 1536
+            return [0.0] * DEFAULT_EMBEDDING_DIMS
 
     else:
         logger.warning(f"Unsupported provider '{provider}' - returning zero vector")
-        return [0.0] * 1536
+        return [0.0] * DEFAULT_EMBEDDING_DIMS

rem/services/phoenix/EXPERIMENT_DESIGN.md

@@ -164,7 +164,7 @@ cp curated-queries.csv experiments/rem-001/validation/production/
 **Option C: Curated Engrams**
 ```bash
 # Generate engrams from REM data
-rem dreaming full --user-id test-user --tenant-id acme --generate-test-cases
+rem dreaming full --user-id test-user  --generate-test-cases
 
 # Review and select high-quality engrams
 rem engram list --quality high --limit 100 --output engrams.csv
@@ -357,7 +357,7 @@ Level 4 (Mature):   Multiple cycles, full query capabilities
 # Generate engrams from REM data
 rem dreaming full \
   --user-id test-user \
-  --tenant-id acme \
+   \
   --generate-test-cases \
   --quality-level 3
 
@@ -1027,7 +1027,7 @@ rem experiments experiment run rem-lookup-ask_rem-golden \
 
 ```bash
 # 1. Generate high-quality engrams
-rem dreaming full --tenant-id acme --generate-test-cases --quality-level 4
+rem dreaming full  --generate-test-cases --quality-level 4
 
 # 2. Export engrams
 rem engram export rem-engrams-mature-mixed --output engrams.csv --format phoenix