remdb 0.3.0__py3-none-any.whl → 0.3.114__py3-none-any.whl

rem/services/content/providers.py

@@ -2,17 +2,27 @@
 
 import json
 import multiprocessing
-import os
 import random
 import subprocess
 import sys
-import tempfile
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, Optional
 
 from loguru import logger
 
+from rem.utils.constants import (
+    AUDIO_CHUNK_TARGET_SECONDS,
+    AUDIO_CHUNK_WINDOW_SECONDS,
+    MIN_SILENCE_MS,
+    SILENCE_THRESHOLD_DB,
+    SUBPROCESS_TIMEOUT_SECONDS,
+    WAV_HEADER_MIN_BYTES,
+    WHISPER_COST_PER_MINUTE,
+)
+from rem.utils.files import temp_file_from_bytes
+from rem.utils.mime_types import get_extension
+
 
 class ContentProvider(ABC):
     """Base class for content extraction providers."""
@@ -132,7 +142,7 @@ import sys
 from pathlib import Path
 from kreuzberg import ExtractionConfig, extract_file_sync
 
-# Parse document with table extraction
+# Parse document with kreuzberg 3.x
 config = ExtractionConfig(
     extract_tables=True,
     chunk_content=False,
@@ -144,13 +154,7 @@ result = extract_file_sync(Path(sys.argv[1]), config=config)
 # Serialize result to JSON
 output = {
     'content': result.content,
-    'tables': [
-        {
-            'page_number': t.get('page_number', 0),
-            'text': t.get('text', ''),
-        }
-        for t in result.tables
-    ],
+    'tables': [t.model_dump() for t in result.tables] if result.tables else [],
     'metadata': result.metadata
 }
 print(json.dumps(output))
@@ -161,7 +165,7 @@ print(json.dumps(output))
             [sys.executable, "-c", script, str(file_path)],
             capture_output=True,
             text=True,
-            timeout=300,  # 5 minute timeout
+            timeout=SUBPROCESS_TIMEOUT_SECONDS,
         )
 
         if result.returncode != 0:
@@ -183,21 +187,9 @@ print(json.dumps(output))
         # Write bytes to temp file for kreuzberg
         # Detect extension from metadata
         content_type = metadata.get("content_type", "")
-        extension_map = {
-            "application/pdf": ".pdf",
-            "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
-            "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
-            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
-            "image/png": ".png",
-            "image/jpeg": ".jpg",
-        }
-        suffix = extension_map.get(content_type, ".pdf")  # Default to PDF
-
-        with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
-            tmp.write(content)
-            tmp_path = Path(tmp.name)
+        suffix = get_extension(content_type, default=".pdf")
 
-        try:
+        with temp_file_from_bytes(content, suffix=suffix) as tmp_path:
             # Check if running in daemon process
             if self._is_daemon_process():
                 logger.info("Daemon process detected - using subprocess workaround for document parsing")
@@ -211,7 +203,7 @@ print(json.dumps(output))
                     }
                 except Exception as e:
                     logger.error(f"Subprocess parsing failed: {e}. Falling back to text-only.")
-                    # Fallback to simple text extraction
+                    # Fallback to simple text extraction (kreuzberg 3.x API)
                     from kreuzberg import ExtractionConfig, extract_file_sync
                     config = ExtractionConfig(extract_tables=False)
                     result = extract_file_sync(tmp_path, config=config)
@@ -221,17 +213,17 @@ print(json.dumps(output))
                         "file_extension": tmp_path.suffix,
                     }
             else:
-                # Normal execution (not in daemon)
+                # Normal execution (not in daemon) - kreuzberg 4.x with native ONNX/Rust
                 from kreuzberg import ExtractionConfig, extract_file_sync
                 config = ExtractionConfig(
-                    extract_tables=True,
-                    chunk_content=False,
-                    extract_keywords=False,
+                    enable_quality_processing=True,  # Enables table extraction with native ONNX
+                    chunk_content=False,  # We handle chunking ourselves
+                    extract_tables=False,  # Disable table extraction to avoid PyTorch dependency
                 )
                 result = extract_file_sync(tmp_path, config=config)
                 text = result.content
                 extraction_metadata = {
-                    "table_count": len(result.tables),
+                    "table_count": len(result.tables) if result.tables else 0,
                     "parser": "kreuzberg",
                     "file_extension": tmp_path.suffix,
                 }
@@ -241,10 +233,6 @@ print(json.dumps(output))
                 "metadata": extraction_metadata,
             }
 
-        finally:
-            # Clean up temp file
-            tmp_path.unlink(missing_ok=True)
-
 
 class AudioProvider(ContentProvider):
     """
@@ -289,19 +277,20 @@ class AudioProvider(ContentProvider):
             ValueError: If OpenAI API key missing
         """
         # Handle empty or invalid content
-        if not content or len(content) < 44:  # WAV header is minimum 44 bytes
+        if not content or len(content) < WAV_HEADER_MIN_BYTES:
             logger.warning("Audio content too small to be valid WAV file")
             return {
                 "text": "[Invalid or empty audio file]",
                 "metadata": {"error": "invalid_content", "size": len(content)},
             }
 
-        # Check for OpenAI API key
-        api_key = os.getenv("OPENAI_API_KEY")
+        # Check for OpenAI API key (use settings)
+        from rem.settings import settings
+        api_key = settings.llm.openai_api_key
         if not api_key:
-            logger.warning("No OPENAI_API_KEY found - audio transcription disabled")
+            logger.warning("No OpenAI API key found - audio transcription disabled")
             return {
-                "text": "[Audio transcription requires OPENAI_API_KEY environment variable]",
+                "text": "[Audio transcription requires LLM__OPENAI_API_KEY to be set]",
                 "metadata": {"error": "missing_api_key"},
             }
 
@@ -318,83 +307,74 @@ class AudioProvider(ContentProvider):
         # Write bytes to temp file
         # Detect extension from metadata or use .wav as fallback
         content_type = metadata.get("content_type", "audio/wav")
-        extension_map = {
-            "audio/wav": ".wav",
-            "audio/mpeg": ".mp3",
-            "audio/mp4": ".m4a",
-            "audio/x-m4a": ".m4a",
-            "audio/flac": ".flac",
-            "audio/ogg": ".ogg",
-        }
-        extension = extension_map.get(content_type, ".wav")
+        extension = get_extension(content_type, default=".wav")
 
-        with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as tmp:
-            tmp.write(content)
-            tmp_path = Path(tmp.name)
+        chunker = None
+        chunks = None
 
-        try:
-            logger.info(f"Processing audio file: {tmp_path.name} ({len(content) / 1024 / 1024:.1f} MB)")
-
-            # Step 1: Chunk audio by silence
-            chunker = AudioChunker(
-                target_chunk_seconds=60.0,
-                chunk_window_seconds=2.0,
-                silence_threshold_db=-40.0,
-                min_silence_ms=500,
-            )
-
-            chunks = chunker.chunk_audio(tmp_path)
-            logger.info(f"Created {len(chunks)} audio chunks")
-
-            # Step 2: Transcribe chunks
-            transcriber = AudioTranscriber(api_key=api_key)
-            results = transcriber.transcribe_chunks(chunks)
-            logger.info(f"Transcribed {len(results)} chunks")
-
-            # Step 3: Combine into markdown format
-            # Format: Each chunk becomes a section with timestamp
-            markdown_parts = []
-            for result in results:
-                timestamp = f"{result.start_seconds:.1f}s - {result.end_seconds:.1f}s"
-                markdown_parts.append(f"## [{timestamp}]\n\n{result.text}\n")
-
-            markdown_text = "\n".join(markdown_parts)
-
-            # Calculate metadata
-            total_duration = sum(r.duration_seconds for r in results)
-            estimated_cost = (total_duration / 60) * 0.006  # $0.006 per minute
-            successful_chunks = sum(1 for r in results if r.confidence > 0)
-
-            extraction_metadata = {
-                "chunk_count": len(chunks),
-                "transcribed_chunks": successful_chunks,
-                "duration_seconds": total_duration,
-                "estimated_cost": estimated_cost,
-                "parser": "whisper_api",
-            }
+        with temp_file_from_bytes(content, suffix=extension) as tmp_path:
+            try:
+                logger.info(f"Processing audio file: {tmp_path.name} ({len(content) / 1024 / 1024:.1f} MB)")
+
+                # Step 1: Chunk audio by silence
+                chunker = AudioChunker(
+                    target_chunk_seconds=AUDIO_CHUNK_TARGET_SECONDS,
+                    chunk_window_seconds=AUDIO_CHUNK_WINDOW_SECONDS,
+                    silence_threshold_db=SILENCE_THRESHOLD_DB,
+                    min_silence_ms=MIN_SILENCE_MS,
+                )
 
-            logger.info(
-                f"Transcription complete: {successful_chunks}/{len(chunks)} chunks, "
-                f"${estimated_cost:.3f} cost"
-            )
+                chunks = chunker.chunk_audio(tmp_path)
+                logger.info(f"Created {len(chunks)} audio chunks")
 
-            return {
-                "text": markdown_text,
-                "metadata": extraction_metadata,
-            }
+                # Step 2: Transcribe chunks
+                transcriber = AudioTranscriber(api_key=api_key)
+                results = transcriber.transcribe_chunks(chunks)
+                logger.info(f"Transcribed {len(results)} chunks")
 
-        except Exception as e:
-            logger.error(f"Audio extraction failed: {e}")
-            raise RuntimeError(f"Audio transcription failed: {e}") from e
+                # Step 3: Combine into markdown format
+                # Format: Each chunk becomes a section with timestamp
+                markdown_parts = []
+                for result in results:
+                    timestamp = f"{result.start_seconds:.1f}s - {result.end_seconds:.1f}s"
+                    markdown_parts.append(f"## [{timestamp}]\n\n{result.text}\n")
+
+                markdown_text = "\n".join(markdown_parts)
+
+                # Calculate metadata
+                total_duration = sum(r.duration_seconds for r in results)
+                estimated_cost = (total_duration / 60) * WHISPER_COST_PER_MINUTE
+                successful_chunks = sum(1 for r in results if r.confidence > 0)
+
+                extraction_metadata = {
+                    "chunk_count": len(chunks),
+                    "transcribed_chunks": successful_chunks,
+                    "duration_seconds": total_duration,
+                    "estimated_cost": estimated_cost,
+                    "parser": "whisper_api",
+                }
+
+                logger.info(
+                    f"Transcription complete: {successful_chunks}/{len(chunks)} chunks, "
+                    f"${estimated_cost:.3f} cost"
+                )
+
+                return {
+                    "text": markdown_text,
+                    "metadata": extraction_metadata,
+                }
 
-        finally:
-            # Clean up temp file and chunks
-            try:
-                tmp_path.unlink(missing_ok=True)
-                if 'chunker' in locals() and 'chunks' in locals():
-                    chunker.cleanup_chunks(chunks)
             except Exception as e:
-                logger.warning(f"Cleanup failed: {e}")
+                logger.error(f"Audio extraction failed: {e}")
+                raise RuntimeError(f"Audio transcription failed: {e}") from e
+
+            finally:
+                # Clean up audio chunks (temp file cleanup handled by context manager)
+                if chunker is not None and chunks is not None:
+                    try:
+                        chunker.cleanup_chunks(chunks)
+                    except Exception as e:
+                        logger.warning(f"Chunk cleanup failed: {e}")
 
 
 class SchemaProvider(ContentProvider):
@@ -672,19 +652,9 @@ class ImageProvider(ContentProvider):
 
                 # Write bytes to temp file for analysis
                 content_type = metadata.get("content_type", "image/png")
-                extension_map = {
-                    "image/png": ".png",
-                    "image/jpeg": ".jpg",
-                    "image/gif": ".gif",
-                    "image/webp": ".webp",
-                }
-                extension = extension_map.get(content_type, ".png")
-
-                with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as tmp:
-                    tmp.write(content)
-                    tmp_path = Path(tmp.name)
+                extension = get_extension(content_type, default=".png")
 
-                try:
+                with temp_file_from_bytes(content, suffix=extension) as tmp_path:
                     # Analyze image
                     result = analyzer.analyze_image(tmp_path)
                     vision_description = result.description
@@ -692,9 +662,6 @@ class ImageProvider(ContentProvider):
                     vision_model = result.model
 
                     logger.info(f"Vision analysis complete: {len(vision_description)} chars")
-                finally:
-                    # Clean up temp file
-                    tmp_path.unlink(missing_ok=True)
 
             except ImportError as e:
                 logger.warning(f"Vision analysis not available: {e}")
@@ -737,19 +704,9 @@ class ImageProvider(ContentProvider):
                 if embedder.is_available():
                     # Write bytes to temp file for CLIP embedding
                     content_type = metadata.get("content_type", "image/png")
-                    extension_map = {
-                        "image/png": ".png",
-                        "image/jpeg": ".jpg",
-                        "image/gif": ".gif",
-                        "image/webp": ".webp",
-                    }
-                    extension = extension_map.get(content_type, ".png")
+                    extension = get_extension(content_type, default=".png")
 
-                    with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as tmp:
-                        tmp.write(content)
-                        tmp_path = Path(tmp.name)
-
-                    try:
+                    with temp_file_from_bytes(content, suffix=extension) as tmp_path:
                         # Generate CLIP embedding
                         result = embedder.embed_image(tmp_path)
                         if result:
@@ -759,9 +716,6 @@ class ImageProvider(ContentProvider):
                             logger.info(
                                 f"CLIP embedding generated: {clip_dimensions} dims, {clip_tokens} tokens"
                             )
-                    finally:
-                        # Clean up temp file
-                        tmp_path.unlink(missing_ok=True)
                 else:
                     logger.debug(
                         "CLIP embeddings disabled - set CONTENT__JINA_API_KEY to enable. "

rem/services/content/service.py

@@ -278,6 +278,7 @@ class ContentService:
         category: str | None = None,
         tags: list[str] | None = None,
         is_local_server: bool = False,
+        resource_type: str | None = None,
     ) -> dict[str, Any]:
         """
         Complete file ingestion pipeline: read → store → parse → chunk → embed.
@@ -322,6 +323,9 @@ class ContentService:
             category: Optional category tag (document, code, audio, etc.)
             tags: Optional list of tags
             is_local_server: True if running as local/stdio MCP server
+            resource_type: Optional resource type (case-insensitive). Supports:
+                - "resource", "resources", "Resource" → Resource (default)
+                - "domain-resource", "domain_resource", "DomainResource" → DomainResource
 
         Returns:
             dict with:
@@ -366,11 +370,32 @@ class ContentService:
         file_size = len(file_content)
         logger.info(f"Read {file_size} bytes from {file_uri} (source: {source_type})")
 
-        # Step 2: Write to internal storage (user-scoped)
+        # Step 1.5: Early schema detection for YAML/JSON files
+        # Skip File entity creation for schemas (agents/evaluators)
+        file_suffix = Path(file_name).suffix.lower()
+        if file_suffix in ['.yaml', '.yml', '.json']:
+            import yaml
+            import json
+            try:
+                content_text = file_content.decode('utf-8') if isinstance(file_content, bytes) else file_content
+                data = yaml.safe_load(content_text) if file_suffix in ['.yaml', '.yml'] else json.loads(content_text)
+                if isinstance(data, dict):
+                    json_schema_extra = data.get('json_schema_extra', {})
+                    kind = json_schema_extra.get('kind', '')
+                    if kind in ['agent', 'evaluator']:
+                        # Route directly to schema processing, skip File entity
+                        logger.info(f"Detected {kind} schema: {file_name}, routing to _process_schema")
+                        result = self.process_uri(file_uri)
+                        return await self._process_schema(result, file_uri, user_id)
+            except Exception as e:
+                logger.debug(f"Early schema detection failed for {file_name}: {e}")
+                # Fall through to standard file processing
+
+        # Step 2: Write to internal storage (public or user-scoped)
         file_id = str(uuid4())
         storage_uri, internal_key, content_type, _ = await fs_service.write_to_internal_storage(
             content=file_content,
-            tenant_id=user_id,  # Using user_id for storage scoping
+            tenant_id=user_id or "public",  # Storage path: public/ or user_id/
             file_name=file_name,
             file_id=file_id,
         )
@@ -379,7 +404,7 @@ class ContentService:
         # Step 3: Create File entity
         file_entity = File(
             id=file_id,
-            tenant_id=user_id,  # Set tenant_id to user_id (application scoped to user)
+            tenant_id=user_id,  # None = public/shared
             user_id=user_id,
             name=file_name,
             uri=storage_uri,
@@ -418,6 +443,7 @@ class ContentService:
             processing_result = await self.process_and_save(
                 uri=storage_uri,
                 user_id=user_id,
+                resource_type=resource_type,
             )
             processing_status = processing_result.get("status", "completed")
             resources_created = processing_result.get("chunk_count", 0)
@@ -459,7 +485,12 @@ class ContentService:
             "message": f"File ingested and {processing_status}. Created {resources_created} resources.",
         }
 
-    async def process_and_save(self, uri: str, user_id: str | None = None) -> dict[str, Any]:
+    async def process_and_save(
+        self,
+        uri: str,
+        user_id: str | None = None,
+        resource_type: str | None = None,
+    ) -> dict[str, Any]:
         """
         Process file end-to-end: extract → markdown → chunk → save.
 
@@ -474,6 +505,8 @@ class ContentService:
         Args:
             uri: File URI (s3://bucket/key or local path)
             user_id: Optional user ID for multi-tenancy
+            resource_type: Optional resource type (case-insensitive). Defaults to "Resource".
+                Supports: resource, domain-resource, domain_resource, DomainResource, etc.
 
         Returns:
             dict with file metadata and chunk count
@@ -526,7 +559,7 @@ class ContentService:
             size_bytes=result["metadata"].get("size"),
             mime_type=result["metadata"].get("content_type"),
             processing_status="completed",
-            tenant_id=user_id or "default",  # Required field
+            tenant_id=user_id,  # None = public/shared
             user_id=user_id,
         )
 
@@ -534,28 +567,66 @@ class ContentService:
             await self.file_repo.upsert(file)
             logger.info(f"Saved File: {filename}")
 
-        # Create Resource entities for each chunk
-        resources = [
-            Resource(
+        # Resolve resource model class from type parameter (case-insensitive)
+        from typing import cast, Type
+        from pydantic import BaseModel
+        from rem.utils.model_helpers import model_from_arbitrary_casing, get_table_name
+
+        resource_model: Type[BaseModel] = Resource  # Default
+        if resource_type:
+            try:
+                resource_model = model_from_arbitrary_casing(resource_type)
+                logger.info(f"Using resource model: {resource_model.__name__}")
+            except ValueError as e:
+                logger.warning(f"Invalid resource_type '{resource_type}', using default Resource: {e}")
+                resource_model = Resource
+
+        # Get table name for the resolved model
+        table_name = get_table_name(resource_model)
+
+        # Create resource entities for each chunk
+        resources: list[BaseModel] = [
+            resource_model(
                 name=f"{filename}#chunk-{i}",
                 uri=f"{uri}#chunk-{i}",
                 ordinal=i,
                 content=chunk,
                 category="document",
-                tenant_id=user_id or "default",  # Required field
+                tenant_id=user_id,  # None = public/shared
                 user_id=user_id,
             )
             for i, chunk in enumerate(chunks)
         ]
 
-        if self.resource_repo:
-            await self.resource_repo.upsert(
-                resources,
-                embeddable_fields=["content"],
-                generate_embeddings=True,
-            )
-            logger.info(f"Saved {len(resources)} Resource chunks")
-            logger.info(f"Queued {len(resources)} embedding generation tasks for content field")
+        # Save resources to the appropriate table
+        if resources:
+            from rem.services.postgres import get_postgres_service
+
+            postgres = get_postgres_service()
+            if postgres:
+                await postgres.connect()
+                try:
+                    await postgres.batch_upsert(
+                        records=cast(list[BaseModel | dict], resources),
+                        model=resource_model,
+                        table_name=table_name,
+                        entity_key_field="name",
+                        embeddable_fields=["content"],
+                        generate_embeddings=True,
+                    )
+                    logger.info(f"Saved {len(resources)} {resource_model.__name__} chunks to {table_name}")
+                    logger.info(f"Queued {len(resources)} embedding generation tasks for content field")
+                finally:
+                    await postgres.disconnect()
+            elif self.resource_repo:
+                # Fallback to injected repo (only works for default Resource)
+                await self.resource_repo.upsert(
+                    resources,
+                    embeddable_fields=["content"],
+                    generate_embeddings=True,
+                )
+                logger.info(f"Saved {len(resources)} Resource chunks")
+                logger.info(f"Queued {len(resources)} embedding generation tasks for content field")
 
         return {
             "file": file.model_dump(),
@@ -595,9 +666,10 @@ class ContentService:
         # IMPORTANT: category field distinguishes agents from evaluators
         # - kind=agent → category="agent" (AI agents with tools/resources)
         # - kind=evaluator → category="evaluator" (LLM-as-a-Judge evaluators)
+        # Schemas (agents/evaluators) default to system tenant for shared access
         schema_entity = Schema(
-            tenant_id=user_id or "default",
-            user_id=user_id,
+            tenant_id="system",
+            user_id=None,
             name=name,
             spec=schema_data,
             category=kind,  # Maps kind → category for database filtering
@@ -667,7 +739,7 @@ class ContentService:
             processor = EngramProcessor(postgres)
             result = await processor.process_engram(
                 data=data,
-                tenant_id=user_id or "default",
+                tenant_id=user_id,  # None = public/shared
                 user_id=user_id,
             )
             logger.info(f"✅ Engram processed: {result.get('resource_id')} with {len(result.get('moment_ids', []))} moments")

rem/services/dreaming/affinity_service.py

@@ -8,12 +8,11 @@ vector similarity (fast) or LLM analysis (intelligent).
 import json
 from datetime import datetime, timedelta
 from enum import Enum
-from pathlib import Path
 from typing import Any, Optional
 
-import yaml
 from loguru import logger
 
+from ...utils.schema_loader import load_agent_schema
 from ...agentic.providers.pydantic_ai import create_agent
 from ...agentic.serialization import serialize_agent_result
 from ...models.core import QueryType, RemQuery, SearchParameters
@@ -125,20 +124,7 @@ async def build_affinity(
     # Load LLM agent for relationship assessment if needed
     affinity_agent = None
     if mode == AffinityMode.LLM:
-        schema_path = (
-            Path(__file__).parent.parent.parent
-            / "schemas"
-            / "agents"
-            / "resource-affinity-assessor.yaml"
-        )
-
-        if not schema_path.exists():
-            raise FileNotFoundError(
-                f"ResourceAffinityAssessor schema not found: {schema_path}"
-            )
-
-        with open(schema_path) as f:
-            agent_schema = yaml.safe_load(f)
+        agent_schema = load_agent_schema("resource-affinity-assessor")
 
         affinity_agent_runtime = await create_agent(
             agent_schema_override=agent_schema,

rem/services/dreaming/moment_service.py

@@ -8,13 +8,12 @@ with temporal boundaries and metadata.
 
 import json
 from datetime import datetime, timedelta
-from pathlib import Path
 from typing import Any, Optional
 from uuid import uuid4
 
-import yaml
 from loguru import logger
 
+from ...utils.schema_loader import load_agent_schema
 from ...agentic.providers.pydantic_ai import create_agent
 from ...agentic.serialization import serialize_agent_result
 from ...models.entities.moment import Moment, Person
@@ -101,19 +100,7 @@ async def construct_moments(
         }
 
     # Load MomentBuilder agent schema
-    schema_path = (
-        Path(__file__).parent.parent.parent
-        / "schemas"
-        / "agents"
-        / "core"
-        / "moment-builder.yaml"
-    )
-
-    if not schema_path.exists():
-        raise FileNotFoundError(f"MomentBuilder schema not found: {schema_path}")
-
-    with open(schema_path) as f:
-        agent_schema = yaml.safe_load(f)
+    agent_schema = load_agent_schema("moment-builder")
 
     # Prepare input data for agent
     input_data = {