remdb 0.3.7__py3-none-any.whl

rem/workers/engram_processor.py

@@ -0,0 +1,312 @@
+"""
+Engram Processor for REM.
+
+Processes engram YAML/JSON files into Resources and Moments following
+the p8fs-modules engram specification.
+
+Key Design Principles:
+- Engrams ARE Resources (category="engram")
+- Human-friendly labels in graph edges (not UUIDs)
+- Upsert with JSON merge behavior (never overwrite)
+- Dual indexing handled by repository (SQL + embeddings + KV)
+- Moment attachment via part_of relationship
+
+Processing Flow:
+1. Parse YAML/JSON engram
+2. Create Resource from top-level engram fields
+3. Upsert Resource (triggers embeddings + KV store population)
+4. Create Moments from moments array
+5. Link moments to parent engram via graph edges
+
+See: /Users/sirsh/code/p8fs-modules/p8fs/docs/04 engram-specification.md
+"""
+
+"""
+Engram Processor for REM.
+
+Processes engram YAML/JSON files into Resources and Moments following
+the p8fs-modules engram specification.
+
+Key Design Principles:
+- Engrams ARE Resources (category="engram")
+- Human-friendly labels in graph edges (not UUIDs)
+- Upsert with JSON merge behavior (never overwrite)
+- Dual indexing handled by repository (SQL + embeddings + KV)
+- Moment attachment via part_of relationship
+
+Processing Flow:
+1. Parse YAML/JSON engram
+2. Create Resource from top-level engram fields
+3. Upsert Resource (triggers embeddings + KV store population)
+4. Create Moments from moments array
+5. Link moments to parent engram via graph edges
+
+See: /Users/sirsh/code/p8fs-modules/p8fs/docs/04 engram-specification.md
+"""
+
+import logging
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Optional, cast # Added cast
+
+import yaml
+
+from rem.models.core import InlineEdge
+from rem.models.entities import Moment, Resource
+from rem.services.postgres import PostgresService
+
+logger = logging.getLogger(__name__)
+
+
+class EngramProcessor:
+    """
+    Process engram files into REM Resources and Moments.
+
+    Example usage:
+        processor = EngramProcessor(postgres_service)
+        result = await processor.process_file(
+            file_path="/path/to/engram.yaml",
+            tenant_id="acme-corp",
+            user_id="sarah-chen"
+        )
+    """
+
+    def __init__(self, postgres_service: PostgresService):
+        """
+        Initialize engram processor.
+
+        Args:
+            postgres_service: PostgreSQL service for upsert operations
+        """
+        self.postgres = postgres_service
+
+    async def process_file(
+        self,
+        file_path: Path | str,
+        tenant_id: str,
+        user_id: Optional[str] = None,
+    ) -> dict[str, Any]:
+        """
+        Process an engram file (YAML or JSON).
+
+        Args:
+            file_path: Path to engram file
+            tenant_id: Tenant ID for multi-tenancy
+            user_id: Optional user ID (owner)
+
+        Returns:
+            Result dict with resource_id, moment_ids, chunks_created, etc.
+        """
+        file_path = Path(file_path)
+
+        # Parse file
+        with open(file_path) as f:
+            if file_path.suffix in (".yaml", ".yml"):
+                data = yaml.safe_load(f)
+            elif file_path.suffix == ".json":
+                import json
+
+                data = json.load(f)
+            else:
+                raise ValueError(f"Unsupported file format: {file_path.suffix}")
+
+        return await self.process_engram(data, tenant_id, user_id)
+
+    async def process_engram(
+        self,
+        data: dict[str, Any],
+        tenant_id: str,
+        user_id: Optional[str] = None,
+    ) -> dict[str, Any]:
+        """
+        Process engram data into REM entities.
+
+        Args:
+            data: Parsed engram data (dict from YAML/JSON)
+            tenant_id: Tenant ID
+            user_id: Optional user ID
+
+        Returns:
+            Result dict with resource_id, moment_ids, chunks_created, etc.
+        """
+        # Validate kind
+        if data.get("kind") != "engram":
+            raise ValueError(f"Expected kind='engram', got: {data.get('kind')}")
+
+        # Extract top-level engram fields
+        name = data["name"]
+        category = data.get("category", "engram")
+        summary = data.get("summary")
+        content = data.get("content", "")
+        uri = data.get("uri")
+        resource_timestamp = data.get("resource_timestamp")
+        if resource_timestamp:
+            resource_timestamp = datetime.fromisoformat(resource_timestamp)
+        else:
+            resource_timestamp = datetime.utcnow()
+
+        metadata = data.get("metadata", {})
+        graph_edges_data = data.get("graph_edges", [])
+        moments_data = data.get("moments", [])
+
+        # Convert graph edges to InlineEdge objects
+        graph_edges = []
+        for edge_data in graph_edges_data:
+            edge = InlineEdge(
+                dst=edge_data["dst"],
+                rel_type=edge_data["rel_type"],
+                weight=edge_data.get("weight", 0.5),
+                properties=edge_data.get("properties", {}),
+            )
+            graph_edges.append(edge)
+
+        # Create Resource (engram)
+        resource = Resource(
+            tenant_id=tenant_id,
+            user_id=user_id,
+            name=name,
+            category=category,
+            uri=uri,
+            content=content,
+            timestamp=resource_timestamp,
+            metadata=metadata,
+            graph_edges=[edge.model_dump() for edge in graph_edges],
+        )
+
+        # Upsert resource (triggers embeddings + KV store population)
+        logger.info(f"Upserting engram resource: {name}")
+        upsert_result = await self.postgres.batch_upsert(
+            records=[resource.model_dump(mode="json")],
+            model=Resource,
+            table_name="resources",
+            entity_key_field="name",  # Explicit entity_key for KV store
+        )
+        resource_id = upsert_result["ids"][0]
+        logger.info(f"Upserted resource: {resource_id}")
+
+        # Process attached moments
+        moment_ids = []
+        if moments_data:
+            logger.info(f"Processing {len(moments_data)} moments for engram: {name}")
+            moment_ids = await self._process_moments(
+                moments_data=moments_data,
+                parent_resource_name=name,
+                parent_resource_id=resource_id,
+                tenant_id=tenant_id,
+                user_id=user_id,
+            )
+
+        return {
+            "resource_id": resource_id,
+            "moment_ids": moment_ids,
+            "chunks_created": 1 + len(moment_ids),  # Resource + moments
+            "embeddings_generated": 0,  # Handled by embedding worker
+        }
+
+    async def _process_moments(
+        self,
+        moments_data: list[dict],
+        parent_resource_name: str,
+        parent_resource_id: str,
+        tenant_id: str,
+        user_id: Optional[str] = None,
+    ) -> list[str]:
+        """
+        Create Moments from moments array in engram.
+
+        Args:
+            moments_data: List of moment dicts from engram
+            parent_resource_name: Parent engram name
+            parent_resource_id: Parent engram resource ID
+            tenant_id: Tenant ID
+            user_id: Optional user ID
+
+        Returns:
+            List of moment IDs
+        """
+        moments = []
+        for moment_data in moments_data:
+            # Extract moment fields
+            name = moment_data["name"]
+            content = moment_data["content"]
+            summary = moment_data.get("summary")
+            moment_type = moment_data.get("moment_type")
+            category = moment_data.get("category")
+            uri = moment_data.get("uri")
+
+            # Parse timestamps
+            starts_timestamp = moment_data.get("starts_timestamp") or moment_data.get("resource_timestamp")
+            if starts_timestamp:
+                if isinstance(starts_timestamp, str):
+                    starts_timestamp = datetime.fromisoformat(starts_timestamp)
+            else:
+                starts_timestamp = datetime.utcnow()
+
+            ends_timestamp = moment_data.get("ends_timestamp") or moment_data.get("resource_ends_timestamp")
+            if ends_timestamp and isinstance(ends_timestamp, str):
+                ends_timestamp = datetime.fromisoformat(ends_timestamp)
+
+            emotion_tags = moment_data.get("emotion_tags", [])
+            topic_tags = moment_data.get("topic_tags", [])
+            present_persons_data = moment_data.get("present_persons", [])
+            metadata = moment_data.get("metadata", {})
+            graph_edges_data = moment_data.get("graph_edges", [])
+
+            # Create link to parent engram
+            parent_edge = InlineEdge(
+                dst=parent_resource_name,
+                rel_type="part_of",
+                weight=1.0,
+                properties={
+                    "dst_name": parent_resource_name,
+                    "dst_id": parent_resource_id,
+                    "dst_entity_type": "resource/engram",
+                    "match_type": "parent_child",
+                    "confidence": 1.0,
+                },
+            )
+
+            # Combine moment edges with parent edge
+            all_edges = [parent_edge]
+            for edge_data in graph_edges_data:
+                edge = InlineEdge(
+                    dst=edge_data["dst"],
+                    rel_type=edge_data["rel_type"],
+                    weight=edge_data.get("weight", 0.5),
+                    properties=edge_data.get("properties", {}),
+                )
+                all_edges.append(edge)
+
+            # Create Moment entity
+            moment = Moment(
+                tenant_id=tenant_id,
+                user_id=user_id,
+                name=name,
+                moment_type=moment_type,
+                category=category,
+                starts_timestamp=starts_timestamp,
+                ends_timestamp=ends_timestamp,
+                present_persons=present_persons_data,  # Will be validated by Pydantic
+                emotion_tags=emotion_tags,
+                topic_tags=topic_tags,
+                summary=summary or content[:200],  # Fallback to content prefix
+                source_resource_ids=[parent_resource_id],
+                metadata=metadata,
+                graph_edges=[edge.model_dump() for edge in all_edges],
+            )
+            moments.append(moment)
+
+        # Batch upsert all moments
+        if moments:
+            logger.info(f"Batch upserting {len(moments)} moments")
+            upsert_result = await self.postgres.batch_upsert(
+                records=[m.model_dump(mode="json") for m in moments],
+                model=Moment,
+                table_name="moments",
+                entity_key_field="name",  # Explicit entity_key for KV store
+            )
+            moment_ids = upsert_result["ids"]
+            logger.info(f"Upserted {len(moment_ids)} moments")
+            return moment_ids
+
+        return []

rem/workers/sqs_file_processor.py

@@ -0,0 +1,193 @@
+"""
+SQS File Processor Worker.
+
+Consumes S3 ObjectCreated events from SQS queue and processes files.
+Designed to run as a K8s deployment scaled by KEDA based on queue depth.
+"""
+
+import json
+import signal
+import sys
+from typing import Any
+
+import boto3
+from botocore.exceptions import ClientError
+from loguru import logger
+
+from rem.services.content import ContentService
+from rem.settings import settings
+
+
+class SQSFileProcessor:
+    """
+    SQS-based file processor worker.
+
+    Polls SQS queue for S3 ObjectCreated events and processes files
+    using ContentService.
+
+    Gracefully handles:
+    - SIGTERM (K8s pod termination)
+    - SIGINT (Ctrl+C for local testing)
+    - SQS visibility timeout (message redelivery)
+    - DLQ (failed messages after retries)
+    """
+
+    def __init__(self):
+        self.sqs_client = self._create_sqs_client()
+        self.content_service = ContentService()
+        self.running = True
+        self.processing_message = False
+
+        # Register signal handlers
+        signal.signal(signal.SIGTERM, self._handle_shutdown)
+        signal.signal(signal.SIGINT, self._handle_shutdown)
+
+    def _create_sqs_client(self):
+        """Create SQS client with IRSA or configured credentials."""
+        return boto3.client("sqs", region_name=settings.sqs.region)
+
+    def _handle_shutdown(self, signum, frame):
+        """Handle graceful shutdown signals."""
+        logger.info(f"Received shutdown signal ({signum}), finishing current message...")
+        self.running = False
+
+    def run(self):
+        """
+        Main worker loop.
+
+        Long polls SQS queue, processes messages, deletes on success.
+        Exits gracefully on SIGTERM/SIGINT after completing current message.
+        """
+        if not settings.sqs.queue_url:
+            logger.error("SQS_QUEUE_URL not configured")
+            sys.exit(1)
+
+        logger.info(f"Starting file processor worker")
+        logger.info(f"Queue: {settings.sqs.queue_url}")
+        logger.info(f"Polling interval: {settings.sqs.wait_time_seconds}s (long polling)")
+
+        while self.running:
+            try:
+                # Long poll for messages
+                response = self.sqs_client.receive_message(
+                    QueueUrl=settings.sqs.queue_url,
+                    MaxNumberOfMessages=settings.sqs.max_messages,
+                    WaitTimeSeconds=settings.sqs.wait_time_seconds,
+                    AttributeNames=["All"],
+                    MessageAttributeNames=["All"],
+                )
+
+                messages = response.get("Messages", [])
+                if not messages:
+                    continue
+
+                logger.info(f"Received {len(messages)} message(s)")
+
+                # Process each message
+                for message in messages:
+                    if not self.running:
+                        logger.info("Shutdown requested, stopping message processing")
+                        break
+
+                    self.processing_message = True
+                    try:
+                        self._process_message(message)
+                        self._delete_message(message)
+                    except Exception as e:
+                        logger.error(f"Failed to process message: {e}", exc_info=True)
+                        # Message will be redelivered after visibility timeout
+                    finally:
+                        self.processing_message = False
+
+            except KeyboardInterrupt:
+                logger.info("Keyboard interrupt, shutting down...")
+                break
+            except Exception as e:
+                logger.error(f"Error in worker loop: {e}", exc_info=True)
+
+        logger.info("Worker stopped")
+
+    def _process_message(self, message: dict):
+        """
+        Process a single SQS message containing S3 event(s).
+
+        S3 notification format:
+        {
+          "Records": [{
+            "eventName": "ObjectCreated:Put",
+            "s3": {
+              "bucket": {"name": "rem"},
+              "object": {"key": "uploads/file.md", "size": 12345}
+            }
+          }]
+        }
+        """
+        body = json.loads(message["Body"])
+
+        for record in body.get("Records", []):
+            event_name = record.get("eventName", "")
+
+            if not event_name.startswith("ObjectCreated:"):
+                logger.debug(f"Skipping non-create event: {event_name}")
+                continue
+
+            bucket = record["s3"]["bucket"]["name"]
+            key = record["s3"]["object"]["key"]
+            size = record["s3"]["object"].get("size", 0)
+
+            logger.info(f"Processing {event_name}: s3://{bucket}/{key} ({size} bytes)")
+
+            try:
+                # Process file using ContentService
+                uri = f"s3://{bucket}/{key}"
+                result = self.content_service.process_uri(uri)
+
+                # TODO: Store in PostgreSQL with pgvector
+                # TODO: Generate embeddings
+                # TODO: Create graph edges
+                # TODO: Update REM entities
+
+                logger.info(
+                    f"Successfully processed s3://{bucket}/{key} "
+                    f"({len(result['content'])} chars, provider={result['provider']})"
+                )
+
+                # Log extraction metadata
+                logger.debug(f"Metadata: {json.dumps(result['metadata'], default=str)}")
+
+            except FileNotFoundError as e:
+                logger.error(f"File not found: {e}")
+                # Don't retry - file is gone
+                raise
+            except RuntimeError as e:
+                logger.error(f"Processing error: {e}")
+                # Retry on next visibility timeout
+                raise
+            except Exception as e:
+                logger.exception(f"Unexpected error processing s3://{bucket}/{key}: {e}")
+                raise
+
+    def _delete_message(self, message: dict):
+        """Delete message from queue after successful processing."""
+        try:
+            self.sqs_client.delete_message(
+                QueueUrl=settings.sqs.queue_url,
+                ReceiptHandle=message["ReceiptHandle"],
+            )
+            logger.debug("Message deleted from queue")
+        except ClientError as e:
+            logger.error(f"Failed to delete message: {e}")
+            # Message will be redelivered, but processing was successful
+
+
+def main():
+    """Entry point for containerized deployment."""
+    logger.info("REM File Processor Worker")
+    logger.info(f"Environment: {settings.environment}")
+
+    processor = SQSFileProcessor()
+    processor.run()
+
+
+if __name__ == "__main__":
+    main()