ciris-agent 1.7.7__py3-none-any.whl

ciris_engine/logic/adapters/api/api_communication.py

@@ -0,0 +1,370 @@
+"""
+Communication service for API adapter.
+"""
+
+import asyncio
+import logging
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from ciris_engine.logic.services.base_service import BaseService
+from ciris_engine.protocols.services.governance.communication import CommunicationServiceProtocol
+from ciris_engine.schemas.runtime.enums import ServiceType
+from ciris_engine.schemas.runtime.messages import FetchedMessage
+from ciris_engine.schemas.types import JSONDict
+
+if TYPE_CHECKING:
+    from ciris_engine.schemas.services.core import ServiceCapabilities, ServiceStatus
+
+logger = logging.getLogger(__name__)
+
+
+class APICommunicationService(BaseService, CommunicationServiceProtocol):
+    """Communication service for API responses."""
+
+    def __init__(self, config: Optional[Any] = None) -> None:
+        """Initialize API communication service."""
+        # Initialize BaseService for telemetry
+        super().__init__(time_service=None, service_name="APICommunicationService")
+
+        self._response_queue: asyncio.Queue[Any] = asyncio.Queue()
+        self._websocket_clients: JSONDict = {}
+        self._config = config  # Store the API adapter config
+
+        # Metrics tracking
+        self._response_times: List[float] = []  # Track last N response times
+        self._max_response_times = 100  # Keep last 100 response times
+        self._requests_handled = 0
+        self._error_count = 0
+        self._start_time = None
+        self._time_service = None
+
+    def _create_speak_correlation(self, channel_id: str, content: str) -> None:
+        """Create and store a speak correlation for outgoing message."""
+        import uuid
+
+        from ciris_engine.logic import persistence
+        from ciris_engine.schemas.telemetry.core import (
+            ServiceCorrelation,
+            ServiceCorrelationStatus,
+            ServiceRequestData,
+            ServiceResponseData,
+        )
+
+        correlation_id = str(uuid.uuid4())
+        now = datetime.now(timezone.utc)
+
+        correlation = ServiceCorrelation(
+            correlation_id=correlation_id,
+            service_type="api",
+            handler_name="APIAdapter",
+            action_type="speak",
+            request_data=ServiceRequestData(
+                service_type="api",
+                method_name="speak",
+                channel_id=channel_id,
+                parameters={"content": content, "channel_id": channel_id},
+                request_timestamp=now,
+            ),
+            response_data=ServiceResponseData(
+                success=True, result_summary="Message sent", execution_time_ms=0, response_timestamp=now
+            ),
+            status=ServiceCorrelationStatus.COMPLETED,
+            created_at=now,
+            updated_at=now,
+            timestamp=now,
+        )
+
+        time_service = getattr(self, "_time_service", None)
+        persistence.add_correlation(correlation, time_service)
+        logger.debug(f"Created speak correlation for channel {channel_id}")
+
+    async def _send_websocket_message(self, channel_id: str, content: str) -> bool:
+        """Send message through WebSocket if channel is WebSocket type."""
+        if not (channel_id and channel_id.startswith("ws:")):
+            return False
+
+        client_id = channel_id[3:]  # Remove "ws:" prefix
+        if client_id not in self._websocket_clients:
+            logger.warning(f"WebSocket client not found: {client_id}")
+            return False
+
+        ws = self._websocket_clients.get(client_id)
+        # Type guard: ensure ws has send_json method
+        if ws is None or not hasattr(ws, "send_json"):
+            logger.warning(f"Invalid WebSocket client: {client_id}")
+            return False
+
+        await ws.send_json(
+            {
+                "type": "message",
+                "data": {"content": content, "timestamp": datetime.now(timezone.utc).isoformat()},
+            }
+        )
+        return True
+
+    async def _handle_api_interaction_response(self, channel_id: str, content: str) -> None:
+        """Handle API interaction response storage if applicable."""
+        if not (channel_id and channel_id.startswith("api_")):
+            logger.debug(f"[API_INTERACTION] Skipping non-API channel: {channel_id}")
+            return
+
+        logger.info(
+            f"[API_INTERACTION] Processing channel_id={channel_id}, content_len={len(content)}, content_preview={content[:100] if content else 'EMPTY'}"
+        )
+        try:
+            if not hasattr(self, "_app_state"):
+                logger.warning("[API_INTERACTION] No _app_state attribute found")
+                return
+
+            message_channel_map = getattr(self._app_state, "message_channel_map", {})
+            logger.debug(f"[API_INTERACTION] message_channel_map keys: {list(message_channel_map.keys())}")
+            message_id = message_channel_map.get(channel_id)
+            if not message_id:
+                logger.warning(f"[API_INTERACTION] No message_id found for channel {channel_id}")
+                return
+
+            from ciris_engine.logic.adapters.api.routes.agent import store_message_response
+
+            logger.info(f"[API_INTERACTION] About to store: message_id={message_id}, content='{content}'")
+            await store_message_response(message_id, content)
+            logger.info(f"Stored interact response for message {message_id} in channel {channel_id}")
+            # Clean up the mapping
+            del message_channel_map[channel_id]
+        except Exception as e:
+            logger.debug(f"Could not store interact response: {e}")
+
+    def _track_response_time(self, start_time: datetime) -> None:
+        """Track response time metrics."""
+        elapsed_ms = (datetime.now(timezone.utc) - start_time).total_seconds() * 1000
+        self._response_times.append(elapsed_ms)
+        if len(self._response_times) > self._max_response_times:
+            self._response_times = self._response_times[-self._max_response_times :]
+
+    async def send_message(self, channel_id: str, content: str) -> bool:
+        """Send message through API response or WebSocket."""
+        start_time = datetime.now(timezone.utc)
+        logger.info(
+            f"[SEND_MESSAGE] channel_id={channel_id}, content_len={len(content)}, content_preview={content[:100] if content else 'EMPTY'}"
+        )
+        try:
+            # Create correlation for tracking
+            self._create_speak_correlation(channel_id, content)
+
+            # Try WebSocket first
+            if await self._send_websocket_message(channel_id, content):
+                return True
+
+            # Queue for HTTP response
+            await self._response_queue.put({"channel_id": channel_id, "content": content})
+
+            # Handle API interaction response if applicable
+            await self._handle_api_interaction_response(channel_id, content)
+
+            # Track successful request
+            self._track_request()
+            self._track_response_time(start_time)
+
+            return True
+
+        except Exception as e:
+            logger.error(f"Failed to send message: {e}")
+            self._track_error(e)
+            return False
+
+    def register_websocket(self, client_id: str, websocket: Any) -> None:
+        """Register a WebSocket client."""
+        self._websocket_clients[client_id] = websocket
+        logger.info(f"WebSocket client registered: {client_id}")
+
+    def unregister_websocket(self, client_id: str) -> None:
+        """Unregister a WebSocket client."""
+        if client_id in self._websocket_clients:
+            del self._websocket_clients[client_id]
+            logger.info(f"WebSocket client unregistered: {client_id}")
+
+    def _extract_parameters(self, request_data: Any) -> JSONDict:
+        """Extract parameters from request data handling both dict and Pydantic model cases."""
+        if hasattr(request_data, "get"):
+            params = request_data.get("parameters", {})
+            return params if isinstance(params, dict) else {}
+        elif hasattr(request_data, "parameters") and request_data.parameters:
+            params = request_data.parameters
+            return params if isinstance(params, dict) else {}
+        return {}
+
+    def _create_speak_message(self, correlation: Any) -> FetchedMessage:
+        """Create a FetchedMessage from a 'speak' correlation (outgoing agent message)."""
+        params = self._extract_parameters(correlation.request_data)
+        content = params.get("content", "")
+
+        return FetchedMessage(
+            message_id=correlation.correlation_id,
+            author_id="ciris",
+            author_name="CIRIS",
+            content=content,
+            timestamp=self._format_timestamp(correlation),
+            is_bot=True,
+        )
+
+    def _create_observe_message(self, correlation: Any) -> FetchedMessage:
+        """Create a FetchedMessage from an 'observe' correlation (incoming user message)."""
+        params = self._extract_parameters(correlation.request_data)
+
+        return FetchedMessage(
+            message_id=params.get("message_id", correlation.correlation_id),
+            author_id=params.get("author_id", "unknown"),
+            author_name=params.get("author_name", "User"),
+            content=params.get("content", ""),
+            timestamp=self._format_timestamp(correlation),
+            is_bot=False,
+        )
+
+    def _format_timestamp(self, correlation: Any) -> Optional[str]:
+        """Format correlation timestamp for FetchedMessage."""
+        timestamp = correlation.timestamp or correlation.created_at
+        return timestamp.isoformat() if timestamp else None
+
+    def _process_correlation(self, correlation: Any) -> Optional[FetchedMessage]:
+        """Process a single correlation into a FetchedMessage if applicable."""
+        if not correlation.request_data:
+            return None
+
+        if correlation.action_type == "speak":
+            return self._create_speak_message(correlation)
+        elif correlation.action_type == "observe":
+            return self._create_observe_message(correlation)
+
+        return None
+
+    async def fetch_messages(
+        self,
+        channel_id: str,
+        *,
+        limit: int = 50,
+        before: Optional[datetime] = None,
+    ) -> List[FetchedMessage]:
+        """Retrieve messages from a channel using the correlations database."""
+        from ciris_engine.logic.persistence import get_correlations_by_channel
+
+        try:
+            correlations = get_correlations_by_channel(channel_id=channel_id, limit=limit, before=before)
+
+            messages = []
+            for corr in correlations:
+                message = self._process_correlation(corr)
+                if message:
+                    messages.append(message)
+
+            # Sort by timestamp
+            messages.sort(key=lambda m: str(m.timestamp))
+            return messages
+
+        except Exception as e:
+            logger.error(f"Failed to fetch messages from correlations for channel {channel_id}: {e}")
+            return []
+
+    async def start(self) -> None:
+        """Start the communication service."""
+        await BaseService.start(self)
+        logger.info("API communication service started")
+
+    async def stop(self) -> None:
+        """Stop the communication service."""
+        # Clear any pending responses
+        while not self._response_queue.empty():
+            try:
+                self._response_queue.get_nowait()
+            except asyncio.QueueEmpty:
+                break
+        await BaseService.stop(self)
+        logger.info("API communication service stopped")
+
+    def _check_dependencies(self) -> bool:
+        """Check if all dependencies are satisfied."""
+        return True  # No hard dependencies
+
+    def _collect_custom_metrics(self) -> Dict[str, float]:
+        """Collect API communication specific metrics."""
+        avg_response_time = sum(self._response_times) / len(self._response_times) if self._response_times else 0.0
+        return {
+            "websocket_clients": float(len(self._websocket_clients)),
+            "queue_size": float(self._response_queue.qsize()),
+            "avg_response_time_ms": avg_response_time,
+            "response_times_tracked": float(len(self._response_times)),
+        }
+
+    def _get_actions(self) -> List[str]:
+        """Get the list of actions this service supports."""
+        return ["send_message", "fetch_messages", "queue_response", "dequeue_response"]
+
+    def get_service_type(self) -> ServiceType:
+        """Get the type of this service."""
+        return ServiceType.ADAPTER
+
+    def get_home_channel_id(self) -> Optional[str]:
+        """Get the home channel ID for this API adapter.
+
+        Returns:
+            The formatted channel ID (e.g., 'api_0.0.0.0_8080')
+            or None if no home channel is configured.
+        """
+        if self._config and hasattr(self._config, "get_home_channel_id"):
+            # Use the config method if available
+            host = getattr(self._config, "host", "0.0.0.0")
+            port = getattr(self._config, "port", 8080)
+            result = self._config.get_home_channel_id(host, port)
+            return str(result) if result is not None else None
+
+        # Default fallback
+        return "api_0.0.0.0_8080"
+
+    def get_status(self) -> "ServiceStatus":
+        """Get the service status."""
+        from ciris_engine.schemas.services.core import ServiceStatus
+
+        # Calculate uptime
+        uptime_seconds = 0.0
+        if self._start_time:
+            current_time = datetime.now(timezone.utc) if not self._time_service else self._time_service.now()
+            uptime_seconds = (current_time - self._start_time).total_seconds()
+
+        # Calculate average response time
+        avg_response_time = 0.0
+        if self._response_times:
+            avg_response_time = sum(self._response_times) / len(self._response_times)
+
+        return ServiceStatus(
+            service_name="APICommunicationService",
+            service_type="communication",
+            is_healthy=self._started,
+            uptime_seconds=uptime_seconds,
+            last_error=None,  # Could track last error message
+            metrics={
+                "requests_handled": float(self._requests_handled),
+                "error_count": float(self._error_count),
+                "avg_response_time_ms": avg_response_time,
+                "queued_responses": float(self._response_queue.qsize()),
+                "websocket_clients": float(
+                    len(self._websocket_clients) if hasattr(self._websocket_clients, "__len__") else 0
+                ),
+            },
+        )
+
+    def get_capabilities(self) -> "ServiceCapabilities":
+        """Get the service capabilities."""
+        from ciris_engine.schemas.services.core import ServiceCapabilities
+
+        return ServiceCapabilities(
+            service_name="APICommunicationService",
+            actions=[
+                "send_message",
+                "fetch_messages",
+                "broadcast",
+                "get_response",
+                "register_websocket",
+                "unregister_websocket",
+            ],
+            version="1.0.0",
+            metadata=None,
+        )

ciris_engine/logic/adapters/api/api_document.py

@@ -0,0 +1,330 @@
+"""API Document Helper for processing PDF/DOCX documents.
+
+Extends DocumentParser functionality for API-specific document handling.
+Documents can be submitted as base64 data or URLs in the interact endpoint.
+"""
+
+import base64
+import logging
+import tempfile
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import aiohttp
+
+from ciris_engine.logic.adapters.document_parser import DocumentParser
+from ciris_engine.schemas.types import JSONDict
+
+logger = logging.getLogger(__name__)
+
+# Default media types
+DEFAULT_PDF_MEDIA_TYPE = "application/pdf"
+DEFAULT_DOCX_MEDIA_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+
+
+class APIDocumentHelper:
+    """
+    Helper class for processing API documents.
+
+    Converts API document payloads to extracted text using DocumentParser.
+    Supports both base64-encoded documents and URL-based documents.
+    """
+
+    # Security constraints (same as DocumentParser)
+    MAX_FILE_SIZE = 1024 * 1024  # 1MB
+    MAX_DOCUMENTS = 3
+    DOWNLOAD_TIMEOUT = 10.0  # seconds
+
+    # Allowed formats
+    ALLOWED_EXTENSIONS = {".pdf", ".docx"}
+    ALLOWED_CONTENT_TYPES = {
+        DEFAULT_PDF_MEDIA_TYPE,
+        DEFAULT_DOCX_MEDIA_TYPE,
+    }
+
+    def __init__(self) -> None:
+        """Initialize the document helper with parser."""
+        self._parser = DocumentParser()
+
+    def is_available(self) -> bool:
+        """Check if document processing is available.
+
+        Returns:
+            True if at least one document format is supported
+        """
+        return self._parser.is_available()
+
+    def get_status(self) -> JSONDict:
+        """Get current status of document helper.
+
+        Returns:
+            Status dictionary
+        """
+        parser_status = self._parser.get_status()
+        return {
+            "available": self.is_available(),
+            "max_file_size_mb": self.MAX_FILE_SIZE / 1024 / 1024,
+            "max_documents": self.MAX_DOCUMENTS,
+            "pdf_support": parser_status.get("pdf_support", False),
+            "docx_support": parser_status.get("docx_support", False),
+            "allowed_extensions": list(self.ALLOWED_EXTENSIONS),
+        }
+
+    def _get_file_extension(self, media_type: str, filename: Optional[str]) -> Optional[str]:
+        """Determine file extension from media type or filename.
+
+        Args:
+            media_type: MIME type of the document
+            filename: Optional filename
+
+        Returns:
+            File extension (e.g., '.pdf') or None if unsupported
+        """
+        # Try filename first
+        if filename:
+            ext = Path(filename).suffix.lower()
+            if ext in self.ALLOWED_EXTENSIONS:
+                return ext
+
+        # Fall back to media type
+        type_to_ext = {
+            DEFAULT_PDF_MEDIA_TYPE: ".pdf",
+            DEFAULT_DOCX_MEDIA_TYPE: ".docx",
+        }
+        return type_to_ext.get(media_type)
+
+    def _is_valid_media_type(self, media_type: str) -> bool:
+        """Check if media type is allowed."""
+        return media_type in self.ALLOWED_CONTENT_TYPES
+
+    async def process_base64_document(
+        self,
+        base64_data: str,
+        media_type: str,
+        filename: Optional[str] = None,
+    ) -> Optional[str]:
+        """Process a base64-encoded document and extract text.
+
+        Args:
+            base64_data: Base64-encoded document data (with or without data URL prefix)
+            media_type: MIME type of the document
+            filename: Optional filename
+
+        Returns:
+            Extracted text or None if processing failed
+        """
+        try:
+            # Handle data URL format: data:application/pdf;base64,JVBERi0...
+            if base64_data.startswith("data:"):
+                header, data = base64_data.split(",", 1)
+                if ";" in header:
+                    media_type = header.split(":")[1].split(";")[0]
+                base64_data = data
+
+            # Validate media type
+            if not self._is_valid_media_type(media_type):
+                logger.warning(f"Unsupported document type: {media_type}")
+                return None
+
+            # Decode base64
+            try:
+                file_data = base64.b64decode(base64_data)
+            except Exception as e:
+                logger.error(f"Invalid base64 data: {e}")
+                return None
+
+            # Check size
+            if len(file_data) > self.MAX_FILE_SIZE:
+                logger.warning(f"Document too large: {len(file_data)} bytes (max {self.MAX_FILE_SIZE} bytes)")
+                return None
+
+            # Get file extension
+            file_ext = self._get_file_extension(media_type, filename)
+            if not file_ext:
+                logger.warning(f"Could not determine file extension for {media_type}")
+                return None
+
+            # Extract text using parser
+            return self._parser._extract_text_sync(file_data, file_ext)
+
+        except Exception as e:
+            logger.exception(f"Error processing base64 document: {e}")
+            return None
+
+    async def process_url_document(
+        self,
+        url: str,
+        media_type: str,
+        filename: Optional[str] = None,
+    ) -> Optional[str]:
+        """Process a document from URL and extract text.
+
+        Args:
+            url: URL of the document
+            media_type: MIME type of the document
+            filename: Optional filename
+
+        Returns:
+            Extracted text or None if processing failed
+        """
+        try:
+            # Validate media type
+            if not self._is_valid_media_type(media_type):
+                logger.warning(f"Unsupported document type: {media_type}")
+                return None
+
+            # Get file extension
+            file_ext = self._get_file_extension(media_type, filename)
+            if not file_ext:
+                # Try to infer from URL
+                url_path = Path(url.split("?")[0])
+                if url_path.suffix.lower() in self.ALLOWED_EXTENSIONS:
+                    file_ext = url_path.suffix.lower()
+                else:
+                    logger.warning(f"Could not determine file extension for {url}")
+                    return None
+
+            # Download document
+            file_data = await self._download_document(url)
+            if not file_data:
+                return None
+
+            # Extract text using parser
+            return self._parser._extract_text_sync(file_data, file_ext)
+
+        except Exception as e:
+            logger.exception(f"Error processing URL document: {e}")
+            return None
+
+    async def _download_document(self, url: str) -> Optional[bytes]:
+        """Download document from URL with security limits.
+
+        Args:
+            url: URL to download from
+
+        Returns:
+            Document bytes or None if download failed
+        """
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.DOWNLOAD_TIMEOUT)) as response:
+                    if response.status != 200:
+                        logger.error(f"Failed to download document (HTTP {response.status})")
+                        return None
+
+                    # Check content length header
+                    if response.headers.get("content-length"):
+                        content_length = int(response.headers["content-length"])
+                        if content_length > self.MAX_FILE_SIZE:
+                            logger.error(f"Document too large ({content_length} bytes)")
+                            return None
+
+                    # Read with size limit
+                    file_data = b""
+                    async for chunk in response.content.iter_chunked(8192):
+                        file_data += chunk
+                        if len(file_data) > self.MAX_FILE_SIZE:
+                            logger.error("Document size limit exceeded during download")
+                            return None
+
+                    return file_data
+
+        except Exception as e:
+            logger.error(f"Document download failed: {e}")
+            return None
+
+    async def process_document_payload(
+        self,
+        document_data: str,
+        media_type: str = "application/pdf",
+        filename: Optional[str] = None,
+    ) -> Optional[str]:
+        """Process a document payload from the API (auto-detect base64 vs URL).
+
+        Args:
+            document_data: Either base64-encoded data or a URL
+            media_type: MIME type of the document
+            filename: Optional filename
+
+        Returns:
+            Extracted text or None if processing failed
+        """
+        # Detect URL vs base64 (checking format, not making connections)
+        if document_data.startswith(("http://", "https://")):  # NOSONAR - format detection only
+            return await self.process_url_document(document_data, media_type, filename)
+        else:
+            return await self.process_base64_document(document_data, media_type, filename)
+
+    async def process_document_list(
+        self,
+        documents: List[Dict[str, Any]],
+    ) -> Optional[str]:
+        """Process a list of document payloads from the API.
+
+        Each document dict should have:
+        - data: base64 string or URL
+        - media_type: (optional) MIME type, defaults to application/pdf
+        - filename: (optional) filename
+
+        Args:
+            documents: List of document dictionaries
+
+        Returns:
+            Combined extracted text from all documents, or None if no documents processed
+        """
+        if not self.is_available():
+            logger.warning("Document processing not available (missing libraries)")
+            return None
+
+        if not documents:
+            return None
+
+        # Limit number of documents
+        if len(documents) > self.MAX_DOCUMENTS:
+            logger.warning(f"Too many documents ({len(documents)}), processing first {self.MAX_DOCUMENTS}")
+            documents = documents[: self.MAX_DOCUMENTS]
+
+        extracted_texts: List[str] = []
+
+        for doc in documents:
+            if not isinstance(doc, dict):
+                logger.warning(f"Invalid document entry (not a dict): {type(doc)}")
+                continue
+
+            data = doc.get("data")
+            if not data:
+                logger.warning("Document entry missing 'data' field")
+                continue
+
+            media_type = doc.get("media_type", "application/pdf")
+            filename = doc.get("filename")
+
+            text = await self.process_document_payload(data, media_type, filename)
+            if text:
+                doc_name = filename or "document"
+                extracted_texts.append(f"Document '{doc_name}':\n{text}")
+
+        if extracted_texts:
+            combined = "\n\n---\n\n".join(extracted_texts)
+            logger.info(f"Processed {len(extracted_texts)} documents from API payload")
+
+            # Apply text length limit
+            if len(combined) > self._parser.MAX_TEXT_LENGTH:
+                logger.warning(f"Combined text too long ({len(combined)} chars), truncating")
+                combined = combined[: self._parser.MAX_TEXT_LENGTH] + "\n\n[Text truncated]"
+
+            return combined
+
+        return None
+
+
+# Singleton instance
+_api_document_helper: Optional[APIDocumentHelper] = None
+
+
+def get_api_document_helper() -> APIDocumentHelper:
+    """Get or create the API document helper singleton."""
+    global _api_document_helper
+    if _api_document_helper is None:
+        _api_document_helper = APIDocumentHelper()
+    return _api_document_helper