ciris-agent 1.7.7__py3-none-any.whl

ciris_engine/logic/adapters/api/routes/memory_filters.py

@@ -0,0 +1,230 @@
+"""
+Memory query filters for role-based access control.
+
+Provides double-protection filtering for OBSERVER users:
+1. Database-level: Inject SQL filters before query execution
+2. Result-level: Filter returned nodes based on user attribution
+
+This ensures OBSERVER users only see memories they created or participated in.
+"""
+
+import json
+import logging
+from typing import Any, Dict, List, Optional, Set
+
+from ciris_engine.logic.utils.jsondict_helpers import get_list
+from ciris_engine.schemas.api.auth import UserRole
+from ciris_engine.schemas.services.graph_core import GraphNode
+from ciris_engine.schemas.types import JSONDict
+
+logger = logging.getLogger(__name__)
+
+
+async def get_user_allowed_ids(auth_service: Any, user_id: str) -> Set[str]:
+    """
+    Get set of user IDs user is allowed to see (user_id + OAuth links).
+
+    Matches the same logic as reasoning event stream filtering for consistency.
+
+    Args:
+        auth_service: Authentication service instance
+        user_id: Primary user ID
+
+    Returns:
+        Set of allowed user IDs (primary + OAuth linked accounts)
+    """
+    allowed_ids = {user_id}
+
+    try:
+        from ciris_engine.logic.persistence.db import get_db_connection
+
+        # Use db_path with get_db_connection() for proper SQLite/PostgreSQL support
+        # CRITICAL: auth_service.db_path may be a PostgreSQL URL, not a file path!
+        # sqlite3.connect() cannot handle PostgreSQL URLs, causing "unable to open database file"
+        db_path = auth_service.db_path
+        query = """
+            SELECT oauth_provider, oauth_external_id
+            FROM wa_cert
+            WHERE wa_id = ? AND oauth_provider IS NOT NULL AND oauth_external_id IS NOT NULL AND active = 1
+        """
+        with get_db_connection(db_path=db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(query, (user_id,))
+            rows = cursor.fetchall()
+            cursor.close()
+
+            for row in rows:
+                # Handle both tuple and dict rows (PostgreSQL uses RealDictCursor)
+                if isinstance(row, dict):
+                    oauth_provider = row["oauth_provider"]
+                    oauth_external_id = row["oauth_external_id"]
+                else:
+                    oauth_provider, oauth_external_id = row
+
+                # Add both provider:id and bare id format
+                allowed_ids.add(f"{oauth_provider}:{oauth_external_id}")
+                allowed_ids.add(oauth_external_id)
+    except Exception as e:
+        logger.error(
+            f"Error fetching OAuth links for user {user_id} from database {getattr(auth_service, 'db_path', 'unknown')}: {e}",
+            exc_info=True,
+        )
+
+    return allowed_ids
+
+
+def build_user_filter_sql(allowed_user_ids: Set[str]) -> tuple[str, List[str]]:
+    """
+    Build SQL filter clause for user-based filtering (Defense Layer 1).
+
+    Filters on JSON-extracted attributes.created_by field.
+
+    Args:
+        allowed_user_ids: Set of user IDs to allow
+
+    Returns:
+        Tuple of (WHERE clause fragment, query parameters)
+    """
+    if not allowed_user_ids:
+        # No allowed users = block everything
+        return ("AND 1 = 0", [])
+
+    # Build parameterized query for JSON extraction
+    # SQLite: json_extract(attributes_json, '$.created_by')
+    placeholders = ",".join("?" * len(allowed_user_ids))
+    where_clause = f"AND json_extract(attributes_json, '$.created_by') IN ({placeholders})"
+    params = list(allowed_user_ids)
+
+    return (where_clause, params)
+
+
+def _extract_attributes_dict(node: GraphNode) -> Optional[JSONDict]:
+    """
+    Extract attributes as a dictionary from a GraphNode.
+
+    Args:
+        node: GraphNode to extract attributes from
+
+    Returns:
+        Dictionary of attributes, or None if extraction fails
+    """
+    attrs = node.attributes
+    if isinstance(attrs, dict):
+        return attrs
+    elif hasattr(attrs, "model_dump"):
+        return attrs.model_dump()
+    elif hasattr(attrs, "dict"):
+        return attrs.dict()
+    else:
+        logger.warning(f"Cannot inspect attributes for node {node.id}, skipping")
+        return None
+
+
+def _check_created_by(attrs_dict: JSONDict, allowed_user_ids: Set[str]) -> bool:
+    """Check if node creator is in allowed users."""
+    created_by = attrs_dict.get("created_by")
+    return bool(created_by and created_by in allowed_user_ids)
+
+
+def _check_user_list(attrs_dict: JSONDict, allowed_user_ids: Set[str]) -> bool:
+    """Check if any user in user_list is allowed."""
+    user_list = get_list(attrs_dict, "user_list", [])
+    return any(uid in allowed_user_ids for uid in user_list if isinstance(uid, str))
+
+
+def _check_task_summaries(attrs_dict: JSONDict, allowed_user_ids: Set[str]) -> bool:
+    """Check if any task summary user_id is allowed."""
+    task_summaries = attrs_dict.get("task_summaries", {})
+    if not isinstance(task_summaries, dict):
+        return False
+
+    for task_data in task_summaries.values():
+        if isinstance(task_data, dict):
+            task_user_id = task_data.get("user_id")
+            if task_user_id and task_user_id in allowed_user_ids:
+                return True
+    return False
+
+
+def _check_conversations(attrs_dict: JSONDict, allowed_user_ids: Set[str]) -> bool:
+    """Check if any conversation author_id is allowed."""
+    conversations = attrs_dict.get("conversations_by_channel", {})
+    if not isinstance(conversations, dict):
+        return False
+
+    for channel_messages in conversations.values():
+        if isinstance(channel_messages, list):
+            for msg in channel_messages:
+                if isinstance(msg, dict):
+                    author_id = msg.get("author_id")
+                    if author_id and author_id in allowed_user_ids:
+                        return True
+    return False
+
+
+def _node_matches_user_attribution(node: GraphNode, allowed_user_ids: Set[str]) -> bool:
+    """
+    Check if a single node matches user attribution criteria.
+
+    Args:
+        node: GraphNode to check
+        allowed_user_ids: Set of allowed user IDs
+
+    Returns:
+        True if node should be included, False otherwise
+    """
+    attrs_dict = _extract_attributes_dict(node)
+    if attrs_dict is None:
+        return False
+
+    # Check all attribution pathways (order by frequency for performance)
+    return (
+        _check_created_by(attrs_dict, allowed_user_ids)
+        or _check_user_list(attrs_dict, allowed_user_ids)
+        or _check_task_summaries(attrs_dict, allowed_user_ids)
+        or _check_conversations(attrs_dict, allowed_user_ids)
+    )
+
+
+def filter_nodes_by_user_attribution(nodes: List[GraphNode], allowed_user_ids: Set[str]) -> List[GraphNode]:
+    """
+    Filter graph nodes by user attribution (Defense Layer 2).
+
+    Checks multiple attribution fields for comprehensive coverage:
+    - attributes.created_by (direct creator)
+    - attributes.user_list[] (participants in consolidated nodes)
+    - attributes.conversations_by_channel[*][].author_id (conversation participants)
+    - attributes.task_summaries[*].user_id (task creators)
+
+    Args:
+        nodes: List of GraphNode objects to filter
+        allowed_user_ids: Set of user IDs to allow
+
+    Returns:
+        Filtered list of GraphNode objects
+    """
+    if not allowed_user_ids:
+        return []
+
+    filtered = [node for node in nodes if _node_matches_user_attribution(node, allowed_user_ids)]
+
+    logger.debug(
+        f"Filtered {len(nodes)} nodes to {len(filtered)} for user access control "
+        f"(allowed_ids: {len(allowed_user_ids)})"
+    )
+
+    return filtered
+
+
+def should_apply_user_filtering(user_role: UserRole) -> bool:
+    """
+    Determine if user filtering should be applied based on role.
+
+    Args:
+        user_role: User's role
+
+    Returns:
+        True if filtering should be applied (OBSERVER), False for ADMIN+
+    """
+    # ADMIN and higher roles bypass filtering
+    return not user_role.has_permission(UserRole.ADMIN)

ciris_engine/logic/adapters/api/routes/memory_models.py

@@ -0,0 +1,112 @@
+"""
+Data models for memory API endpoints.
+
+Extracted from memory.py to improve modularity and testability.
+"""
+
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field, field_serializer, model_validator
+
+from ciris_engine.schemas.services.graph_core import GraphEdge, GraphNode, GraphScope, NodeType
+from ciris_engine.schemas.types import JSONDict
+
+
+class StoreRequest(BaseModel):
+    """Request to store typed nodes in memory (MEMORIZE)."""
+
+    node: GraphNode = Field(..., description="Typed graph node to store")
+
+
+# Edge creation handled internally - no API model needed
+
+
+class QueryRequest(BaseModel):
+    """Flexible query interface for memory (RECALL).
+
+    Supports multiple query patterns:
+    - By ID: Specify node_id
+    - By type: Specify type filter
+    - By text: Specify query string
+    - By time: Specify since/until filters
+    - By correlation: Specify related_to node
+    """
+
+    # Node-based queries
+    node_id: Optional[str] = Field(None, description="Get specific node by ID")
+    type: Optional[NodeType] = Field(None, description="Filter by node type")
+
+    # Text search
+    query: Optional[str] = Field(None, description="Text search query")
+
+    # Time-based queries
+    since: Optional[datetime] = Field(None, description="Memories since this time")
+    until: Optional[datetime] = Field(None, description="Memories until this time")
+
+    # Correlation queries
+    related_to: Optional[str] = Field(None, description="Find nodes related to this node ID")
+
+    # Filters
+    scope: Optional[GraphScope] = Field(None, description="Memory scope filter")
+    tags: Optional[List[str]] = Field(None, description="Filter by tags")
+
+    # Pagination
+    limit: int = Field(20, ge=1, le=1000, description="Maximum results")
+    offset: int = Field(0, ge=0, description="Pagination offset")
+
+    # Options
+    include_edges: bool = Field(False, description="Include relationship data")
+    depth: int = Field(1, ge=1, le=3, description="Graph traversal depth for relationships")
+
+    @model_validator(mode="after")
+    def validate_query_params(self: "QueryRequest") -> "QueryRequest":
+        """Ensure at least one query parameter is specified."""
+        if not any(
+            [
+                self.node_id,
+                self.type,
+                self.query,
+                self.since,
+                self.until,
+                self.related_to,
+                self.scope,
+                self.tags,
+            ]
+        ):
+            # Default to recent memories if no query specified
+            self.since = datetime.now() - timedelta(hours=24)
+        return self
+
+    @field_serializer("since", "until")
+    def serialize_datetime(self, dt: Optional[datetime], _info: Any) -> Optional[str]:
+        return dt.isoformat() if dt else None
+
+
+class TimelineResponse(BaseModel):
+    """Response containing timeline of memories."""
+
+    memories: List[GraphNode] = Field(..., description="List of memory nodes")
+    buckets: JSONDict = Field(default_factory=dict, description="Time buckets with counts")
+    start_time: datetime = Field(..., description="Start of time range")
+    end_time: datetime = Field(..., description="End of time range")
+    total: int = Field(..., description="Total number of memories")
+
+    @field_serializer("start_time", "end_time")
+    def serialize_datetime(self, dt: Optional[datetime], _info: Any) -> Optional[str]:
+        return dt.isoformat() if dt else None
+
+
+class MemoryStats(BaseModel):
+    """Statistics about memory service."""
+
+    total_nodes: int = Field(..., description="Total number of nodes")
+    nodes_by_type: Dict[str, int] = Field(..., description="Node count by type")
+    nodes_by_scope: Dict[str, int] = Field(..., description="Node count by scope")
+    recent_nodes_24h: int = Field(..., description="Nodes created in last 24h")
+    oldest_node_date: Optional[datetime] = Field(None, description="Oldest node date")
+    newest_node_date: Optional[datetime] = Field(None, description="Newest node date")
+
+    @field_serializer("oldest_node_date", "newest_node_date")
+    def serialize_datetime(self, dt: Optional[datetime], _info: Any) -> Optional[str]:
+        return dt.isoformat() if dt else None

ciris_engine/logic/adapters/api/routes/memory_queries.py

@@ -0,0 +1,236 @@
+"""
+Database query utilities for memory API.
+
+Extracted from memory.py to improve modularity and testability.
+"""
+
+import json
+import logging
+from datetime import datetime, timedelta
+from typing import Any, Dict, List, Optional
+
+from ciris_engine.logic.persistence.db.core import get_db_connection
+from ciris_engine.logic.utils.jsondict_helpers import get_dict
+from ciris_engine.schemas.services.graph_core import GraphNode, GraphScope, NodeType
+from ciris_engine.schemas.types import JSONDict
+
+from .memory_query_helpers import DatabaseExecutor, DateTimeParser, GraphNodeBuilder, QueryBuilder, TimeRangeCalculator
+
+logger = logging.getLogger(__name__)
+
+# SQL Query Constants
+SQL_SELECT_NODES = "SELECT node_id, scope, node_type, attributes_json, version, updated_by, updated_at, created_at"
+SQL_FROM_NODES = "FROM graph_nodes"
+SQL_WHERE_TIME_RANGE = "WHERE updated_at >= ? AND updated_at < ?"
+SQL_EXCLUDE_METRICS = "AND NOT (node_type = 'tsdb_data' AND node_id LIKE 'metric_%')"
+SQL_WHERE_SCOPE = "AND scope = ?"
+SQL_WHERE_NODE_TYPE = "AND node_type = ?"
+SQL_ORDER_RANDOM = "ORDER BY RANDOM()"
+SQL_LIMIT = "LIMIT ?"
+
+
+async def query_timeline_nodes(
+    memory_service: Any,
+    hours: int = 24,
+    scope: Optional[str] = None,
+    node_type: Optional[str] = None,
+    limit: int = 100,
+    exclude_metrics: bool = True,
+    user_filter_ids: Optional[List[str]] = None,
+) -> List[GraphNode]:
+    """
+    Query nodes from memory within a time range.
+
+    Args:
+        memory_service: Memory service instance
+        hours: Number of hours to look back
+        scope: Optional scope filter
+        node_type: Optional node type filter
+        limit: Maximum number of results
+        exclude_metrics: Whether to exclude metric nodes
+        user_filter_ids: Optional list of user IDs for OBSERVER filtering (SQL Layer 1)
+
+    Returns:
+        List of GraphNode objects
+    """
+    # Get database path
+    db_path = DatabaseExecutor.get_db_path(memory_service)
+    if not db_path:
+        return []
+
+    # Calculate time range
+    start_time, end_time = TimeRangeCalculator.calculate_range(hours)
+
+    # Build query with all filters (including user filtering for OBSERVER users)
+    query, params = QueryBuilder.build_timeline_query(
+        start_time=start_time,
+        end_time=end_time,
+        scope=scope,
+        node_type=node_type,
+        exclude_metrics=exclude_metrics,
+        limit=limit,
+        user_filter_ids=user_filter_ids,
+    )
+
+    # Execute query and get rows
+    rows = DatabaseExecutor.execute_query(db_path, query, params)
+
+    # Build GraphNode objects from rows
+    return GraphNodeBuilder.build_from_rows(rows)
+
+
+async def get_memory_stats(memory_service: Any) -> JSONDict:
+    """
+    Get statistics about memory storage.
+
+    Args:
+        memory_service: Memory service instance
+
+    Returns:
+        Dictionary with memory statistics
+    """
+    stats: JSONDict = {
+        "total_nodes": 0,
+        "total_edges": 0,
+        "nodes_by_type": {},
+        "nodes_by_scope": {},
+        "recent_activity": {},
+        "storage_size_mb": 0.0,
+    }
+
+    try:
+        db_path = getattr(memory_service, "db_path", None)
+        if not db_path:
+            return stats
+
+        with get_db_connection(db_path=db_path) as conn:
+            cursor = conn.cursor()
+
+            # Total nodes
+            cursor.execute("SELECT COUNT(*) FROM graph_nodes")
+            stats["total_nodes"] = cursor.fetchone()[0]
+
+            # Total edges
+            cursor.execute("SELECT COUNT(*) FROM graph_edges")
+            stats["total_edges"] = cursor.fetchone()[0]
+
+            # Nodes by type
+            nodes_by_type = get_dict(stats, "nodes_by_type", {})
+            cursor.execute("SELECT node_type, COUNT(*) FROM graph_nodes GROUP BY node_type")
+            for row in cursor.fetchall():
+                nodes_by_type[row[0]] = row[1]
+
+            # Nodes by scope
+            nodes_by_scope = get_dict(stats, "nodes_by_scope", {})
+            cursor.execute("SELECT scope, COUNT(*) FROM graph_nodes GROUP BY scope")
+            for row in cursor.fetchall():
+                nodes_by_scope[row[0]] = row[1]
+
+            # Recent activity (last 24 hours)
+            recent_activity = get_dict(stats, "recent_activity", {})
+            now = datetime.now()
+            yesterday = now - timedelta(days=1)
+
+            from ciris_engine.logic.persistence.db.dialect import get_adapter
+
+            placeholder = get_adapter().placeholder()
+
+            cursor.execute(
+                f"SELECT COUNT(*) FROM graph_nodes WHERE updated_at >= {placeholder}", (yesterday.isoformat(),)
+            )
+            recent_activity["nodes_24h"] = cursor.fetchone()[0]
+
+            cursor.execute(
+                f"SELECT COUNT(*) FROM graph_edges WHERE created_at >= {placeholder}", (yesterday.isoformat(),)
+            )
+            recent_activity["edges_24h"] = cursor.fetchone()[0]
+
+            # Storage size
+            import os
+
+            if os.path.exists(db_path):
+                stats["storage_size_mb"] = os.path.getsize(db_path) / (1024 * 1024)
+
+    except Exception as e:
+        logger.error(f"Failed to get memory stats: {e}")
+
+    return stats
+
+
+async def search_nodes(
+    memory_service: Any,
+    query: Optional[str] = None,
+    node_type: Optional[NodeType] = None,
+    scope: Optional[GraphScope] = None,
+    since: Optional[datetime] = None,
+    until: Optional[datetime] = None,
+    tags: Optional[List[str]] = None,
+    limit: int = 20,
+    offset: int = 0,
+    user_filter_ids: Optional[List[str]] = None,
+) -> List[GraphNode]:
+    """
+    Search for nodes in memory with various filters.
+
+    Args:
+        memory_service: Memory service instance
+        query: Text search query
+        node_type: Filter by node type
+        scope: Filter by scope
+        since: Filter by minimum timestamp
+        until: Filter by maximum timestamp
+        tags: Filter by tags
+        limit: Maximum results
+        offset: Pagination offset
+        user_filter_ids: Optional list of user IDs for OBSERVER filtering (SQL Layer 1)
+
+    Returns:
+        List of matching GraphNode objects
+    """
+    # Get database path
+    db_path = DatabaseExecutor.get_db_path(memory_service)
+    if not db_path:
+        return []
+
+    # Build search query with all filters (including user filtering for OBSERVER users)
+    sql_query, params = QueryBuilder.build_search_query(
+        query=query,
+        node_type=node_type,
+        scope=scope,
+        since=since,
+        until=until,
+        tags=tags,
+        limit=limit,
+        offset=offset,
+        user_filter_ids=user_filter_ids,
+    )
+
+    # Execute query and get rows
+    rows = DatabaseExecutor.execute_query(db_path, sql_query, params)
+
+    # Build GraphNode objects from rows
+    return GraphNodeBuilder.build_from_rows(rows)
+
+
+def _parse_datetime(value: Any) -> Optional[datetime]:
+    """Parse datetime from various formats."""
+    if not value:
+        return None
+
+    if isinstance(value, datetime):
+        return value
+
+    if isinstance(value, str):
+        try:
+            # Try ISO format
+            return datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except ValueError:
+            pass
+
+        try:
+            # Try without timezone
+            return datetime.fromisoformat(value.split("+")[0])
+        except ValueError:
+            pass
+
+    return None