remdb 0.3.7__py3-none-any.whl

rem/services/dreaming/moment_service.py

@@ -0,0 +1,264 @@
+"""
+Moment Service - Extracts temporal narratives from resources.
+
+Analyzes recent resources and sessions to identify temporal narratives
+(meetings, coding sessions, conversations) and creates Moment entities
+with temporal boundaries and metadata.
+"""
+
+import json
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any, Optional
+from uuid import uuid4
+
+import yaml
+from loguru import logger
+
+from ...agentic.providers.pydantic_ai import create_agent
+from ...agentic.serialization import serialize_agent_result
+from ...models.entities.moment import Moment, Person
+from ...models.entities.resource import Resource
+from ...models.entities.message import Message
+from ...services.postgres.repository import Repository
+from ...services.postgres.service import PostgresService
+
+
+async def construct_moments(
+    user_id: str,
+    db: PostgresService,
+    default_model: str = "gpt-4o",
+    lookback_hours: int = 24,
+    limit: Optional[int] = None,
+) -> dict[str, Any]:
+    """
+    Extract moments from resources.
+
+    Analyzes recent resources to identify temporal narratives
+    (meetings, coding sessions, conversations) and creates
+    Moment entities with temporal boundaries and metadata.
+
+    Process:
+    1. Query PostgreSQL for recent resources and sessions for this user
+    2. Load MomentBuilder agent schema from filesystem
+    3. Run agent to extract moments from data
+    4. Create Moment entities via Repository
+    5. Link moments to source resources via graph edges
+    6. Embeddings auto-generated by embedding worker
+
+    Args:
+        user_id: User to process
+        db: Database service (already connected)
+        default_model: LLM model for analysis (default: gpt-4o)
+        lookback_hours: Hours to look back (default: 24)
+        limit: Max resources to process
+
+    Returns:
+        Statistics about moment construction
+    """
+    cutoff = datetime.utcnow() - timedelta(hours=lookback_hours)
+
+    # Create repositories
+    resource_repo = Repository(Resource, "resources", db=db)
+    message_repo = Repository(Message, "messages", db=db)
+    moment_repo = Repository(Moment, "moments", db=db)
+
+    # Query recent resources
+    resources = await resource_repo.find(
+        filters={
+            "user_id": user_id,
+        },
+        order_by="created_at DESC",
+        limit=limit,
+    )
+
+    # Filter by timestamp (SQL doesn't support comparisons in find yet)
+    resources = [
+        r for r in resources if r.created_at and r.created_at >= cutoff
+    ]
+
+    # Query recent messages (grouped by session_id for context)
+    messages = await message_repo.find(
+        filters={
+            "user_id": user_id,
+        },
+        order_by="created_at DESC",
+        limit=limit,
+    )
+
+    # Filter by timestamp
+    messages = [m for m in messages if m.created_at >= cutoff]
+
+    if not resources and not messages:
+        return {
+            "user_id": user_id,
+            "lookback_hours": lookback_hours,
+            "resources_queried": 0,
+            "messages_queried": 0,
+            "moments_created": 0,
+            "graph_edges_added": 0,
+            "status": "no_data",
+        }
+
+    # Load MomentBuilder agent schema
+    schema_path = (
+        Path(__file__).parent.parent.parent
+        / "schemas"
+        / "agents"
+        / "core"
+        / "moment-builder.yaml"
+    )
+
+    if not schema_path.exists():
+        raise FileNotFoundError(f"MomentBuilder schema not found: {schema_path}")
+
+    with open(schema_path) as f:
+        agent_schema = yaml.safe_load(f)
+
+    # Prepare input data for agent
+    input_data = {
+        "resources": [
+            {
+                "id": str(r.id),
+                "name": r.name,
+                "category": r.category,
+                "content": r.content,
+                "created_at": (
+                    r.created_at.isoformat() if r.created_at else None
+                ),
+            }
+            for r in resources
+        ],
+        "messages": [
+            {
+                "id": str(m.id),
+                "session_id": m.session_id,
+                "message_type": m.message_type,
+                "content": m.content,
+                "created_at": m.created_at.isoformat(),
+            }
+            for m in messages
+        ],
+    }
+
+    # Create and run MomentBuilder agent
+    agent_runtime = await create_agent(
+        agent_schema_override=agent_schema,
+        model_override=default_model,  # type: ignore[arg-type]
+    )
+
+    result = await agent_runtime.run(json.dumps(input_data, indent=2))
+
+    # Serialize result (critical for Pydantic models!)
+    output_data = serialize_agent_result(result.output)
+
+    # Type guard: ensure we have a dict
+    if not isinstance(output_data, dict):
+        raise ValueError(f"Expected dict from MomentBuilder agent, got {type(output_data)}")
+
+    # Extract moments
+    moments_data = output_data.get("moments", [])
+    analysis_summary = output_data.get("analysis_summary", "")
+
+    logger.info(
+        f"MomentBuilder extracted {len(moments_data)} moments. Summary: {analysis_summary}"
+    )
+
+    # Create Moment entities
+    created_moments = []
+    total_edges = 0
+
+    for moment_data in moments_data:
+        # Map created_at/resource_ends_timestamp to starts_timestamp/ends_timestamp
+        starts_ts_str = moment_data.get("created_at")
+        ends_ts_str = moment_data.get("resource_ends_timestamp")
+
+        if not starts_ts_str:
+            logger.warning(f"Skipping moment without start timestamp: {moment_data.get('name')}")
+            continue
+
+        starts_ts = datetime.fromisoformat(starts_ts_str.replace("Z", "+00:00"))
+        ends_ts = (
+            datetime.fromisoformat(ends_ts_str.replace("Z", "+00:00"))
+            if ends_ts_str
+            else None
+        )
+
+        # Build graph edges to source resources
+        source_resource_ids = moment_data.get("source_resource_ids", [])
+        source_session_ids = moment_data.get("source_session_ids", [])
+
+        graph_edges = []
+
+        # Add edges to source resources
+        for resource_id in source_resource_ids:
+            graph_edges.append(
+                {
+                    "dst": resource_id,
+                    "rel_type": "extracted_from",
+                    "weight": 1.0,
+                    "properties": {
+                        "entity_type": "resource",
+                        "extraction_method": "moment_builder_agent",
+                    },
+                    "created_at": datetime.utcnow().isoformat(),
+                }
+            )
+
+        # Add edges to source sessions
+        for session_id in source_session_ids:
+            graph_edges.append(
+                {
+                    "dst": session_id,
+                    "rel_type": "extracted_from",
+                    "weight": 0.8,
+                    "properties": {
+                        "entity_type": "session",
+                        "extraction_method": "moment_builder_agent",
+                    },
+                    "created_at": datetime.utcnow().isoformat(),
+                }
+            )
+
+        # Create Moment entity
+        moment = Moment(
+            id=str(uuid4()),
+            tenant_id=user_id,  # Set tenant_id = user_id
+            user_id=user_id,
+            name=moment_data.get("name"),
+            moment_type=moment_data.get("moment_type"),
+            category=moment_data.get("moment_type"),  # Use moment_type as category
+            starts_timestamp=starts_ts,
+            ends_timestamp=ends_ts,
+            present_persons=[
+                Person(id=p["id"], name=p["name"], role=p.get("comment"))
+                for p in moment_data.get("present_persons", [])
+            ],
+            emotion_tags=moment_data.get("emotion_tags", []),
+            topic_tags=moment_data.get("topic_tags", []),
+            summary=moment_data.get("content"),  # Use content as summary
+            source_resource_ids=source_resource_ids,
+            graph_edges=graph_edges,
+            created_at=datetime.utcnow(),
+            updated_at=datetime.utcnow(),
+        )
+
+        # Save to database (embeddings auto-generated by embedding worker)
+        await moment_repo.upsert(moment)
+        created_moments.append(moment)
+        total_edges += len(graph_edges)
+
+        logger.debug(
+            f"Created moment: {moment.name} ({moment.moment_type}) with {len(graph_edges)} edges"
+        )
+
+    return {
+        "user_id": user_id,
+        "lookback_hours": lookback_hours,
+        "resources_queried": len(resources),
+        "messages_queried": len(messages),
+        "moments_created": len(created_moments),
+        "graph_edges_added": total_edges,
+        "analysis_summary": analysis_summary,
+        "status": "success",
+    }

rem/services/dreaming/ontology_service.py

@@ -0,0 +1,54 @@
+"""
+Ontology Service - Extracts domain-specific knowledge from files.
+
+Finds files processed within lookback window and applies matching
+OntologyConfig rules to extract structured knowledge using custom agents.
+"""
+
+from typing import Any, Optional
+
+
+async def extract_ontologies(
+    user_id: str,
+    lookback_hours: int = 24,
+    limit: Optional[int] = None,
+) -> dict[str, Any]:
+    """
+    Extract domain-specific knowledge from files using custom agents.
+
+    Finds files processed within lookback window and applies matching
+    OntologyConfig rules to extract structured knowledge.
+
+    Process:
+    1. Query REM for files processed by this user (lookback window)
+    2. For each file, find matching OntologyConfig rules
+    3. Load agent schemas from database
+    4. Execute agents on file content
+    5. Generate embeddings for extracted data
+    6. Store Ontology entities
+
+    Args:
+        user_id: User to process
+        lookback_hours: Hours to look back (default: 24)
+        limit: Max files to process
+
+    Returns:
+        Statistics about ontology extraction
+    """
+    # TODO: Implement using REM query API + OntologyExtractorService
+    # Query files with timestamp filter and processing_status='completed'
+    # Load matching OntologyConfigs from database
+    # Use OntologyExtractorService to extract ontologies
+    # Generate embeddings for embedding_text field
+
+    # Stub implementation
+    return {
+        "user_id": user_id,
+        "lookback_hours": lookback_hours,
+        "files_queried": 0,
+        "configs_matched": 0,
+        "ontologies_created": 0,
+        "embeddings_generated": 0,
+        "agent_calls_made": 0,
+        "status": "stub_not_implemented",
+    }

rem/services/dreaming/user_model_service.py

@@ -0,0 +1,297 @@
+"""
+User Model Service - Updates user profiles from activity.
+
+Analyzes recent sessions, moments, and resources to generate
+comprehensive user profile summaries using LLM analysis.
+"""
+
+import json
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Any, Optional
+
+import yaml
+from loguru import logger
+
+from ...agentic.providers.pydantic_ai import create_agent
+from ...agentic.serialization import serialize_agent_result
+from ...models.entities.moment import Moment
+from ...models.entities.resource import Resource
+from ...models.entities.message import Message
+from ...models.entities.user import User
+from ...services.postgres.repository import Repository
+from ...services.postgres.service import PostgresService
+
+
+async def update_user_model(
+    user_id: str,
+    db: PostgresService,
+    default_model: str = "gpt-4o",
+    time_window_days: int = 30,
+    max_messages: int = 100,
+    max_moments: int = 20,
+    max_resources: int = 20,
+) -> dict[str, Any]:
+    """
+    Update user model from recent activity.
+
+    Reads recent messages, moments, and resources to generate
+    a comprehensive user profile summary using LLM analysis.
+
+    Process:
+    1. Query PostgreSQL for recent messages, moments, resources for this user
+    2. Load UserProfileBuilder agent schema
+    3. Generate user profile using LLM
+    4. Update User entity with profile data and metadata
+    5. Add graph edges to key resources and moments
+
+    Args:
+        user_id: User to process
+        db: Database service (already connected)
+        default_model: LLM model for analysis (default: gpt-4o)
+        time_window_days: Days to look back for activity (default: 30)
+        max_messages: Max messages to analyze
+        max_moments: Max moments to include
+        max_resources: Max resources to include
+
+    Returns:
+        Statistics about user model update
+    """
+    cutoff = datetime.utcnow() - timedelta(days=time_window_days)
+
+    # Create repositories
+    message_repo = Repository(Message, "messages", db=db)
+    moment_repo = Repository(Moment, "moments", db=db)
+    resource_repo = Repository(Resource, "resources", db=db)
+    user_repo = Repository(User, "users", db=db)
+
+    # Build filters using user_id
+    filters = {"user_id": user_id}
+
+    # Query recent messages
+    messages = await message_repo.find(
+        filters=filters,
+        order_by="created_at DESC",
+        limit=max_messages,
+    )
+    # Filter by cutoff (both are UTC-naive)
+    messages = [m for m in messages if m.created_at >= cutoff]
+
+    # Query recent moments
+    moments = await moment_repo.find(
+        filters=filters,
+        order_by="starts_timestamp DESC",
+        limit=max_moments,
+    )
+    # Filter by cutoff (both are UTC-naive)
+    moments = [m for m in moments if m.starts_timestamp >= cutoff]
+
+    # Query recent resources
+    resources = await resource_repo.find(
+        filters=filters,
+        order_by="created_at DESC",
+        limit=max_resources,
+    )
+    # Filter by cutoff (both are UTC-naive)
+    resources = [r for r in resources if r.created_at and r.created_at >= cutoff]
+
+    if not messages and not moments and not resources:
+        return {
+            "user_id": user_id,
+            "time_window_days": time_window_days,
+            "messages_analyzed": 0,
+            "moments_included": 0,
+            "resources_included": 0,
+            "user_updated": False,
+            "status": "no_data",
+        }
+
+    logger.info(
+        f"Building user profile for {user_id}: "
+        f"{len(messages)} messages, {len(moments)} moments, {len(resources)} resources"
+    )
+
+    # Load UserProfileBuilder agent schema
+    schema_path = (
+        Path(__file__).parent.parent.parent
+        / "schemas"
+        / "agents"
+        / "core"
+        / "user-profile-builder.yaml"
+    )
+
+    if not schema_path.exists():
+        raise FileNotFoundError(f"UserProfileBuilder schema not found: {schema_path}")
+
+    with open(schema_path) as f:
+        agent_schema = yaml.safe_load(f)
+
+    # Prepare input data for agent
+    input_data = {
+        "user_id": user_id,
+        "time_window_days": time_window_days,
+        "messages": [
+            {
+                "id": str(m.id),
+                "session_id": m.session_id,
+                "message_type": m.message_type,
+                "content": m.content[:500] if m.content else "",  # Limit for token efficiency
+                "created_at": m.created_at.isoformat(),
+            }
+            for m in messages
+        ],
+        "moments": [
+            {
+                "id": str(m.id),
+                "name": m.name,
+                "moment_type": m.moment_type,
+                "emotion_tags": m.emotion_tags,
+                "topic_tags": m.topic_tags,
+                "present_persons": [
+                    {"id": str(p.id), "name": p.name, "role": p.role}
+                    for p in m.present_persons
+                ],
+                "starts_timestamp": m.starts_timestamp.isoformat(),
+                "summary": m.summary[:300] if m.summary else "",
+            }
+            for m in moments
+        ],
+        "resources": [
+            {
+                "id": str(r.id),
+                "name": r.name,
+                "category": r.category,
+                "content": r.content[:1000] if r.content else "",  # First 1000 chars
+                "created_at": (
+                    r.created_at.isoformat() if r.created_at else None
+                ),
+            }
+            for r in resources
+        ],
+    }
+
+    # Create and run UserProfileBuilder agent
+    agent_runtime = await create_agent(
+        agent_schema_override=agent_schema,
+        model_override=default_model,  # type: ignore[arg-type]
+    )
+
+    result = await agent_runtime.run(json.dumps(input_data, indent=2))
+
+    # Serialize result (critical for Pydantic models!)
+    raw_result = serialize_agent_result(result.output)
+
+    # Ensure result is a dict (agent should always return structured data)
+    if not isinstance(raw_result, dict):
+        raise ValueError(f"Expected dict from user profile agent, got {type(raw_result)}")
+
+    profile_data: dict[str, Any] = raw_result
+
+    logger.info(
+        f"Generated user profile. Summary: {profile_data.get('summary', '')[:100]}..."
+    )
+
+    # Get or create User entity
+    # Use find() with user_id filter instead of get_by_id() since user_id is a string, not UUID
+    existing_users = await user_repo.find(filters={"user_id": user_id}, limit=1)
+    user = existing_users[0] if existing_users else None
+
+    if not user:
+        # Create new user (id will be auto-generated as UUID)
+        user = User(
+            tenant_id=user_id,  # Set tenant_id = user_id
+            user_id=user_id,
+            name=user_id,  # Default to user_id, can be updated later
+            metadata={},
+            graph_edges=[],
+            created_at=datetime.utcnow(),
+            updated_at=datetime.utcnow(),
+        )
+
+    # Update user metadata with full profile
+    user.metadata = user.metadata or {}
+    user.metadata.update(
+        {
+            "profile": profile_data,
+            "profile_generated_at": datetime.utcnow().isoformat(),
+            "profile_time_window_days": time_window_days,
+        }
+    )
+
+    # Update user model fields from profile
+    user.summary = profile_data.get("summary", "")
+    user.interests = [
+        area for area in profile_data.get("expertise_areas", [])
+    ] + [
+        interest for interest in profile_data.get("learning_interests", [])
+    ]
+    user.preferred_topics = profile_data.get("recommended_tags", [])
+
+    # Determine activity level based on data volume
+    total_activity = len(messages) + len(moments) + len(resources)
+    if total_activity >= 50:
+        user.activity_level = "active"
+    elif total_activity >= 10:
+        user.activity_level = "moderate"
+    else:
+        user.activity_level = "inactive"
+
+    user.last_active_at = datetime.utcnow()
+
+    # Build graph edges to key resources and moments
+    from .utils import merge_graph_edges
+
+    graph_edges = []
+
+    # Add edges to recent resources (top 5)
+    for resource in resources[:5]:
+        graph_edges.append(
+            {
+                "dst": str(resource.id),  # Convert UUID to string
+                "rel_type": "recently_worked_on",
+                "weight": 0.8,
+                "properties": {
+                    "entity_type": "resource",
+                    "dst_name": resource.name,
+                    "dst_category": resource.category,
+                },
+                "created_at": datetime.utcnow().isoformat(),
+            }
+        )
+
+    # Add edges to recent moments (top 5)
+    for moment in moments[:5]:
+        graph_edges.append(
+            {
+                "dst": str(moment.id),  # Convert UUID to string
+                "rel_type": "participated_in",
+                "weight": 0.9,
+                "properties": {
+                    "entity_type": "moment",
+                    "dst_name": moment.name,
+                    "dst_moment_type": moment.moment_type,
+                },
+                "created_at": datetime.utcnow().isoformat(),
+            }
+        )
+
+    # Merge edges with existing
+    user.graph_edges = merge_graph_edges(user.graph_edges or [], graph_edges)
+    user.updated_at = datetime.utcnow()
+
+    # Save user
+    await user_repo.upsert(user)
+
+    return {
+        "user_id": user_id,
+        "time_window_days": time_window_days,
+        "messages_analyzed": len(messages),
+        "moments_included": len(moments),
+        "resources_included": len(resources),
+        "current_projects": len(profile_data.get("current_projects", [])),
+        "technical_stack_size": len(profile_data.get("technical_stack", [])),
+        "key_collaborators": len(profile_data.get("key_collaborators", [])),
+        "graph_edges_added": len(graph_edges),
+        "user_updated": True,
+        "status": "success",
+    }

rem/services/dreaming/utils.py

@@ -0,0 +1,39 @@
+"""
+Dreaming utilities - Common functions for dreaming services.
+"""
+
+from typing import Any
+
+
+def merge_graph_edges(
+    existing_edges: list[dict[str, Any]], new_edges: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    """
+    Merge graph edges with deduplication.
+
+    Keep highest weight edge for each (dst, rel_type) pair.
+    This prevents duplicate edges while preserving the strongest relationships.
+
+    Args:
+        existing_edges: Current edges on the resource
+        new_edges: New edges to add
+
+    Returns:
+        Merged list of edges with duplicates removed
+    """
+    edges_map: dict[tuple[str, str], dict[str, Any]] = {}
+
+    # Add existing edges
+    for edge in existing_edges:
+        key = (edge.get("dst", ""), edge.get("rel_type", ""))
+        edges_map[key] = edge
+
+    # Add new edges (replace if higher weight)
+    for edge in new_edges:
+        key = (edge.get("dst", ""), edge.get("rel_type", ""))
+        if key not in edges_map or edge.get("weight", 0) > edges_map[key].get(
+            "weight", 0
+        ):
+            edges_map[key] = edge
+
+    return list(edges_map.values())

rem/services/embeddings/__init__.py

@@ -0,0 +1,11 @@
+"""Embeddings service for background embedding generation."""
+
+from .api import generate_embedding, generate_embedding_async
+from .worker import EmbeddingTask, EmbeddingWorker
+
+__all__ = [
+    "EmbeddingTask",
+    "EmbeddingWorker",
+    "generate_embedding",
+    "generate_embedding_async",
+]