apsimo 1.3.0__py3-none-any.whl

apsimo/intelligence/components/research_orchestrator.py

@@ -0,0 +1,358 @@
+"""Research Orchestrator — coordinate research across multiple sources.
+
+Capabilities:
+    - Source prioritization
+    - Result synthesis
+    - Citation tracking
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Awaitable, Callable, Dict, List, Optional
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class SourceType(str, Enum):
+    """Types of research sources."""
+
+    WEB = "web"
+    KNOWLEDGE_GRAPH = "knowledge_graph"
+    MEMORY = "memory"
+    API = "api"
+
+
+@dataclass
+class ResearchSource:
+    """A research source.
+
+    Attributes:
+        type: Category of source
+        name: Human-readable source name
+        priority: Trust/quality weight (0-1, higher = more trusted)
+        rate_limit: Optional requests-per-minute cap
+    """
+
+    type: SourceType
+    name: str
+    priority: float = 0.5
+    rate_limit: Optional[int] = None
+
+
+@dataclass
+class ResearchResult:
+    """A result from a single research source.
+
+    Attributes:
+        source: Name of the source that produced this result
+        content: The actual content/finding
+        confidence: How confident the source is (0-1)
+        timestamp: When the result was obtained
+        citations: Reference URLs or identifiers
+    """
+
+    source: str
+    content: str
+    confidence: float
+    timestamp: datetime = field(default_factory=datetime.now)
+    citations: List[str] = field(default_factory=list)
+
+
+@dataclass
+class ResearchReport:
+    """Aggregated research report.
+
+    Attributes:
+        query: The original research query
+        results: Individual results from each source
+        synthesized_summary: Combined summary across all results
+        confidence: Overall confidence score (average of results)
+    """
+
+    query: str
+    results: List[ResearchResult] = field(default_factory=list)
+    synthesized_summary: Optional[str] = None
+    confidence: float = 0.0
+
+
+class ResearchOrchestrator:
+    """Orchestrate multi-source research.
+
+    Manages a registry of research sources, queries them in priority
+    order, synthesizes findings, and tracks citations.
+
+    Source routing:
+    - MEMORY: queries graph memory via ``graph.recall()``
+    - KNOWLEDGE_GRAPH: returns structured knowledge graph context
+    - WEB: queries the injected ``SearchOrchestrator`` if one is provided.
+    - API: delegates to a handler registered via ``register_api_handler``.
+
+    Args:
+        graph_client: Colony graph client for knowledge graph queries
+        event_bus: Colony event bus for research lifecycle events
+        search_orchestrator: Optional SearchOrchestrator for WEB sources.
+    """
+
+    def __init__(
+        self,
+        graph_client: Any = None,
+        event_bus: Any = None,
+        search_orchestrator: Any = None,
+    ) -> None:
+        self.graph = graph_client
+        self.events = event_bus
+        self.search = search_orchestrator
+        self._sources: List[ResearchSource] = []
+        self._api_handlers: Dict[str, Callable[[str], Awaitable[Optional[ResearchResult]]]] = {}
+
+    def register_api_handler(
+        self,
+        name: str,
+        handler: Callable[[str], Awaitable[Optional[ResearchResult]]],
+    ) -> None:
+        """Register an async handler for an API-typed source.
+
+        The handler receives the raw query string and must return a
+        ``ResearchResult`` or ``None``.
+        """
+        self._api_handlers[name] = handler
+
+    def register_source(self, source: ResearchSource) -> None:
+        """Register a research source.
+
+        Args:
+            source: Source to add to the registry
+        """
+        self._sources.append(source)
+        logger.debug("Registered research source: %s (%s)", source.name, source.type.value)
+
+    def unregister_source(self, name: str) -> None:
+        """Remove a research source by name.
+
+        Args:
+            name: Name of the source to remove
+        """
+        self._sources = [s for s in self._sources if s.name != name]
+
+    async def research(self, query: str, max_sources: int = 3) -> ResearchReport:
+        """Conduct research across registered sources.
+
+        Queries sources in priority order (up to max_sources),
+        then synthesizes results into a unified report.
+
+        Args:
+            query: The research question
+            max_sources: Maximum number of sources to query
+
+        Returns:
+            Aggregated research report with synthesized findings
+        """
+        sorted_sources = sorted(
+            self._sources,
+            key=lambda s: s.priority,
+            reverse=True,
+        )[:max_sources]
+
+        results: List[ResearchResult] = []
+        for source in sorted_sources:
+            result = await self._query_source(source, query)
+            if result:
+                results.append(result)
+
+        summary = await self._synthesize(results, query)
+
+        confidence = (
+            sum(r.confidence for r in results) / len(results) if results else 0.0
+        )
+
+        report = ResearchReport(
+            query=query,
+            results=results,
+            synthesized_summary=summary,
+            confidence=confidence,
+        )
+
+        logger.debug(
+            "Research complete for '%s': %d results, confidence=%.2f",
+            query[:50],
+            len(results),
+            confidence,
+        )
+        return report
+
+    async def _query_source(
+        self,
+        source: ResearchSource,
+        query: str,
+    ) -> Optional[ResearchResult]:
+        """Query a single source, routing by source type.
+
+        Returns:
+            A ResearchResult if the source returned data, else None.
+        """
+        try:
+            if source.type == SourceType.MEMORY:
+                return await self._query_memory_source(source, query)
+            elif source.type == SourceType.KNOWLEDGE_GRAPH:
+                return await self._query_graph_source(source, query)
+            elif source.type == SourceType.WEB:
+                return await self._query_web_source(source, query)
+            elif source.type == SourceType.API:
+                return await self._query_api_source(source, query)
+        except Exception as exc:
+            logger.warning("Source %s query failed: %s", source.name, exc)
+        return None
+
+    async def _query_web_source(
+        self,
+        source: ResearchSource,
+        query: str,
+    ) -> Optional[ResearchResult]:
+        """Query the injected SearchOrchestrator for web results."""
+        if self.search is None or not getattr(self.search, "has_providers", False):
+            return None
+        try:
+            results = await self.search.search(query, max_results=5)
+            if not results:
+                return None
+            parts: List[str] = []
+            citations: List[str] = []
+            for r in results[:5]:
+                title = getattr(r, "title", "") or ""
+                snippet = getattr(r, "snippet", "") or ""
+                if title or snippet:
+                    parts.append(f"{title}: {snippet}" if title else snippet)
+                url = getattr(r, "url", "")
+                if url:
+                    citations.append(url)
+            content = "\n".join(parts).strip()
+            if not content:
+                return None
+            return ResearchResult(
+                source=source.name,
+                content=content,
+                confidence=min(0.8, source.priority * 0.8),
+                citations=citations,
+            )
+        except Exception as exc:
+            logger.debug("Web source query failed: %s", exc)
+            return None
+
+    async def _query_api_source(
+        self,
+        source: ResearchSource,
+        query: str,
+    ) -> Optional[ResearchResult]:
+        """Dispatch to a registered API handler (if any) for this source."""
+        handler = self._api_handlers.get(source.name)
+        if handler is None:
+            return None
+        try:
+            return await handler(query)
+        except Exception as exc:
+            logger.debug("API source '%s' handler failed: %s", source.name, exc)
+            return None
+
+    async def _query_memory_source(
+        self,
+        source: ResearchSource,
+        query: str,
+    ) -> Optional[ResearchResult]:
+        """Query graph memory for relevant information."""
+        try:
+            memories = await self.graph.recall(query, limit=5)
+            if not memories:
+                return None
+            # Combine up to 3 most relevant memories into one result
+            content = "\n".join(
+                m.get("content", "") if isinstance(m, dict) else str(m)
+                for m in memories[:3]
+                if m
+            )
+            if not content.strip():
+                return None
+            return ResearchResult(
+                source=source.name,
+                content=content,
+                confidence=min(0.9, source.priority * 0.9),
+            )
+        except Exception as exc:
+            logger.debug("Memory source query failed: %s", exc)
+            return None
+
+    async def _query_graph_source(
+        self,
+        source: ResearchSource,
+        query: str,
+    ) -> Optional[ResearchResult]:
+        """Query the knowledge graph using entity and type recall.
+
+        Runs RECALL_BY_ENTITY (treating the query as an entity name) and
+        RECALL_BY_TYPE (treating the query as a memory type) against Neo4j,
+        then combines up to 3 results into a single ResearchResult.
+        """
+        from apsimo.intelligence.graph.queries import RECALL_BY_ENTITY, RECALL_BY_TYPE
+
+        try:
+            memories: List[Any] = []
+            async with self.graph.driver.session(database=self.graph.database) as session:
+                entity_result = await session.run(
+                    RECALL_BY_ENTITY,
+                    entity_name=query,
+                    min_strength=0.1,
+                    limit=5,
+                )
+                memories.extend([record["memory"] async for record in entity_result])
+
+                type_result = await session.run(
+                    RECALL_BY_TYPE,
+                    memory_type=query,
+                    min_strength=0.1,
+                    limit=5,
+                )
+                memories.extend([record["memory"] async for record in type_result])
+
+            if not memories:
+                return None
+
+            content = "\n".join(
+                m.get("content", "") if isinstance(m, dict) else str(m)
+                for m in memories[:3]
+                if m
+            )
+            if not content.strip():
+                return None
+
+            return ResearchResult(
+                source=source.name,
+                content=content,
+                confidence=min(0.85, source.priority * 0.85),
+            )
+        except Exception as exc:
+            logger.debug("Graph source query failed: %s", exc)
+            return None
+
+    async def _synthesize(
+        self,
+        results: List[ResearchResult],
+        query: str,
+    ) -> Optional[str]:
+        """Synthesize results into a coherent summary.
+
+        Combines the top results (by confidence) into a single narrative,
+        prefixed with a summary header.
+        """
+        if not results:
+            return None
+
+        sorted_results = sorted(results, key=lambda r: r.confidence, reverse=True)
+        parts = [f"Research summary for '{query[:60]}' — {len(results)} source(s):"]
+        for r in sorted_results[:3]:
+            if r.content:
+                excerpt = r.content[:200].replace("\n", " ")
+                parts.append(f"[{r.source}] {excerpt}")
+
+        return "\n".join(parts)

apsimo/intelligence/components/self_directed_thinker.py

@@ -0,0 +1,221 @@
+"""Self-directed thinking — Colony's internal cognition phase (v0.17.0).
+
+The InitiativeEngine's generators are data-reactive: they scan existing
+graph state for known shapes of work (stale goals, neglected contacts,
+failing CI). The thinker is the complement: on a slow cadence it hands
+an LLM a snapshot of Colony's situation — goals, commitments, recent
+work, capability gaps, open questions — and asks what work SHOULD exist
+that nothing has surfaced yet. Output becomes ordinary initiatives that
+flow through the same store, delivery, and approval machinery as
+everything else.
+
+Safety posture:
+
+- Disabled unless ``COLONY_ENABLE_INTERNAL_THINKING=true``.
+- Runs at most once per ``COLONY_THINKING_INTERVAL_SECS`` (default 1h),
+  not every loop tick.
+- At most ``COLONY_THINKING_MAX_INITIATIVES`` (default 3) per cycle.
+- Proposals are capped at priority 0.85 and may never carry an
+  ``action_hint`` — thought-up work always lands as review/decide
+  initiatives, never as directly executable agent actions. Anything
+  mutating or outbound therefore still crosses the action registry and
+  the owner-approval gate before an agent can touch it.
+- Dedup keys remember recent proposals so repeated thinking cycles
+  don't re-pitch the same idea.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+import time
+import uuid
+from typing import Any, Dict, List, Optional
+
+from apsimo.intelligence.components.initiative_engine import (
+    Initiative,
+    InitiativeType,
+)
+
+logger = logging.getLogger(__name__)
+
+# Types the LLM may propose. Deliberately excludes AGENT_ACTION (and
+# anything else that auto-executes) — see module docstring.
+_ALLOWED_TYPES = {
+    "research": InitiativeType.RESEARCH,
+    "task": InitiativeType.TASK,
+    "project": InitiativeType.PROJECT,
+    "system": InitiativeType.SYSTEM,
+    "knowledge_acquisition": InitiativeType.KNOWLEDGE_ACQUISITION,
+    "capability_gap": InitiativeType.CAPABILITY_GAP,
+}
+
+_MAX_PRIORITY = 0.85
+_RECENT_KEY_CAP = 200
+
+
+class SelfDirectedThinker:
+    """Generates novel initiatives from periodic LLM reflection.
+
+    Prompted through the shared cognition charter (role "thinker"); the
+    output schema REQUIRES per-item confidence (trust-engine calibration
+    input) and grounding evidence (items without it are dropped).
+    """
+
+    def __init__(self, router: Any,
+                 interval_secs: Optional[int] = None,
+                 max_per_cycle: Optional[int] = None,
+                 self_brief_fn: Any = None,
+                 boundaries_fn: Any = None) -> None:
+        self._router = router
+        self._interval = interval_secs if interval_secs is not None else int(
+            os.environ.get("COLONY_THINKING_INTERVAL_SECS", "3600"))
+        self._max = max_per_cycle if max_per_cycle is not None else int(
+            os.environ.get("COLONY_THINKING_MAX_INITIATIVES", "3"))
+        # Optional zero-arg callables -> str, evaluated fresh each cycle.
+        self._self_brief_fn = self_brief_fn
+        self._boundaries_fn = boundaries_fn
+        self._last_run: Optional[float] = None
+        self._recent_keys: List[str] = []
+
+    # -- scheduling ---------------------------------------------------------
+
+    def due(self, now: Optional[float] = None) -> bool:
+        now = now if now is not None else time.monotonic()
+        if self._last_run is None:
+            return True
+        return (now - self._last_run) >= self._interval
+
+    def mark_ran(self, now: Optional[float] = None) -> None:
+        self._last_run = now if now is not None else time.monotonic()
+
+    # -- thinking -----------------------------------------------------------
+
+    async def think(self, situation: Dict[str, Any]) -> List[Initiative]:
+        """One reflection cycle: situation report in, initiatives out."""
+        if self._router is None:
+            return []
+        prompt = self._build_messages(situation)
+        try:
+            response = await self._router.complete(
+                prompt, context={"task": "internal_thinking"})
+            content = getattr(response, "content", "") or ""
+        except Exception as exc:
+            logger.warning("Self-directed thinking LLM call failed: %s", exc)
+            return []
+
+        initiatives: List[Initiative] = []
+        for item in self._parse(content)[: self._max]:
+            initiative = self._to_initiative(item)
+            if initiative is None:
+                continue
+            if initiative.dedup_key in self._recent_keys:
+                logger.debug("Thinking dedup: %s", initiative.dedup_key)
+                continue
+            self._remember(initiative.dedup_key)
+            initiatives.append(initiative)
+        if initiatives:
+            logger.info("Self-directed thinking produced %d initiative(s): %s",
+                        len(initiatives),
+                        "; ".join(i.description[:60] for i in initiatives))
+        return initiatives
+
+    # -- internals ----------------------------------------------------------
+
+    def _build_messages(self, situation: Dict[str, Any]) -> List[dict]:
+        sections = []
+        for name, value in situation.items():
+            if value in (None, "", [], {}):
+                continue
+            try:
+                rendered = json.dumps(value, default=str, indent=None)[:4000]
+            except (TypeError, ValueError):
+                rendered = str(value)[:4000]
+            sections.append(f"## {name}\n{rendered}")
+        report = "\n\n".join(sections) or "(no situation data available)"
+
+        def _call(fn) -> str:
+            if fn is None:
+                return ""
+            try:
+                return str(fn() or "")
+            except Exception:
+                return ""
+
+        from apsimo.cognition.charter import build_system_prompt
+        system = build_system_prompt(
+            "thinker",
+            self_brief=_call(self._self_brief_fn) or None,
+            boundaries=_call(self._boundaries_fn) or None,
+            max_items=self._max, allowed=sorted(_ALLOWED_TYPES))
+        return [
+            {"role": "system", "content": system},
+            {"role": "user", "content": f"SITUATION REPORT\n\n{report}"},
+        ]
+
+    @staticmethod
+    def _parse(content: str) -> List[dict]:
+        text = content.strip()
+        # Tolerate markdown fences and leading prose despite instructions.
+        if "```" in text:
+            match = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL)
+            if match:
+                text = match.group(1).strip()
+        if not text.startswith("["):
+            match = re.search(r"\[.*\]", text, re.DOTALL)
+            if not match:
+                return []
+            text = match.group(0)
+        try:
+            data = json.loads(text)
+        except json.JSONDecodeError:
+            logger.warning("Thinking output was not valid JSON; dropping")
+            return []
+        return [d for d in data if isinstance(d, dict)] if isinstance(data, list) else []
+
+    def _to_initiative(self, item: dict) -> Optional[Initiative]:
+        title = str(item.get("title", "")).strip()
+        type_key = str(item.get("type", "")).strip().lower()
+        rationale = str(item.get("rationale", "")).strip()
+        evidence = str(item.get("evidence", "")).strip()
+        if not title or type_key not in _ALLOWED_TYPES:
+            return None
+        # Charter contract: a proposal without grounding evidence does not
+        # ship (the schema demands the report lines that ground it).
+        if not evidence:
+            logger.debug("Thinking item dropped (no grounding evidence): %s",
+                         title[:60])
+            return None
+        try:
+            priority = float(item.get("priority", 0.5))
+        except (TypeError, ValueError):
+            priority = 0.5
+        priority = max(0.0, min(_MAX_PRIORITY, priority))
+        try:
+            confidence = max(0.0, min(1.0, float(item.get("confidence", 0.5))))
+        except (TypeError, ValueError):
+            confidence = 0.5
+        slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")[:60]
+        return Initiative(
+            id=f"init-think-{uuid.uuid4().hex[:12]}",
+            type=_ALLOWED_TYPES[type_key],
+            description=title,
+            priority=priority,
+            rationale=f"[self-directed thinking] {rationale}" if rationale
+            else "[self-directed thinking]",
+            action_hint=None,  # never directly executable (see docstring)
+            dedup_key=f"thinking:{slug}",
+            # Stated confidence + evidence ride along for trust-engine
+            # calibration (stated vs realized) and provenance.
+            trigger_data={"stated_confidence": confidence,
+                          "evidence": evidence[:400]},
+        )
+
+    def _remember(self, key: Optional[str]) -> None:
+        if not key:
+            return
+        self._recent_keys.append(key)
+        if len(self._recent_keys) > _RECENT_KEY_CAP:
+            self._recent_keys = self._recent_keys[-_RECENT_KEY_CAP:]