apsimo 1.3.0__py3-none-any.whl

apsimo/research/search/duckduckgo.py

@@ -0,0 +1,103 @@
+"""DuckDuckGo HTML search provider — zero-config fallback.
+
+Uses the lite HTML endpoint (https://html.duckduckgo.com/html/) because
+it requires no API key and no JS rendering. This is best-effort — DDG
+can change their markup at any time — so any failure degrades to an
+empty result set rather than raising.
+"""
+
+from __future__ import annotations
+
+import html as _html
+import logging
+import os
+import re
+from typing import List
+
+import httpx
+
+from apsimo.research.search.base import SearchProvider, SearchResult
+
+logger = logging.getLogger(__name__)
+
+
+_RESULT_RE = re.compile(
+    r'<a rel="nofollow" class="result__a" href="(?P<url>[^"]+)"[^>]*>'
+    r"(?P<title>.*?)</a>"
+    r'.*?<a class="result__snippet"[^>]*>(?P<snippet>.*?)</a>',
+    re.DOTALL,
+)
+
+_TAG_RE = re.compile(r"<[^>]+>")
+
+
+def _strip_tags(s: str) -> str:
+    return _html.unescape(_TAG_RE.sub("", s)).strip()
+
+
+class DuckDuckGoProvider(SearchProvider):
+    """HTML-endpoint DuckDuckGo provider.
+
+    No API key required. Fragile against markup changes — callers
+    MUST tolerate empty results silently.
+    """
+
+    _ENDPOINT = "https://html.duckduckgo.com/html/"
+
+    def __init__(self, timeout_secs: float = 8.0) -> None:
+        self._timeout = timeout_secs
+
+    @property
+    def name(self) -> str:
+        return "duckduckgo"
+
+    @property
+    def rate_limit_per_minute(self) -> int:
+        # DuckDuckGo has no documented limit; stay polite.
+        return 20
+
+    async def search(self, query: str, max_results: int = 5) -> List[SearchResult]:
+        params = {"q": query}
+        try:
+            # UA is deployment-overridable (COLONY_SEARCH_USER_AGENT); the
+            # default is a neutral project identifier with no deployment URL.
+            _ua = os.environ.get(
+                "COLONY_SEARCH_USER_AGENT",
+                "Mozilla/5.0 (compatible; ColonyAI research crawler)")
+            async with httpx.AsyncClient(
+                timeout=self._timeout,
+                headers={
+                    "User-Agent": _ua,
+                    "Accept": "text/html,application/xhtml+xml",
+                },
+                follow_redirects=True,
+            ) as client:
+                resp = await client.post(self._ENDPOINT, data=params)
+                resp.raise_for_status()
+                body = resp.text
+        except Exception as exc:
+            logger.warning("DuckDuckGo request failed: %s", exc)
+            return []
+
+        results: List[SearchResult] = []
+        for i, m in enumerate(_RESULT_RE.finditer(body)):
+            if i >= max_results:
+                break
+            url = m.group("url")
+            # DDG wraps external URLs in a redirect; strip it when present.
+            if url.startswith("//duckduckgo.com/l/?uddg="):
+                from urllib.parse import parse_qs, unquote, urlparse
+                qs = parse_qs(urlparse("https:" + url).query)
+                uddg = qs.get("uddg", [""])[0]
+                if uddg:
+                    url = unquote(uddg)
+            results.append(
+                SearchResult(
+                    title=_strip_tags(m.group("title")),
+                    url=url,
+                    snippet=_strip_tags(m.group("snippet")),
+                    source=self.name,
+                    rank=i + 1,
+                )
+            )
+        return results

apsimo/research/search/orchestrator.py

@@ -0,0 +1,119 @@
+"""Search orchestrator — routes queries, handles rate limits, caches results."""
+
+from __future__ import annotations
+
+import logging
+import time
+from typing import Dict, List, Optional
+
+from apsimo.research.search.base import SearchProvider, SearchResult
+from apsimo.research.search.cache import SearchCache
+
+logger = logging.getLogger(__name__)
+
+
+class SearchOrchestrator:
+    """Routes search queries to providers, handles rate limits and caching.
+
+    If no providers are configured, returns empty results (graceful degradation).
+    """
+
+    def __init__(
+        self,
+        providers: Optional[List[SearchProvider]] = None,
+        cache_ttl: int = 3600,
+        max_cache_entries: int = 500,
+    ):
+        self._providers: Dict[str, SearchProvider] = {}
+        if providers:
+            for p in providers:
+                self._providers[p.name] = p
+        self._cache = SearchCache(ttl_seconds=cache_ttl, max_entries=max_cache_entries)
+        self._rate_tracker: Dict[str, List[float]] = {name: [] for name in self._providers}
+        self._default_provider = next(iter(self._providers), None)
+
+    @property
+    def has_providers(self) -> bool:
+        return len(self._providers) > 0
+
+    def add_provider(self, provider: SearchProvider) -> None:
+        """Add a search provider."""
+        self._providers[provider.name] = provider
+        self._rate_tracker[provider.name] = []
+        if not self._default_provider:
+            self._default_provider = provider
+
+    async def search(
+        self,
+        query: str,
+        max_results: int = 5,
+        provider: str = "",
+    ) -> List[SearchResult]:
+        """Execute a search query.
+
+        Args:
+            query: Search query string
+            max_results: Maximum number of results
+            provider: Preferred provider name (empty = auto-select)
+
+        Returns:
+            List of search results, empty if no providers available.
+        """
+        if not self._providers:
+            return []
+
+        # Check cache
+        cached = self._cache.get(query)
+        if cached is not None:
+            logger.debug("Search cache hit for '%s'", query[:50])
+            return cached[:max_results]
+
+        # Select provider
+        prov = self._select_provider(provider)
+        if not prov:
+            return []
+
+        # Rate limit check
+        if not self._check_rate_limit(prov.name):
+            prov = self._fallback_provider(prov.name)
+            if not prov:
+                logger.warning("All search providers rate-limited")
+                return []
+
+        # Execute search
+        try:
+            results = await prov.search(query, max_results)
+            self._cache.put(query, results)
+            logger.info("Search via %s: '%s' → %d results", prov.name, query[:50], len(results))
+            return results
+        except Exception as e:
+            logger.warning("Search via %s failed: %s", prov.name, e)
+            return []
+
+    def _select_provider(self, preferred: str) -> Optional[SearchProvider]:
+        if preferred and preferred in self._providers:
+            return self._providers[preferred]
+        return self._default_provider
+
+    def _fallback_provider(self, exclude: str) -> Optional[SearchProvider]:
+        for name, prov in self._providers.items():
+            if name != exclude and self._check_rate_limit(name):
+                return prov
+        return None
+
+    def _check_rate_limit(self, name: str) -> bool:
+        if name not in self._rate_tracker:
+            return False
+        now = time.time()
+        # Clean up entries older than 60 seconds
+        self._rate_tracker[name] = [t for t in self._rate_tracker[name] if now - t < 60]
+        prov = self._providers.get(name)
+        if not prov:
+            return False
+        return len(self._rate_tracker[name]) < prov.rate_limit_per_minute
+
+    def clear_cache(self) -> None:
+        self._cache.clear()
+
+    def list_providers(self) -> List[str]:
+        return list(self._providers.keys())

apsimo/research/search/serpapi.py

@@ -0,0 +1,59 @@
+"""SerpAPI search provider."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from apsimo.research.search.base import SearchProvider, SearchResult
+
+logger = logging.getLogger(__name__)
+
+try:
+    import httpx
+    HAS_HTTPX = True
+except ImportError:
+    HAS_HTTPX = False
+
+
+class SerpAPIProvider(SearchProvider):
+    """Google search via SerpAPI."""
+
+    def __init__(self, api_key: str):
+        self._api_key = api_key
+        self._base_url = "https://serpapi.com/search"
+
+    @property
+    def name(self) -> str:
+        return "serpapi"
+
+    async def search(self, query: str, max_results: int = 5) -> list[SearchResult]:
+        if not HAS_HTTPX:
+            logger.warning("httpx not installed — cannot use SerpAPI")
+            return []
+
+        try:
+            async with httpx.AsyncClient(timeout=15) as client:
+                resp = await client.get(self._base_url, params={
+                    "q": query,
+                    "api_key": self._api_key,
+                    "engine": "google",
+                    "num": max_results,
+                    "output": "json",
+                })
+                resp.raise_for_status()
+                data = resp.json()
+                results = []
+                for i, r in enumerate(data.get("organic_results", [])[:max_results]):
+                    results.append(SearchResult(
+                        title=r.get("title", ""),
+                        url=r.get("link", ""),
+                        snippet=r.get("snippet", ""),
+                        content=r.get("snippet"),  # SerpAPI doesn't return full content
+                        source="serpapi",
+                        rank=i,
+                    ))
+                return results
+        except Exception as e:
+            logger.warning("SerpAPI search failed: %s", e)
+            return []

apsimo/research/search/tavily.py

@@ -0,0 +1,59 @@
+"""Tavily search provider — designed for AI agents, returns clean content."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from apsimo.research.search.base import SearchProvider, SearchResult
+
+logger = logging.getLogger(__name__)
+
+try:
+    import httpx
+    HAS_HTTPX = True
+except ImportError:
+    HAS_HTTPX = False
+
+
+class TavilyProvider(SearchProvider):
+    """Tavily search API — optimized for AI agent use cases."""
+
+    def __init__(self, api_key: str):
+        self._api_key = api_key
+        self._base_url = "https://api.tavily.com/search"
+
+    @property
+    def name(self) -> str:
+        return "tavily"
+
+    async def search(self, query: str, max_results: int = 5) -> list[SearchResult]:
+        if not HAS_HTTPX:
+            logger.warning("httpx not installed — cannot use Tavily")
+            return []
+
+        try:
+            async with httpx.AsyncClient(timeout=15) as client:
+                resp = await client.post(self._base_url, json={
+                    "api_key": self._api_key,
+                    "query": query,
+                    "max_results": max_results,
+                    "include_raw_content": True,
+                    "search_depth": "advanced",
+                })
+                resp.raise_for_status()
+                data = resp.json()
+                return [
+                    SearchResult(
+                        title=r.get("title", ""),
+                        url=r.get("url", ""),
+                        snippet=r.get("content", "")[:500],
+                        content=r.get("raw_content"),
+                        source="tavily",
+                        rank=i,
+                    )
+                    for i, r in enumerate(data.get("results", []))
+                ]
+        except Exception as e:
+            logger.warning("Tavily search failed: %s", e)
+            return []

apsimo/research/synthesizer.py

@@ -0,0 +1,309 @@
+"""Research synthesizer — Stage 3 of the Research-to-Artifact Pipeline.
+
+Runs gathered evidence through Colony's intelligence synthesis layer to
+produce a structured ``SynthesisReport`` containing validated, novelty-ranked
+insights and annotated contradictions.
+
+Steps:
+  1. Connection discovery  — temporal, causal, entity, behavioural patterns
+  2. Cross-domain analysis — groups evidence by domain, extracts insights
+  3. Novelty scoring       — rank by formula: 0.4*(1/freq) + 0.3*domain_dist + 0.3*conf_delta
+  4. Quality validation    — min evidence count, confidence thresholds, recency
+  5. Contradiction         — conflicting evidence annotated with provenance
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from apsimo.research.gatherer import EvidenceItem, EvidencePackage, SourceType
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Synthesis models
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class DomainInsight:
+    """A validated, novelty-ranked insight derived from evidence."""
+
+    id: str
+    title: str
+    summary: str
+    domains: List[str]
+    supporting_citations: List[str]       # Evidence item citations
+    confidence: float = 0.0               # 0.0–1.0
+    novelty_score: float = 0.0            # 0.0–1.0
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Contradiction:
+    """Two evidence items with conflicting claims."""
+
+    id: str
+    claim_a: str
+    claim_b: str
+    citation_a: str
+    citation_b: str
+    topic: str
+    confidence_a: float = 0.5
+    confidence_b: float = 0.5
+
+
+@dataclass
+class SynthesisReport:
+    """Output of the synthesis stage."""
+
+    goal_id: str
+    insights: List[DomainInsight]
+    contradictions: List[Contradiction]
+    evidence_count: int
+    source_breakdown: Dict[str, int]      # SourceType.value -> count
+    synthesis_confidence: float           # 0.0–1.0, aggregate
+    graph_entities_referenced: List[str]  # Neo4j node IDs
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+
+
+# ---------------------------------------------------------------------------
+# Synthesis configuration
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SynthesisConfig:
+    min_evidence_for_insight: int = 2
+    min_confidence: float = 0.55
+    min_novelty: float = 0.30
+    max_insights: int = 50
+    timeout_seconds: float = 90.0
+
+
+# ---------------------------------------------------------------------------
+# Synthesizer
+# ---------------------------------------------------------------------------
+
+
+class ResearchSynthesizer:
+    """Synthesize an EvidencePackage into a SynthesisReport.
+
+    Tries to use Colony's intelligence synthesis layer when available;
+    falls back to a lightweight built-in implementation for offline/test use.
+    """
+
+    def __init__(self, config: Optional[SynthesisConfig] = None) -> None:
+        self.config = config or SynthesisConfig()
+
+    async def synthesize(
+        self,
+        evidence: EvidencePackage,
+        goal_id: str,
+        goal_text: str = "",
+    ) -> SynthesisReport:
+        """Run evidence through the synthesis pipeline.
+
+        Args:
+            evidence: List of EvidenceItem objects (already deduplicated).
+            goal_id: Unique ID of the parent research run.
+            goal_text: Original research goal text (used for context).
+
+        Returns:
+            A ``SynthesisReport`` with ranked insights and contradictions.
+        """
+        # Filter out flagged items before any LLM exposure
+        safe_evidence = [e for e in evidence if e.is_safe()]
+        flagged_count = len(evidence) - len(safe_evidence)
+        if flagged_count:
+            logger.warning(
+                "Synthesizer: excluded %d flagged evidence items (pii/injection)",
+                flagged_count,
+            )
+
+        source_breakdown: Dict[str, int] = {}
+        for item in safe_evidence:
+            source_breakdown[item.source_type.value] = (
+                source_breakdown.get(item.source_type.value, 0) + 1
+            )
+
+        insights = await self._extract_insights(safe_evidence, goal_text)
+        contradictions = self._detect_contradictions(safe_evidence)
+
+        # Enforce limits
+        insights = insights[: self.config.max_insights]
+
+        aggregate_confidence = (
+            sum(i.confidence for i in insights) / len(insights) if insights else 0.0
+        )
+
+        graph_entities: List[str] = []
+        for item in safe_evidence:
+            if item.source_type == SourceType.GRAPH:
+                node_id = item.citation.replace("graph://", "")
+                if node_id and node_id not in graph_entities:
+                    graph_entities.append(node_id)
+
+        return SynthesisReport(
+            goal_id=goal_id,
+            insights=insights,
+            contradictions=contradictions,
+            evidence_count=len(safe_evidence),
+            source_breakdown=source_breakdown,
+            synthesis_confidence=aggregate_confidence,
+            graph_entities_referenced=graph_entities,
+        )
+
+    async def _extract_insights(
+        self,
+        evidence: EvidencePackage,
+        goal_text: str,
+    ) -> List[DomainInsight]:
+        """Extract and rank insights from evidence."""
+        # Attempt to use Colony's CrossDomainAnalyzer
+        try:
+            return await self._run_colony_synthesis(evidence, goal_text)
+        except Exception as exc:
+            logger.debug("CrossDomainAnalyzer unavailable (%s), using built-in", exc)
+            return self._builtin_extract(evidence, goal_text)
+
+    async def _run_colony_synthesis(
+        self,
+        evidence: EvidencePackage,
+        goal_text: str,
+    ) -> List[DomainInsight]:
+        """Run Colony intelligence synthesis layer."""
+        from apsimo.intelligence.synthesis.cross_domain_analyzer import CrossDomainAnalyzer
+        from apsimo.intelligence.synthesis.novelty_scorer import NoveltyScorer
+        from apsimo.intelligence.synthesis.insight_validator import InsightValidator
+
+        # Build a minimal data structure the analyzer expects
+        signals = [
+            {
+                "domain": item.source_type.value,
+                "content": item.content,
+                "timestamp": item.retrieved_at.isoformat(),
+                "confidence": item.relevance_score,
+                "source": item.citation,
+            }
+            for item in evidence
+        ]
+
+        analyzer = CrossDomainAnalyzer()
+        raw_insights = await analyzer.analyze(signals)
+
+        validator = InsightValidator(
+            min_confidence=self.config.min_confidence,
+            min_evidence_count=self.config.min_evidence_for_insight,
+        )
+        scorer = NoveltyScorer()
+
+        results: List[DomainInsight] = []
+        import secrets as _secrets
+
+        for raw in raw_insights:
+            validation = validator.validate(raw)
+            if not validation.valid:
+                continue
+            novelty = scorer.score(raw)
+            if novelty.score < self.config.min_novelty:
+                continue
+            results.append(
+                DomainInsight(
+                    id=_secrets.token_hex(6),
+                    title=getattr(raw, "title", str(raw)[:80]),
+                    summary=getattr(raw, "summary", str(raw)),
+                    domains=getattr(raw, "domains", []),
+                    supporting_citations=[],
+                    confidence=getattr(raw, "confidence", 0.6),
+                    novelty_score=novelty.score,
+                )
+            )
+
+        return sorted(results, key=lambda x: x.novelty_score, reverse=True)
+
+    def _builtin_extract(
+        self,
+        evidence: EvidencePackage,
+        goal_text: str,
+    ) -> List[DomainInsight]:
+        """Lightweight built-in insight extractor (no external deps)."""
+        import secrets as _secrets
+
+        cfg = self.config
+        insights: List[DomainInsight] = []
+
+        # Group evidence by source type
+        by_source: Dict[str, List[EvidenceItem]] = {}
+        for item in evidence:
+            by_source.setdefault(item.source_type.value, []).append(item)
+
+        for source_type, items in by_source.items():
+            if len(items) < cfg.min_evidence_for_insight:
+                continue
+            # One insight per source type, summarizing top items
+            top_items = sorted(items, key=lambda x: x.relevance_score, reverse=True)[:5]
+            combined = " ".join(i.content[:200] for i in top_items)
+            avg_confidence = sum(i.relevance_score for i in top_items) / len(top_items)
+
+            if avg_confidence < cfg.min_confidence:
+                continue
+
+            # Compute a simple novelty score from source diversity
+            novelty = min(1.0, 0.3 + 0.1 * len(items))
+            if novelty < cfg.min_novelty:
+                continue
+
+            insights.append(
+                DomainInsight(
+                    id=_secrets.token_hex(6),
+                    title=f"Findings from {source_type} sources",
+                    summary=combined[:500],
+                    domains=[source_type],
+                    supporting_citations=[i.citation for i in top_items],
+                    confidence=avg_confidence,
+                    novelty_score=novelty,
+                )
+            )
+
+        return sorted(insights, key=lambda x: x.novelty_score, reverse=True)
+
+    def _detect_contradictions(
+        self,
+        evidence: EvidencePackage,
+    ) -> List[Contradiction]:
+        """Detect pairs of evidence items with potentially conflicting claims.
+
+        Uses a heuristic: items from different sources with similar topics
+        but significantly different relevance scores are flagged.
+        """
+        import secrets as _secrets
+
+        contradictions: List[Contradiction] = []
+        # Simple heuristic — compare web vs graph items on same topic keywords
+        web_items = [e for e in evidence if e.source_type == SourceType.WEB]
+        graph_items = [e for e in evidence if e.source_type == SourceType.GRAPH]
+
+        for w in web_items[:5]:
+            for g in graph_items[:5]:
+                # If relevance scores differ significantly, flag as potential contradiction
+                if abs(w.relevance_score - g.relevance_score) > 0.4:
+                    contradictions.append(
+                        Contradiction(
+                            id=_secrets.token_hex(6),
+                            claim_a=w.content[:200],
+                            claim_b=g.content[:200],
+                            citation_a=w.citation,
+                            citation_b=g.citation,
+                            topic="relevance discrepancy",
+                            confidence_a=w.relevance_score,
+                            confidence_b=g.relevance_score,
+                        )
+                    )
+
+        return contradictions[:10]  # Cap at 10 contradictions

apsimo/router/__init__.py

@@ -0,0 +1,30 @@
+"""Colony LLM Cost Router — intelligent model tier selection.
+
+Route LLM requests to the cheapest capable model based on prompt complexity.
+Expected cost savings: 30–40% vs always using the largest model.
+
+Usage::
+
+    from apsimo.router import LLMRouter, ModelTier
+
+    router = LLMRouter()
+    response = await router.complete(messages, context={"task": "summarise"})
+"""
+
+from apsimo.router.tiers import ModelTier, TierConfig, DEFAULT_TIERS
+from apsimo.router.complexity_scorer import ComplexityScorer, ComplexitySignals
+from apsimo.router.router import LLMRouter, LLMResponse
+from apsimo.router.self_learning import RouterSelfLearner
+from apsimo.router.fallback import FallbackHandler
+
+__all__ = [
+    "LLMRouter",
+    "LLMResponse",
+    "ModelTier",
+    "TierConfig",
+    "DEFAULT_TIERS",
+    "ComplexityScorer",
+    "ComplexitySignals",
+    "RouterSelfLearner",
+    "FallbackHandler",
+]