hypermind 0.18.0__tar.gz → 0.20.0__tar.gz

{hypermind-0.18.0 → hypermind-0.20.0}/.gitignore

@@ -143,3 +143,6 @@ data/relata/
 
 # Benchmark run artefacts (large raw traces). Slim traces + report.html kept locally.
 experiments/output/
+
+# Local scratch experiments (reports, benchmarks) — not shipped
+experiments/

{hypermind-0.18.0 → hypermind-0.20.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypermind
-Version: 0.18.0
+Version: 0.20.0
 Summary: Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile
 Project-URL: Homepage, https://github.com/ZySec-AI/hypermind
 Project-URL: Documentation, https://github.com/ZySec-AI/hypermind/tree/main/docs

{hypermind-0.18.0 → hypermind-0.20.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hypermind"
-version = "0.18.0"
+version = "0.20.0"
 description = "Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile"
 readme = "README.md"
 requires-python = ">=3.11,<3.14"

{hypermind-0.18.0 → hypermind-0.20.0}/src/hypermind/__init__.py

@@ -73,7 +73,7 @@ open = HyperMindAgent.open
 #: is held until the §External & human gates clear (IETF I-D, IANA,
 #: third-party audit, OpenSSF acceptance, ≥3 pilots) — at that point we
 #: cut ``1.0.0`` GA. See ROADMAP.md §Status snapshot.
-__version__ = "0.18.0"
+__version__ = "0.20.0"
 
 #: SLSA build-provenance level committed for every release artefact.
 #: Cosign keyless signing + slsa-github-generator + PEP 740 attestations

{hypermind-0.18.0 → hypermind-0.20.0}/src/hypermind/orgs/__init__.py

@@ -64,8 +64,15 @@ from hypermind.orgs.receipt import VerifyResult, build_receipt, verify_receipt
 from hypermind.orgs.report import MemberStanding, OrgReport
 from hypermind.orgs.retrieval import (
     MentionGraph,
+    chat_ner_extractor,
+    cosine,
     extract_mentions,
     graph_retriever,
+    hybrid_retriever,
+    llm_ner_extractor,
+    openai_embedder,
+    pioneer_ner_extractor,
+    reciprocal_rank_fusion,
     relata_retriever,
 )
 from hypermind.orgs.sim import OrgSim
@@ -114,16 +121,23 @@ __all__ = [
     "build_deliberator",
     "build_receipt",
     "build_tool",
+    "chat_ner_extractor",
     "compute_scorecard",
+    "cosine",
     "declarative_grants",
     "default_context_builder",
     "extract_mentions",
     "get_interaction",
     "graph_retriever",
     "graph_snapshot",
+    "hybrid_retriever",
     "list_tools",
+    "llm_ner_extractor",
+    "openai_embedder",
+    "pioneer_ner_extractor",
     "policies_for_role",
     "random_grants",
+    "reciprocal_rank_fusion",
     "register_tool",
     "relata_retriever",
     "unregister_tool",

hypermind-0.20.0/src/hypermind/orgs/retrieval.py

@@ -0,0 +1,352 @@
+"""Graph- and identity-aware retrieval (entity mentions).
+
+Plain relevance ranking retrieves claims whose *text* overlaps the question.
+That misses the graph: a claim about "ML-DSA" is relevant to a question about
+"post-quantum signatures" even with no word overlap, if the two are linked by
+a shared entity elsewhere in the ledger.
+
+This module extracts **entity mentions** from findings, builds a
+claim↔entity **mention graph**, and offers a graph-aware retriever that:
+  1. pulls entities from the question,
+  2. expands them by co-mention (1-hop graph neighbours),
+  3. returns claims mentioning any of those entities, ranked by match count.
+
+The entity extractor is injectable (swap in an LLM/NER or Relata identity
+resolution); the default is a dependency-free heuristic. A Relata-backed
+retriever adapter (``relata_retriever``) uses the server's real
+``search_knowledge`` / ``find_connections`` graph.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+from collections.abc import Callable
+from typing import Any
+
+# Capitalised phrases (proper nouns), ALLCAPS acronyms, and hyphenated
+# technical tokens — a cheap entity-mention heuristic with no NLP dependency.
+_PHRASE = re.compile(r"\b([A-Z][A-Za-z0-9]+(?:[- ][A-Z][A-Za-z0-9]+)*)\b")
+_ACRONYM = re.compile(r"\b([A-Z]{2,}(?:-[A-Z0-9]+)*)\b")
+_TECH = re.compile(r"\b([a-z]+(?:-[a-z0-9]+)+)\b")  # e.g. ab-initio, post-quantum
+_STOP = {
+    "The",
+    "A",
+    "An",
+    "This",
+    "That",
+    "It",
+    "We",
+    "I",
+    "In",
+    "On",
+    "Of",
+    "To",
+    "For",
+    "And",
+    "But",
+    "Or",
+    "If",
+    "Is",
+    "Are",
+    "Would",
+    "Should",
+    "Will",
+    "Does",
+    "Do",
+    "Can",
+    "Could",
+    "May",
+    "Might",
+    "Question",
+    "Answer",
+}
+
+
+def extract_mentions(text: str, *, max_mentions: int = 24) -> set[str]:
+    """Extract entity mentions from ``text`` (normalised lowercase)."""
+    if not text:
+        return set()
+    out: set[str] = set()
+    for pat in (_ACRONYM, _PHRASE, _TECH):
+        for m in pat.findall(text):
+            token = m.strip()
+            if token in _STOP or len(token) < 3:
+                continue
+            out.add(token.lower())
+            if len(out) >= max_mentions:
+                return out
+    return out
+
+
+class MentionGraph:
+    """A bipartite claim↔entity graph, built incrementally as claims arrive."""
+
+    def __init__(self, extractor: Callable[[str], set[str]] | None = None) -> None:
+        self._extract = extractor or extract_mentions
+        self._claim_entities: dict[str, set[str]] = {}
+        self._entity_claims: dict[str, set[str]] = defaultdict(set)
+
+    def add(self, claim_kid: str, text: str) -> set[str]:
+        ents = self._extract(text)
+        if ents:
+            self._claim_entities[claim_kid] = ents
+            for e in ents:
+                self._entity_claims[e].add(claim_kid)
+        return ents
+
+    def connected(self, entity: str) -> set[str]:
+        """Entities co-mentioned with ``entity`` in some claim (1-hop)."""
+        neighbours: set[str] = set()
+        for kid in self._entity_claims.get(entity, ()):
+            neighbours |= self._claim_entities.get(kid, set())
+        neighbours.discard(entity)
+        return neighbours
+
+    def claims_for(self, entities: set[str]) -> set[str]:
+        kids: set[str] = set()
+        for e in entities:
+            kids |= self._entity_claims.get(e, set())
+        return kids
+
+    def stats(self) -> dict[str, int]:
+        return {
+            "claims": len(self._claim_entities),
+            "entities": len(self._entity_claims),
+            "edges": sum(len(v) for v in self._claim_entities.values()),
+        }
+
+
+def graph_retriever(
+    graph: MentionGraph,
+    claim_lookup: Callable[[str], tuple[str, str] | None],
+    *,
+    hops: int = 1,
+) -> Callable[[str, int], list[tuple[str, str]]]:
+    """A retriever that expands the question's entities through the graph.
+
+    ``claim_lookup(kid)`` returns ``(author_label, finding_text)`` or None.
+    """
+
+    def retrieve(question: str, k: int) -> list[tuple[str, str]]:
+        q_ents = extract_mentions(question)
+        expanded = set(q_ents)
+        for _ in range(max(0, hops)):
+            for e in list(expanded):
+                expanded |= graph.connected(e)
+        candidate_kids = graph.claims_for(expanded)
+        scored: list[tuple[int, str, str]] = []
+        for kid in candidate_kids:
+            hit = claim_lookup(kid)
+            if hit is None:
+                continue
+            label, text = hit
+            matched = len(expanded & extract_mentions(text))
+            scored.append((matched, label, text))
+        scored.sort(key=lambda t: t[0], reverse=True)
+        return [(label, text) for _n, label, text in scored[:k]]
+
+    return retrieve
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+    """Cosine similarity of two vectors (0 if either is empty/zero)."""
+    if not a or not b:
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    na = sum(x * x for x in a) ** 0.5
+    nb = sum(y * y for y in b) ** 0.5
+    return dot / (na * nb) if na and nb else 0.0
+
+
+def openai_embedder(
+    client: Any, *, model: str = "text-embedding-3-small", batch_size: int = 96
+) -> Callable[[list[str]], list[list[float]]]:
+    """Build a sync embed function from an OpenAI-compatible client.
+
+    Works with OpenAI or any `/embeddings`-compatible endpoint (e.g. OpenRouter
+    ``openai/text-embedding-3-small``). Batches inputs. Plug the result into
+    ``OrgSim(embedder=...)`` for semantic (not lexical) retrieval.
+    """
+
+    def embed(texts: list[str]) -> list[list[float]]:
+        vecs: list[list[float]] = []
+        for i in range(0, len(texts), batch_size):
+            resp = client.embeddings.create(model=model, input=texts[i : i + batch_size])
+            vecs.extend(d.embedding for d in resp.data)
+        return vecs
+
+    return embed
+
+
+def llm_ner_extractor(nlp: Any) -> Callable[[str], set[str]]:
+    """Entity extractor for :class:`MentionGraph` backed by a loaded spaCy-style
+    model (``nlp(text).ents``). Falls back to the heuristic on any error."""
+
+    def extract(text: str) -> set[str]:
+        try:
+            return {e.text.lower() for e in nlp(text).ents if len(e.text) >= 3}
+        except Exception:
+            return extract_mentions(text)
+
+    return extract
+
+
+_NER_SYS = (
+    "Extract the named entities (people, organizations, locations, products, "
+    "and technical terms) from the user's text. Respond ONLY with a JSON object "
+    '{"entities": ["..."]} and nothing else.'
+)
+
+
+def _parse_entities(content: str) -> set[str]:
+    """Lenient parse of an NER model's response into a set of entities."""
+    import json
+
+    m = re.search(r"\{.*\}|\[.*\]", content, re.S)
+    if m:
+        try:
+            data = json.loads(m.group(0))
+            if isinstance(data, dict):
+                data = data.get("entities", [])
+            if isinstance(data, list):
+                return {str(e).strip().lower() for e in data if len(str(e).strip()) >= 3}
+        except Exception:
+            pass
+    return extract_mentions(content)
+
+
+def chat_ner_extractor(
+    client: Any, *, model: str, timeout_s: float = 30.0
+) -> Callable[[str], set[str]]:
+    """NER extractor backed by any OpenAI-compatible **chat** endpoint.
+
+    Prompts the model to return a JSON entity list; parses leniently and falls
+    back to the heuristic on any error. Plug into ``MentionGraph`` /
+    ``OrgSim(mention_extractor=...)``.
+    """
+
+    def extract(text: str) -> set[str]:
+        if not text:
+            return set()
+        try:
+            resp = client.chat.completions.create(
+                model=model,
+                messages=[
+                    {"role": "system", "content": _NER_SYS},
+                    {"role": "user", "content": text},
+                ],
+                timeout=timeout_s,
+            )
+            content = (resp.choices[0].message.content or "") if resp.choices else ""
+            return _parse_entities(content) or extract_mentions(text)
+        except Exception:
+            return extract_mentions(text)
+
+    return extract
+
+
+def pioneer_ner_extractor(
+    *,
+    api_key: str | None = None,
+    model: str = "pioneer/auto",
+    base_url: str = "https://api.pioneer.ai/v1",
+    client: Any = None,
+    timeout_s: float = 30.0,
+) -> Callable[[str], set[str]]:
+    """NER extractor backed by pioneer.ai (OpenAI-compatible chat).
+
+    Builds an OpenAI client pointed at pioneer unless one is injected. Usage::
+
+        from hypermind.orgs import pioneer_ner_extractor, OrgSim
+        ner = pioneer_ner_extractor(api_key=os.environ["PIONEER_API_KEY"])
+        OrgSim(model, ..., mention_extractor=ner, use_graph_retrieval=True)
+    """
+    if client is None:
+        from openai import OpenAI
+
+        client = OpenAI(base_url=base_url, api_key=api_key)
+    return chat_ner_extractor(client, model=model, timeout_s=timeout_s)
+
+
+def reciprocal_rank_fusion(
+    ranked_lists: list[list[tuple[str, str]]], *, top_k: int, rrf_k: int = 60
+) -> list[tuple[str, str]]:
+    """Fuse several ranked ``(label, text)`` lists into one (RRF).
+
+    Each list is one retrieval signal (dense, graph, lexical). An item's fused
+    score is ``sum(1 / (rrf_k + rank))`` across the lists it appears in, so a
+    doc ranked well by *several* signals beats one ranked first by a single
+    signal. Identity is the text (dedups the same finding from different
+    signals). ``rrf_k=60`` is the standard constant (Cormack et al.).
+    """
+    scores: dict[str, float] = defaultdict(float)
+    label_of: dict[str, str] = {}
+    for ranked in ranked_lists:
+        for rank, (label, text) in enumerate(ranked):
+            scores[text] += 1.0 / (rrf_k + rank)
+            label_of.setdefault(text, label)
+    ordered = sorted(scores, key=lambda t: scores[t], reverse=True)
+    return [(label_of[t], t) for t in ordered[:top_k]]
+
+
+def hybrid_retriever(
+    *retrievers: Callable[[str, int], list[tuple[str, str]]],
+    reranker: Callable[[str, list[tuple[str, str]]], list[tuple[str, str]]] | None = None,
+    rrf_k: int = 60,
+    fetch_mult: int = 3,
+) -> Callable[[str, int], list[tuple[str, str]]]:
+    """Fuse multiple retrievers (dense + graph + lexical) via RRF, then rerank.
+
+    Borrows the hybrid-search shape used by state-of-the-art RAG stacks, over
+    *our* data: each sub-retriever is any ``(question, k) -> [(label, text)]``
+    (e.g. ``relata_retriever(p, "search_knowledge")`` for Relata's semantic
+    index and ``relata_retriever(p, "find_connections")`` for its graph). Each
+    is asked for ``fetch_mult*k`` candidates, results are fused with
+    :func:`reciprocal_rank_fusion`, and an optional cross-encoder ``reranker``
+    (``(question, cands) -> reordered``) does the final ordering.
+
+        r = hybrid_retriever(
+            relata_retriever(provider, mode="search_knowledge"),
+            relata_retriever(provider, mode="find_connections"),
+        )
+        OrgSim(model, ..., retriever=r)
+    """
+
+    def retrieve(question: str, k: int) -> list[tuple[str, str]]:
+        # ponytail: sub-retrievers run sequentially; parallelise with threads
+        # only if one signal's latency dominates and it measurably hurts.
+        want = max(k, k * fetch_mult)
+        lists = [list(r(question, want)) for r in retrievers]
+        fused = reciprocal_rank_fusion(lists, top_k=want, rrf_k=rrf_k)
+        if reranker is not None:
+            fused = list(reranker(question, fused))
+        return fused[:k]
+
+    return retrieve
+
+
+def relata_retriever(
+    provider: Any, *, mode: str = "search_knowledge"
+) -> Callable[[str, int], list[tuple[str, str]]]:
+    """Retriever backed by Relata's real graph/identity via a RelataToolProvider.
+
+    ``mode`` is ``search_knowledge`` (semantic + keyword over the KB) or
+    ``find_connections`` (graph traversal from question entities).
+    """
+
+    def retrieve(question: str, k: int) -> list[tuple[str, str]]:
+        if mode == "find_connections":
+            ent = next(iter(extract_mentions(question)), question)
+            result = provider.call("find_connections", {"entity": ent, "limit": k})
+        else:
+            result = provider.call("search_knowledge", {"query": question})
+        rows = result.get("results") or result.get("rows") or []
+        out: list[tuple[str, str]] = []
+        for r in rows[:k]:
+            label = str(r.get("source") or r.get("entity") or r.get("id") or "relata")
+            text = str(r.get("content") or r.get("text") or r.get("finding") or r)
+            out.append((label, text))
+        return out
+
+    return retrieve

{hypermind-0.18.0 → hypermind-0.20.0}/src/hypermind/orgs/sim.py

@@ -121,6 +121,8 @@ class OrgSim:
         retrieval_k: int = 5,
         retriever: Callable[[str, int], list[tuple[str, str]]] | None = None,
         use_graph_retrieval: bool = False,
+        embedder: Callable[[list[str]], list[list[float]]] | None = None,
+        mention_extractor: Callable[[str], set[str]] | None = None,
         members: int | None = None,
     ) -> None:
         self.model = model.scaled(members) if members is not None else model
@@ -172,7 +174,11 @@ class OrgSim:
         self._use_graph_retrieval = use_graph_retrieval
         from hypermind.orgs.retrieval import MentionGraph
 
-        self._mentions = MentionGraph()  # claim↔entity graph, built on publish
+        # claim↔entity graph; pass a real NER (e.g. pioneer_ner_extractor) via
+        # mention_extractor, else the dependency-free heuristic is used.
+        self._mentions = MentionGraph(extractor=mention_extractor)
+        self._embedder = embedder  # semantic embeddings for retrieval (else lexical)
+        self._embed_cache: dict[str, list[float]] = {}  # kid → vector
 
     async def _emit(self, kind: str, **data: Any) -> None:
         """Fire a structured event to the on_event sink (async or sync)."""
@@ -225,6 +231,8 @@ class OrgSim:
         # recent). Priority: injected retriever > entity-graph > lexical.
         if self._retriever is not None:
             relevant = list(self._retriever(question, self._retrieval_k))
+        elif self._embedder is not None:
+            relevant = await self._embedding_retrieve(question, self._retrieval_k)
         elif self._use_graph_retrieval:
             relevant = self._graph_retrieve(question, self._retrieval_k)
         else:
@@ -618,6 +626,29 @@ class OrgSim:
 
         return graph_retriever(self._mentions, lookup)(question, k)
 
+    async def _embedding_retrieve(self, question: str, k: int) -> list[tuple[str, str]]:
+        """Semantic retrieval: embed the question + findings, rank by cosine.
+
+        Findings are embedded lazily and cached by kid; only new ones + the
+        question are sent to the embedder each call, batched in one request.
+        The (blocking) embedder runs in a thread so it doesn't stall the loop.
+        """
+        from hypermind.orgs.retrieval import cosine
+
+        items = [(kid, c.label, c.finding) for kid, c in self._claims.items() if c.finding]
+        if not items:
+            return []
+        uncached = [t for (kid, _l, t) in items if kid not in self._embed_cache]
+        vecs = await asyncio.to_thread(self._embedder, [question, *uncached])
+        q_vec = vecs[0]
+        i = 1
+        for kid, _l, _t in items:
+            if kid not in self._embed_cache:
+                self._embed_cache[kid] = vecs[i]
+                i += 1
+        ranked = sorted(items, key=lambda it: cosine(q_vec, self._embed_cache[it[0]]), reverse=True)
+        return [(label, text) for _kid, label, text in ranked[:k]]
+
     _TOKEN = re.compile(r"[a-z0-9]{3,}")
 
     def _relevant_findings(