hypermind 0.33.0__tar.gz → 0.34.0__tar.gz

{hypermind-0.33.0 → hypermind-0.34.0}/CHANGELOG.md

@@ -16,6 +16,18 @@ changes only on major-version bumps.
 
 _Current trunk._
 
+## [0.34.0] — 2026-07-09
+
+### Added
+- **100% evidence retention** — every claim now persists a COMPLETE evidence record: untruncated reasoning, verbatim LLM payload, and **full raw tool output** (previously capped at 500 chars). Kept off the size-bounded signed claim and stored durably in the agent's Relata notebook. `OrgReport.evidence()` / `evidence_for(kid)` expose it. (`responders/tools.py`, `responders/structured.py` adds `reasoning_full` + uncapped `raw_json`, `orgs/sim.py`).
+- **Structured citations + real grounding** (`hypermind.orgs.evidence`) — parses raw tool output (searchapi JSON + DuckDuckGo text) into de-duplicated `Citation`s ranked by multi-expert corroboration. `OrgSim._retrieve` now grounds the judge/answer on **real source snippets**, not only the swarm's own summaries. `OrgReport.references()` is the study bibliography. Pure parsing, cached by claim count — no added LLM latency.
+- **Self-refined master decomposition** — `hypermind.orgs.prompts` (planner prompt + curated few-shot exemplars, one-/multi-shot), `charter.plan_research(shots=, critique=)`, and `hypermind.orgs.refine` (`score_plan` + `plan_and_refine`) re-plan the decomposition on a structural critique (distinct areas, role diversity, tool breadth, on-topic, vs-k) **before** spending the simulation. No framework, reproducible.
+- `RelataMemoryStore(single_tenant=True)` — omit `X-Organization-Id` for single-tenant / trial (`max_tenants:0`) servers, isolating by `session_id` (agent kid); `OrgSim(relata_single_tenant=…)`.
+
+### Fixed
+- **MCP transport** — 16 MB stdio buffer for MCP subprocesses; large Google Scholar/Patents results no longer trip asyncio's 64 KB `LimitOverrunError` (`orgs/mcp_config.py`).
+- Tool-call traces key the tool name under `tool` (matching production `ToolCallingResponder`); tests aligned.
+
 ## [0.33.0] — 2026-07-09
 
 ### Added

{hypermind-0.33.0 → hypermind-0.34.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypermind
-Version: 0.33.0
+Version: 0.34.0
 Summary: Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile
 Project-URL: Homepage, https://github.com/ZySec-AI/hypermind
 Project-URL: Documentation, https://github.com/ZySec-AI/hypermind/tree/main/docs

{hypermind-0.33.0 → hypermind-0.34.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hypermind"
-version = "0.33.0"
+version = "0.34.0"
 description = "Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile"
 readme = "README.md"
 requires-python = ">=3.11,<3.14"

{hypermind-0.33.0 → hypermind-0.34.0}/src/hypermind/__init__.py

@@ -73,7 +73,7 @@ open = HyperMindAgent.open
 #: is held until the §External & human gates clear (IETF I-D, IANA,
 #: third-party audit, OpenSSF acceptance, ≥3 pilots) — at that point we
 #: cut ``1.0.0`` GA. See ROADMAP.md §Status snapshot.
-__version__ = "0.33.0"
+__version__ = "0.34.0"
 
 #: SLSA build-provenance level committed for every release artefact.
 #: Cosign keyless signing + slsa-github-generator + PEP 740 attestations

{hypermind-0.33.0 → hypermind-0.34.0}/src/hypermind/mind/memory_store.py

@@ -48,6 +48,7 @@ class RelataMemoryStore:
         purpose: str = _PURPOSE,
         bearer_token: str | None = None,
         memory_factory: MemoryFactory | None = None,
+        single_tenant: bool = False,
     ) -> None:
         self._url = base_url
         self._ns = namespace_id
@@ -55,6 +56,10 @@ class RelataMemoryStore:
         self._purpose = purpose
         self._tok = bearer_token
         self._factory = memory_factory
+        # Single-tenant servers (e.g. a trial license with max_tenants=0) reject
+        # any X-Organization-Id; omit the tenant header and rely on session_id
+        # (the agent kid) for per-agent isolation.
+        self._single_tenant = single_tenant
         self._m: Any = None
 
     def open(self) -> None:
@@ -65,12 +70,15 @@ class RelataMemoryStore:
         else:  # pragma: no cover - requires the optional `relata` SDK + live server
             from relata import Memory
 
+            # Multi-tenant: partition by X-Organization-Id. Single-tenant: omit it
+            # (the server rejects org headers) and isolate by session_id alone.
+            extra = {} if self._single_tenant else {"X-Organization-Id": self._ns}
             self._m = Memory(
                 self._url,
                 purpose=self._purpose,
                 bearer_token=self._tok,
                 session_id=self._session,  # per-agent partition
-                extra_headers={"X-Organization-Id": self._ns},  # tenant isolation
+                extra_headers=extra,
             )
 
     def close(self) -> None:
@@ -93,7 +101,7 @@ class RelataMemoryStore:
 
     def remember_batch(
         self,
-        items: "list[str | dict]",
+        items: list[str | dict],
         *,
         session_id: str | None = None,
     ) -> list[str]:

{hypermind-0.33.0 → hypermind-0.34.0}/src/hypermind/orgs/__init__.py

@@ -24,7 +24,9 @@ worked examples of the mapping.
 
 from __future__ import annotations
 
+from hypermind.orgs.charter import ResearchPlan, Subsection, build_expert_model, plan_research
 from hypermind.orgs.context import ContextBuilder, default_context_builder
+from hypermind.orgs.refine import plan_and_refine, score_plan
 from hypermind.orgs.deliberate import build_deliberator, policies_for_role
 from hypermind.orgs.experiment import (
     Experiment,
@@ -130,10 +132,12 @@ __all__ = [
     "OrgSim",
     "QualityReport",
     "RemoveMember",
+    "ResearchPlan",
     "ScoreCard",
     "Seat",
     "Slot",
     "StaticGroundTruth",
+    "Subsection",
     "ToolGateway",
     "ToolProvider",
     "Unit",
@@ -144,6 +148,7 @@ __all__ = [
     "assess_answer",
     "attach_mcp_config",
     "build_deliberator",
+    "build_expert_model",
     "build_receipt",
     "build_tool",
     "chat_ner_extractor",
@@ -166,12 +171,15 @@ __all__ = [
     "load_mcp_config",
     "openai_embedder",
     "pioneer_ner_extractor",
+    "plan_and_refine",
+    "plan_research",
     "policies_for_role",
     "random_grants",
     "reciprocal_rank_fusion",
     "register_provider",
     "register_tool",
     "relata_retriever",
+    "score_plan",
     "unregister_tool",
     "verify_receipt",
 ]

hypermind-0.34.0/src/hypermind/orgs/charter.py

@@ -0,0 +1,164 @@
+"""Master-planned research teams (#charter).
+
+Instead of a fixed template staffed with N clones, a **master (principal
+investigator)** decomposes a research problem into distinct sub-areas and staffs
+each with a purpose-built **expert** — a role, a persona/system-prompt, a skill
+set, preferred tools, and a precise research angle. :func:`plan_research` runs
+that decomposition (one LLM call); :func:`build_expert_model` turns the plan into
+an :class:`OrgModel` (one unit per sub-area + a coordination unit with the PI
+oracle and a synthesist). The result is a genuinely differentiated swarm: each
+expert attacks a different angle with different tools, rather than everyone
+running the same search.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+from hypermind.orgs.model import VALID_ROLES, DynamicsConfig, OrgModel, Seat, Unit
+from hypermind.orgs.prompts import KNOWN_TOOLS, Exemplar, build_plan_messages
+
+# Prompt contract + few-shot exemplars live in prompts.py (one editable place).
+_KNOWN_TOOLS = KNOWN_TOOLS
+
+
+@dataclass
+class Subsection:
+    area: str
+    expert_title: str
+    role: str
+    persona: str
+    angle: str
+    skills: list[str] = field(default_factory=list)
+    tools: list[str] = field(default_factory=list)
+
+
+@dataclass
+class ResearchPlan:
+    topic: str
+    title: str
+    subsections: list[Subsection]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "topic": self.topic,
+            "title": self.title,
+            "subsections": [vars(s) for s in self.subsections],
+        }
+
+
+def _slug(s: str, i: int) -> str:
+    out = re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")[:24]
+    return out or f"area-{i}"
+
+
+def plan_from_dict(topic: str, data: dict[str, Any]) -> ResearchPlan:
+    """Build a validated ResearchPlan from a planner's JSON (lenient)."""
+    subs: list[Subsection] = []
+    for s in data.get("subsections", []):
+        role = str(s.get("role", "scout")).lower()
+        if role not in VALID_ROLES:
+            role = "scout"
+        tools = [t for t in s.get("tools", []) if t in _KNOWN_TOOLS] or ["web_search_ddg"]
+        subs.append(
+            Subsection(
+                area=str(s.get("area", "area")),
+                expert_title=str(s.get("expert_title", "Researcher"))[:48],
+                role=role,
+                persona=str(s.get("persona", "A domain expert.")),
+                angle=str(s.get("angle", "")),
+                skills=[str(x) for x in s.get("skills", [])][:6],
+                tools=tools,
+            )
+        )
+    return ResearchPlan(topic=topic, title=str(data.get("title", topic))[:120], subsections=subs)
+
+
+async def plan_research(
+    topic: str,
+    *,
+    client: Any,
+    model: str,
+    k: int = 6,
+    shots: int = 2,
+    critique: str | None = None,
+    exemplars: list[Exemplar] | None = None,
+) -> ResearchPlan:
+    """Master decomposition: one LLM call → a ResearchPlan of expert subsections.
+
+    ``shots`` few-shot exemplars (0 = zero-shot) steer the decomposition toward
+    distinct, well-staffed experts; ``critique`` lets a prior scored run ask for
+    a targeted revision (self-refine loop). See :mod:`hypermind.orgs.prompts`."""
+    resp = await client.chat.completions.create(
+        model=model,
+        temperature=0.3,
+        messages=build_plan_messages(
+            topic, k=k, shots=shots, critique=critique, exemplars=exemplars
+        ),
+        response_format={"type": "json_object"},
+    )
+    content = resp.choices[0].message.content or "{}"
+    m = re.search(r"\{.*\}", content, re.S)
+    data = json.loads(m.group(0)) if m else {}
+    plan = plan_from_dict(topic, data)
+    if not plan.subsections:  # planner returned nothing usable → one generalist
+        plan.subsections = [
+            Subsection("general", "Lead Researcher", "generalist", "A rigorous generalist.", topic)
+        ]
+    return plan
+
+
+def build_expert_model(
+    plan: ResearchPlan, *, namespace: str, dynamics: DynamicsConfig | None = None
+) -> OrgModel:
+    """Turn a ResearchPlan into an OrgModel: one unit per expert sub-area + a
+    coordination unit (PI oracle + synthesist) so disputes resolve and findings
+    are synthesised across areas."""
+    units: list[Unit] = []
+    seen: set[str] = set()
+    for i, s in enumerate(plan.subsections):
+        name = _slug(s.area, i)
+        while name in seen:  # keep unit names unique
+            name = f"{name}-{i}"
+        seen.add(name)
+        system = f"{s.persona}\nYour research angle: {s.angle}" if s.angle else s.persona
+        units.append(
+            Unit(
+                name=name,
+                seats=(
+                    Seat(
+                        title=s.expert_title or f"Expert {i + 1}",
+                        role=s.role,
+                        system=system,
+                        authority=0.6,
+                        tools=tuple(s.tools),
+                    ),
+                ),
+            )
+        )
+    units.append(
+        Unit(
+            name="coordination",
+            seats=(
+                Seat(
+                    "Principal Investigator", role="oracle", authority=0.95,
+                    system="You are the PI. Ground and adjudicate claims across sub-areas; "
+                    "reward rigor and evidence, penalise hand-waving.",
+                    tools=("searchapi_google_scholar",),
+                ),
+                Seat(
+                    "Synthesist", role="synthesist", authority=0.8,
+                    system="You synthesise findings across the lab's sub-areas into a coherent "
+                    "position, citing the strongest evidence.",
+                    tools=("web_search_ddg",),
+                ),
+            ),
+        )
+    )
+    return OrgModel(
+        name=plan.title or "research", namespace=namespace, units=tuple(units),
+        dynamics=dynamics or DynamicsConfig(rounds=6),
+    )

hypermind-0.34.0/src/hypermind/orgs/evidence.py

@@ -0,0 +1,131 @@
+"""Turn captured raw tool output into structured, usable citations (#evidence).
+
+We already persist every tool call's full raw output (see ``sim._publish``). This
+module parses that JSON/text into :class:`Citation` records — title, url,
+snippet, authors, year — so the swarm can *use* its primary sources instead of
+only its own summaries:
+
+* **grounding** — feed real source snippets to the judge (an external check,
+  not summary-vs-summary);
+* **bibliography** — a de-duplicated, multi-expert-ranked reference list;
+* **retrieval** — real passages for later rounds.
+
+Everything here is pure parsing (no LLM, no network) — cheap enough to run per
+retrieval and cached by claim count in the engine, so it adds no latency.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+
+_YEAR = re.compile(r"\b(19|20)\d{2}\b")
+_DDG_URL = re.compile(r"\((https?://[^)]+)\)\s*$")
+# searchapi/SERP shapes vary by endpoint; these are the arrays that hold hits.
+_RESULT_KEYS = ("organic_results", "results", "scholar_results", "news_results", "items")
+
+
+@dataclass
+class Citation:
+    title: str
+    url: str = ""
+    snippet: str = ""
+    authors: str = ""
+    year: str = ""
+    source: str = ""  # the tool that surfaced it
+    found_by: set[str] = field(default_factory=set)  # expert labels that cited it
+
+    def key(self) -> str:
+        """Dedup key: prefer URL, else normalised title."""
+        return (self.url or self.title).strip().lower()
+
+
+def _year_of(*texts: str) -> str:
+    for t in texts:
+        m = _YEAR.search(t or "")
+        if m:
+            return m.group(0)
+    return ""
+
+
+def parse_tool_result(tool: str, result: str) -> list[Citation]:
+    """Parse one tool result (searchapi JSON or DuckDuckGo text) into citations.
+
+    Lenient by design: unknown shapes yield an empty list, never an error."""
+    result = result or ""
+    out: list[Citation] = []
+    try:
+        data = json.loads(result)
+    except (json.JSONDecodeError, ValueError):
+        data = None
+
+    if isinstance(data, dict):
+        arr = next((data[k] for k in _RESULT_KEYS if isinstance(data.get(k), list)), None)
+        for r in arr or []:
+            if not isinstance(r, dict):
+                continue
+            title = str(r.get("title") or r.get("name") or "").strip()
+            url = str(r.get("link") or r.get("url") or r.get("patent_link") or "").strip()
+            snippet = str(r.get("snippet") or r.get("description") or r.get("abstract") or "").strip()
+            pub = r.get("publication_info")
+            authors = str(pub.get("summary", "")).strip() if isinstance(pub, dict) else ""
+            year = _year_of(authors, str(r.get("year", "")), title)
+            if title or url:
+                out.append(Citation(title, url, snippet, authors, year, tool))
+    else:
+        # DuckDuckGo _fmt lines: "- title: snippet (url)"
+        for line in result.splitlines():
+            line = line.strip().lstrip("-").strip()
+            if not line:
+                continue
+            m = _DDG_URL.search(line)
+            if not m:  # real result lines carry a "(url)"; skip free prose
+                continue
+            url, line = m.group(1), line[: m.start()].strip()
+            title, _sep, snippet = line.partition(":")
+            out.append(Citation(title.strip(), url, snippet.strip(), "", _year_of(line), tool))
+    return out
+
+
+def extract_citations(evidence_records: list[dict]) -> list[Citation]:
+    """Parse + de-duplicate citations across all evidence records.
+
+    Merges duplicates (same URL/title), unions the experts that surfaced each,
+    keeps the longest snippet, and ranks by **multi-expert corroboration** then
+    snippet richness — sources several experts independently found rank first."""
+    by_key: dict[str, Citation] = {}
+    for rec in evidence_records:
+        by = rec.get("by")
+        for tc in rec.get("tool_calls", []):
+            for c in parse_tool_result(str(tc.get("tool", "")), str(tc.get("result", ""))):
+                k = c.key()
+                if not k:
+                    continue
+                cur = by_key.get(k)
+                if cur is None:
+                    if by:
+                        c.found_by.add(by)
+                    by_key[k] = c
+                else:
+                    if by:
+                        cur.found_by.add(by)
+                    if len(c.snippet) > len(cur.snippet):
+                        cur.snippet = c.snippet
+                    if not cur.year:
+                        cur.year = c.year
+    cites = list(by_key.values())
+    cites.sort(key=lambda c: (-len(c.found_by), -len(c.snippet), c.title))
+    return cites
+
+
+def citation_context(cites: list[Citation], k: int) -> list[tuple[str, str]]:
+    """Top-``k`` citations as ``(source_label, text)`` grounding pairs — real
+    source snippets for the judge/answer to ground on."""
+    out: list[tuple[str, str]] = []
+    for c in cites[: max(0, k)]:
+        head = c.title or c.url or "source"
+        meta = f" ({c.authors})" if c.authors else (f" ({c.year})" if c.year else "")
+        body = c.snippet or c.title
+        out.append((f"src:{c.source}", f"{head}{meta}: {body}".strip()))
+    return out

{hypermind-0.33.0 → hypermind-0.34.0}/src/hypermind/orgs/mcp_config.py

@@ -64,6 +64,7 @@ async def connect_mcp_server(spec: dict[str, Any]) -> tuple[Any, Any]:
     proc = await asyncio.create_subprocess_exec(
         spec["command"], *spec.get("args", []),
         stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, env=env,
+        limit=2**24,  # 16 MB line buffer — MCP tool results (Scholar/Patents) blow past asyncio's 64 KB default
     )
     client = MCPClient(StdioTransport(proc.stdout, proc.stdin))
     await client.start()