hypermind 0.22.0__tar.gz → 0.23.0__tar.gz

{hypermind-0.22.0 → hypermind-0.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hypermind
-Version: 0.22.0
+Version: 0.23.0
 Summary: Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile
 Project-URL: Homepage, https://github.com/ZySec-AI/hypermind
 Project-URL: Documentation, https://github.com/ZySec-AI/hypermind/tree/main/docs

{hypermind-0.22.0 → hypermind-0.23.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "hypermind"
-version = "0.22.0"
+version = "0.23.0"
 description = "Federated agent intelligence — reference SDK for the HyperMind SCITT contested-claims profile"
 readme = "README.md"
 requires-python = ">=3.11,<3.14"

{hypermind-0.22.0 → hypermind-0.23.0}/src/hypermind/__init__.py

@@ -73,7 +73,7 @@ open = HyperMindAgent.open
 #: is held until the §External & human gates clear (IETF I-D, IANA,
 #: third-party audit, OpenSSF acceptance, ≥3 pilots) — at that point we
 #: cut ``1.0.0`` GA. See ROADMAP.md §Status snapshot.
-__version__ = "0.22.0"
+__version__ = "0.23.0"
 
 #: SLSA build-provenance level committed for every release artefact.
 #: Cosign keyless signing + slsa-github-generator + PEP 740 attestations

hypermind-0.23.0/src/hypermind/orgs/distributed/__init__.py

@@ -0,0 +1,103 @@
+"""Distributed, Locust-style execution for OrgSim (#177).
+
+A **master coordinator** shards a swarm across **workers** (processes on the
+same box or other machines) over a TCP control plane; every worker runs its
+shard as an ``OrgSim(lightweight=True)`` against the **shared Relata namespace**
+(the source of truth) and returns bounded aggregates, which the coordinator
+merges into one :class:`~hypermind.orgs.report.OrgReport`.
+
+Multi-machine (the Locust model)::
+
+    # on the master host
+    python -m hypermind.orgs.distributed master \
+        --template university --members 10000 --rounds 8 \
+        --expected-workers 4 --host 0.0.0.0 --port 8765
+
+    # on each worker host
+    python -m hypermind.orgs.distributed worker \
+        --master-host <master-ip> --port 8765 --capacity 2500 \
+        --llm openai --llm-model openai/gpt-4o-mini
+
+Single host (convenience — coordinator + in-process workers over loopback)::
+
+    from hypermind.orgs.distributed import run_distributed
+    from hypermind.orgs.templates import university
+    report = await run_distributed(university(), "protein folding",
+                                   members=2000, rounds=6, workers=8, config={...})
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from hypermind.orgs.distributed.coordinator import Coordinator
+from hypermind.orgs.distributed.planning import merge_results, plan_shards
+from hypermind.orgs.distributed.worker import Worker, default_shard_runner
+from hypermind.orgs.model import OrgModel
+from hypermind.orgs.report import OrgReport
+
+__all__ = [
+    "Coordinator",
+    "Worker",
+    "default_shard_runner",
+    "merge_results",
+    "plan_shards",
+    "run_distributed",
+]
+
+
+async def run_distributed(
+    model: OrgModel,
+    topic: str,
+    *,
+    members: int,
+    rounds: int,
+    workers: int = 4,
+    capacity_each: int = 1,
+    config: dict[str, Any] | None = None,
+    host: str = "127.0.0.1",
+    shard_runner: Any = None,
+) -> OrgReport:
+    """Run a distributed simulation on this host: start a coordinator and
+    ``workers`` in-process workers (over loopback), shard ``members`` across
+    them, and return the merged report.
+
+    Same code path as multi-machine — the only difference is the workers live in
+    this process instead of on remote hosts. Pass ``shard_runner`` to stub the
+    per-shard work (tests); omit it for the real ``OrgSim`` runner.
+    """
+    coord = Coordinator(
+        model,
+        topic,
+        members=members,
+        rounds=rounds,
+        expected_workers=workers,
+        host=host,
+        port=0,
+        config=config,
+    )
+
+    async def _serve() -> OrgReport:
+        return await coord.run()
+
+    coord_task = asyncio.create_task(_serve())
+    # Wait for the server to bind before workers dial in.
+    while coord.port == 0:
+        await asyncio.sleep(0.005)
+
+    worker_tasks = [
+        asyncio.create_task(
+            Worker(
+                host,
+                coord.port,
+                worker_id=f"w{i}",
+                capacity=capacity_each,
+                shard_runner=shard_runner,
+            ).run()
+        )
+        for i in range(workers)
+    ]
+    report = await coord_task
+    await asyncio.gather(*worker_tasks, return_exceptions=True)
+    return report

hypermind-0.23.0/src/hypermind/orgs/distributed/__main__.py

@@ -0,0 +1,110 @@
+"""CLI for distributed OrgSim (#177): ``master`` and ``worker`` subcommands.
+
+    python -m hypermind.orgs.distributed master --template university \
+        --members 10000 --rounds 8 --expected-workers 4 --host 0.0.0.0 --port 8765
+
+    python -m hypermind.orgs.distributed worker --master-host <ip> --port 8765 \
+        --capacity 2500 --llm openai --llm-model openai/gpt-4o-mini
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+
+from hypermind.orgs.distributed.coordinator import Coordinator
+from hypermind.orgs.distributed.worker import Worker
+
+
+def _load_model(args: argparse.Namespace):
+    from hypermind.orgs.templates import get_template
+
+    return get_template(args.template)(namespace=args.namespace)
+
+
+async def _run_master(args: argparse.Namespace) -> int:
+    model = _load_model(args)
+    config = {
+        "llm": args.llm,
+        "llm_model": args.llm_model,
+        "relata_url": args.relata_url,
+        "relata_token": args.relata_token,
+        "base_url": args.base_url,
+        "api_key_env": args.api_key_env,
+    }
+    coord = Coordinator(
+        model,
+        args.topic,
+        members=args.members,
+        rounds=args.rounds,
+        expected_workers=args.expected_workers,
+        host=args.host,
+        port=args.port,
+        config=config,
+    )
+    print(
+        f"master: binding {args.host}:{args.port}, waiting for {args.expected_workers} worker(s)…"
+    )
+    report = await coord.run()
+    out = (
+        report.to_json() if hasattr(report, "to_json") else json.dumps(report.__dict__, default=str)
+    )
+    if args.output:
+        with open(args.output, "w") as f:
+            f.write(out)
+        print(f"master: wrote report → {args.output}")
+    else:
+        report.print()
+    return 0
+
+
+async def _run_worker(args: argparse.Namespace) -> int:
+    print(
+        f"worker {args.worker_id}: connecting to {args.master_host}:{args.port} (cap {args.capacity})…"
+    )
+    await Worker(
+        args.master_host, args.port, worker_id=args.worker_id, capacity=args.capacity
+    ).run()
+    print(f"worker {args.worker_id}: done")
+    return 0
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(prog="python -m hypermind.orgs.distributed")
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    m = sub.add_parser("master", help="run the coordinator")
+    m.add_argument("--template", default="university")
+    m.add_argument("--namespace", default="dist-run")
+    m.add_argument("--topic", required=True)
+    m.add_argument("--members", type=int, required=True)
+    m.add_argument("--rounds", type=int, default=6)
+    m.add_argument("--expected-workers", type=int, required=True)
+    m.add_argument("--host", default="0.0.0.0")
+    m.add_argument("--port", type=int, default=8765)
+    m.add_argument("--llm", default="openai")
+    m.add_argument("--llm-model", default=None)
+    m.add_argument("--base-url", default=None)
+    m.add_argument("--api-key-env", default="OPENAI_API_KEY")
+    m.add_argument("--relata-url", default="http://localhost:9090")
+    m.add_argument("--relata-token", default=None)
+    m.add_argument("--output", default=None)
+
+    w = sub.add_parser("worker", help="run a worker")
+    w.add_argument("--master-host", required=True)
+    w.add_argument("--port", type=int, default=8765)
+    w.add_argument("--worker-id", default="worker-1")
+    w.add_argument("--capacity", type=int, default=1)
+    return ap
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    runner = _run_master if args.cmd == "master" else _run_worker
+    return asyncio.run(runner(args))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

hypermind-0.23.0/src/hypermind/orgs/distributed/coordinator.py

@@ -0,0 +1,175 @@
+"""The master: a Locust-style coordinator over a TCP control plane (#177).
+
+Waits for workers to announce capacity, shards the swarm across them, streams
+each an assignment, collects bounded aggregates (heartbeating meanwhile), and
+merges everything into one :class:`OrgReport`. Workers that drop before
+returning a result don't hang the run — their shard is recorded as failed and
+the merge proceeds with a clear note.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import Any
+
+from hypermind.orgs.distributed import protocol as p
+from hypermind.orgs.distributed.planning import merge_results, plan_shards
+from hypermind.orgs.distributed.serde import model_to_dict
+from hypermind.orgs.model import OrgModel
+from hypermind.orgs.report import OrgReport
+
+
+class Coordinator:
+    """Master coordinator. Bind, gather ``expected_workers``, shard, merge."""
+
+    def __init__(
+        self,
+        model: OrgModel,
+        topic: str,
+        *,
+        members: int,
+        rounds: int,
+        expected_workers: int,
+        host: str = "127.0.0.1",
+        port: int = 0,
+        config: dict[str, Any] | None = None,
+        worker_timeout_s: float = 600.0,
+    ) -> None:
+        self.model = model
+        self.topic = topic
+        self.members = members
+        self.rounds = rounds
+        self.expected_workers = expected_workers
+        self.host = host
+        self.port = port
+        self.config = dict(config or {})
+        self.worker_timeout_s = worker_timeout_s
+
+        self._writers: dict[str, asyncio.StreamWriter] = {}
+        self._order: list[str] = []  # worker_ids in arrival order
+        self._capacity: dict[str, int] = {}
+        self._results: dict[int, dict[str, Any]] = {}
+        self._failed: dict[int, str] = {}
+        self._worker_shard: dict[str, int] = {}
+        self._n_shards = 0
+        self._enough_workers = asyncio.Event()
+        self._complete = asyncio.Event()
+        self._lock = asyncio.Lock()
+        self._assigned = False
+
+    async def run(self) -> OrgReport:
+        server = await asyncio.start_server(self._on_conn, self.host, self.port)
+        self.port = server.sockets[0].getsockname()[1]
+        async with server:
+            try:
+                await asyncio.wait_for(self._enough_workers.wait(), self.worker_timeout_s)
+                await self._assign()
+                await asyncio.wait_for(self._complete.wait(), self.worker_timeout_s)
+            finally:
+                await self._quit_all()
+        return self._merge()
+
+    async def _on_conn(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
+        worker_id: str | None = None
+        try:
+            hello = await p.read_frame(reader)
+            if hello.get("type") != p.HELLO:
+                await writer.wait_closed()
+                return
+            worker_id = str(hello["worker_id"])
+            async with self._lock:
+                self._writers[worker_id] = writer
+                self._capacity[worker_id] = max(0, int(hello.get("capacity", 1)))
+                self._order.append(worker_id)
+                if len(self._order) >= self.expected_workers:
+                    self._enough_workers.set()
+            await self._read_loop(worker_id, reader)
+        except (asyncio.IncompleteReadError, ConnectionError, RuntimeError):
+            pass
+        finally:
+            if worker_id is not None:
+                await self._on_worker_gone(worker_id)
+
+    async def _read_loop(self, worker_id: str, reader: asyncio.StreamReader) -> None:
+        while True:
+            msg = await p.read_frame(reader)
+            mtype = msg.get("type")
+            if mtype == p.RESULT:
+                async with self._lock:
+                    self._results[int(msg["shard_id"])] = msg["aggregates"]
+                    self._maybe_complete()
+                return
+            if mtype == p.ERROR:
+                async with self._lock:
+                    sid = self._worker_shard.get(worker_id)
+                    if sid is not None:
+                        self._failed[sid] = str(msg.get("message", "worker error"))
+                        self._maybe_complete()
+                return
+            # HEARTBEAT and anything else: keep listening.
+
+    async def _assign(self) -> None:
+        async with self._lock:
+            if self._assigned:
+                return
+            self._assigned = True
+            ids = list(self._order)
+            caps = [self._capacity[w] for w in ids]
+            counts = plan_shards(self.members, caps)
+            model_d = model_to_dict(self.model)
+            shard_id = 0
+            for w, n in zip(ids, counts, strict=True):
+                if n <= 0:
+                    await p.write_frame(self._writers[w], {"type": p.QUIT})
+                    continue
+                self._worker_shard[w] = shard_id
+                await p.write_frame(
+                    self._writers[w],
+                    {
+                        "type": p.ASSIGN,
+                        "shard_id": shard_id,
+                        "model": model_d,
+                        "topic": self.topic,
+                        "rounds": self.rounds,
+                        "members": n,
+                        "namespace": self.model.namespace,
+                        "config": self.config,
+                    },
+                )
+                shard_id += 1
+            self._n_shards = shard_id
+            if self._n_shards == 0:
+                self._complete.set()
+
+    async def _on_worker_gone(self, worker_id: str) -> None:
+        async with self._lock:
+            sid = self._worker_shard.get(worker_id)
+            if sid is not None and sid not in self._results and sid not in self._failed:
+                self._failed[sid] = "worker disconnected before returning a result"
+                self._maybe_complete()
+
+    def _maybe_complete(self) -> None:
+        if self._n_shards and len(self._results) + len(self._failed) >= self._n_shards:
+            self._complete.set()
+
+    async def _quit_all(self) -> None:
+        for writer in self._writers.values():
+            with contextlib.suppress(Exception):
+                await p.write_frame(writer, {"type": p.QUIT})
+                writer.close()
+
+    def _merge(self) -> OrgReport:
+        results = [{"shard_id": sid, **agg} for sid, agg in sorted(self._results.items())]
+        report = merge_results(
+            org=self.model.name,
+            topic=self.topic,
+            rounds=self.rounds,
+            interaction=self.model.dynamics.interaction,
+            results=results,
+        )
+        if self._failed:
+            report.events.append(
+                {"event": "shards_failed", "count": len(self._failed), "detail": dict(self._failed)}
+            )
+        return report

hypermind-0.23.0/src/hypermind/orgs/distributed/planning.py

@@ -0,0 +1,139 @@
+"""Shard planning + result merge (#177) — pure, no I/O.
+
+The coordinator splits a swarm of ``members`` across workers weighted by each
+worker's declared capacity, then merges the workers' bounded aggregates
+(:meth:`OrgSim.export_aggregates`) into one :class:`OrgReport`. Merging is exact
+for counts and Brier; per-worker member labels are namespaced by shard so they
+never collide.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from hypermind.orgs.metrics import scorecard_from_aggregates
+from hypermind.orgs.report import MemberStanding, OrgReport
+
+
+def plan_shards(members: int, capacities: list[int]) -> list[int]:
+    """Split ``members`` across workers proportional to ``capacities``.
+
+    Largest-remainder apportionment. When ``members`` ≥ the number of workers
+    with capacity, every such worker gets at least 1 (no capable worker idles);
+    when ``members`` is smaller, the highest-capacity workers get the members
+    and the rest get 0. The result always sums to ``members``.
+    """
+    caps = [max(0, c) for c in capacities]
+    total_cap = sum(caps)
+    if members < 1 or total_cap == 0:
+        return [0 for _ in caps]
+
+    capable = [i for i, c in enumerate(caps) if c > 0]
+    alloc = [0 for _ in caps]
+
+    # Fewer members than capable workers → the biggest workers each take one.
+    if members < len(capable):
+        for i in sorted(capable, key=lambda i: caps[i], reverse=True)[:members]:
+            alloc[i] = 1
+        return alloc
+
+    # Every capable worker gets 1 (affordable: members ≥ #capable), then the
+    # remainder is handed out proportional to capacity (largest-remainder).
+    for i in capable:
+        alloc[i] = 1
+    remaining = members - len(capable)
+    if remaining > 0:
+        share = [remaining * caps[i] / total_cap for i in range(len(caps))]
+        for i in range(len(caps)):
+            alloc[i] += int(share[i])
+        leftover = remaining - sum(int(s) for s in share)
+        for i in sorted(capable, key=lambda i: share[i] - int(share[i]), reverse=True)[:leftover]:
+            alloc[i] += 1
+    return alloc
+
+
+def _prefix(shard_id: int, label: str) -> str:
+    return f"s{shard_id}:{label}"
+
+
+def merge_results(
+    *,
+    org: str,
+    topic: str,
+    rounds: int,
+    interaction: str,
+    results: list[dict[str, Any]],
+) -> OrgReport:
+    """Merge worker shard aggregates into one OrgReport.
+
+    Each entry in ``results`` is ``{"shard_id": int, **export_aggregates()}``.
+    Labels are namespaced by shard id so standings/aggregates never collide.
+    """
+    standings: list[MemberStanding] = []
+    n_claims = n_citations = n_disputes = 0
+    brier_sums: dict[str, tuple[float, int]] = {}
+    influence: dict[str, float] = {}
+    publishers: set[str] = set()
+    member_units: dict[str, str] = {}
+    # Per-round posteriors merged across shards (rounds align by index).
+    per_round: list[dict[str, float]] = []
+
+    for r in results:
+        sid = r["shard_id"]
+        n_claims += r.get("n_claims", 0)
+        n_citations += r.get("n_citations", 0)
+        n_disputes += r.get("n_disputes", 0)
+        for s in r.get("standings", []):
+            standings.append(
+                MemberStanding(
+                    label=_prefix(sid, s["label"]),
+                    unit=s["unit"],
+                    role=s["role"],
+                    reputation=s["reputation"],
+                    claims=s["claims"],
+                    citations_received=s["citations_received"],
+                    disputes_received=s["disputes_received"],
+                    tools=list(s.get("tools", [])),
+                )
+            )
+        for lbl, sn in r.get("brier_sums", {}).items():
+            brier_sums[_prefix(sid, lbl)] = (sn[0], sn[1])
+        for lbl, v in r.get("influence", {}).items():
+            influence[_prefix(sid, lbl)] = v
+        for lbl in r.get("publishers", []):
+            publishers.add(_prefix(sid, lbl))
+        for lbl, unit in r.get("member_units", {}).items():
+            member_units[_prefix(sid, lbl)] = unit
+        for i, p in enumerate(r.get("per_round_posteriors", [])):
+            if i >= len(per_round):
+                per_round.append({})
+            for lbl, mean in p.items():
+                per_round[i][_prefix(sid, lbl)] = mean
+
+    standings.sort(key=lambda s: s.reputation, reverse=True)
+    scorecard = scorecard_from_aggregates(
+        per_round_posteriors=per_round,
+        member_units=member_units,
+        brier_sums=brier_sums,
+        influence=influence,
+        publishers=publishers,
+        claims_count=n_claims,
+        citations_count=n_citations,
+        disputes_count=n_disputes,
+    )
+    return OrgReport(
+        org=org,
+        topic=topic,
+        rounds=rounds,
+        interaction=interaction,
+        standings=standings,
+        claims=[],  # payloads never leave the workers; totals live in scorecard
+        citations=[],
+        disputes=[],
+        tool_grants={},
+        scorecard=scorecard.to_dict(),
+        stop_reason="distributed",
+        events=[{"event": "distributed", "shards": len(results), "members": len(standings)}],
+        interaction_state={},
+        decisions=[],
+    )

hypermind-0.23.0/src/hypermind/orgs/distributed/protocol.py

@@ -0,0 +1,47 @@
+"""Master↔worker wire protocol (#177).
+
+Length-prefixed CBOR frames — the same framing the SDK's TCP transport uses:
+``[u32 BE length][CBOR map {"type": str, ...}]``. A tiny message set drives a
+Locust-style control plane: workers announce capacity, the master assigns
+shards, workers heartbeat and return bounded aggregates, the master quits them.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import struct
+from typing import Any
+
+import cbor2
+
+FRAME_HEADER = struct.Struct(">I")
+MAX_FRAME = 64 * 1024 * 1024  # 64 MB — a shard's model + aggregates, generously
+
+# Message types.
+HELLO = "hello"  # worker → master: {worker_id, capacity}
+ASSIGN = "assign"  # master → worker: {job_id, shard_id, model, topic, rounds, members, config}
+HEARTBEAT = "heartbeat"  # worker → master: {shard_id, phase}
+RESULT = "result"  # worker → master: {shard_id, aggregates}
+QUIT = "quit"  # master → worker: {}
+ERROR = "error"  # either way: {shard_id?, message}
+
+
+async def read_frame(reader: asyncio.StreamReader) -> dict[str, Any]:
+    header = await reader.readexactly(FRAME_HEADER.size)
+    (length,) = FRAME_HEADER.unpack(header)
+    if length > MAX_FRAME:
+        raise RuntimeError(f"frame too large: {length} bytes")
+    body = await reader.readexactly(length)
+    msg = cbor2.loads(body)
+    if not isinstance(msg, dict) or "type" not in msg:
+        raise RuntimeError("malformed frame: expected a map with a 'type' key")
+    return msg
+
+
+async def write_frame(writer: asyncio.StreamWriter, msg: dict[str, Any]) -> None:
+    body = cbor2.dumps(msg, canonical=True)
+    if len(body) > MAX_FRAME:
+        raise RuntimeError(f"frame too large: {len(body)} bytes")
+    writer.write(FRAME_HEADER.pack(len(body)))
+    writer.write(body)
+    await writer.drain()

hypermind-0.23.0/src/hypermind/orgs/distributed/serde.py

@@ -0,0 +1,71 @@
+"""CBOR/JSON-friendly (de)serialisation of an OrgModel for the wire (#177).
+
+The model core stays dependency-free, so serialisation lives here. Everything
+in an OrgModel is primitive (str/float/int/tuple/dict), so this is a plain
+structural round-trip — no pickling of code across the network.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from hypermind.orgs.model import DynamicsConfig, OrgModel, Seat, Unit
+
+
+def model_to_dict(model: OrgModel) -> dict[str, Any]:
+    return {
+        "name": model.name,
+        "namespace": model.namespace,
+        "lexicon": dict(model.lexicon),
+        "dynamics": vars(model.dynamics),
+        "units": [
+            {
+                "name": u.name,
+                "tools": list(u.tools),
+                "seats": [
+                    {
+                        "title": s.title,
+                        "role": s.role,
+                        "system": s.system,
+                        "authority": s.authority,
+                        "headcount": s.headcount,
+                        "llm": s.llm,
+                        "model": s.model,
+                        "tools": list(s.tools),
+                    }
+                    for s in u.seats
+                ],
+            }
+            for u in model.units
+        ],
+    }
+
+
+def model_from_dict(data: dict[str, Any]) -> OrgModel:
+    units = tuple(
+        Unit(
+            name=u["name"],
+            tools=tuple(u.get("tools", ())),
+            seats=tuple(
+                Seat(
+                    title=s["title"],
+                    role=s.get("role", "generalist"),
+                    system=s.get("system"),
+                    authority=s.get("authority", 0.5),
+                    headcount=s.get("headcount", 1),
+                    llm=s.get("llm"),
+                    model=s.get("model"),
+                    tools=tuple(s.get("tools", ())),
+                )
+                for s in u["seats"]
+            ),
+        )
+        for u in data["units"]
+    )
+    return OrgModel(
+        name=data["name"],
+        namespace=data["namespace"],
+        units=units,
+        lexicon=dict(data.get("lexicon", {})),
+        dynamics=DynamicsConfig(**data.get("dynamics", {})),
+    )