passiveworkers 0.1.0__py3-none-any.whl

council/__init__.py

@@ -0,0 +1 @@
+"""Passive Workers — the Council: mutual-aid collective-intelligence MVP."""

council/artifacts.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""
+council/artifacts.py — content-addressed, chunked, integrity-verified file delivery (D22)
+=========================================================================================
+Real marketplace work produces FILES, not just text. This is the lean, dependency-free
+codec for moving them between machines safely (FEDERATION_V2 step 3):
+
+  • split a file into fixed-size chunks, hash each (sha256) → the chunk is its own address
+  • a manifest records {name, size, chunk_size, root, chunks:[hashes]}; `root` = sha256 of
+    the ordered chunk hashes (a flat Merkle root) — the tamper-evident fingerprint
+  • the coordinator stores chunks as OPAQUE content-addressed blobs (dedup for free)
+  • the receiver fetches each chunk by hash, verifies it against the manifest, and only
+    reassembles if every chunk and the root check out — a corrupted or swapped chunk is
+    detected, not written
+
+Encryption + producer signatures layer on top of this later (the [crypto] extra); the
+content-addressing here already gives integrity. Stdlib only.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import pathlib
+
+CHUNK_SIZE = 256 * 1024          # 256 KiB chunks
+MAX_FILE_BYTES = 50 * 1024 * 1024  # 50 MiB per deliverable file (v1 cap)
+
+
+_ARTIFACT_TAG = "__pw_artifact__"
+
+
+def _h(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def wrap_artifact(manifest: dict) -> str:
+    """Serialize a manifest as a file-deliverable, tagged so it can't be confused with a
+    user's text deliverable that merely happens to be JSON."""
+    import json
+    return json.dumps({_ARTIFACT_TAG: 1, "manifest": manifest})
+
+
+def read_artifact(deliverable: str):
+    """Return the manifest if `deliverable` is a tagged file artifact, else None (it's text)."""
+    import json
+    try:
+        d = json.loads(deliverable)
+    except Exception:
+        return None
+    if isinstance(d, dict) and d.get(_ARTIFACT_TAG) == 1 and isinstance(d.get("manifest"), dict):
+        return d["manifest"]
+    return None
+
+
+def manifest_root(chunk_hashes: list[str]) -> str:
+    """Flat Merkle root: sha256 over the ordered chunk hashes. Order matters (reassembly)."""
+    return hashlib.sha256("".join(chunk_hashes).encode()).hexdigest()
+
+
+def chunk_file(path: str) -> tuple[dict, dict[str, bytes]]:
+    """(manifest, {hash: bytes}). Raises if the file is missing or over the size cap."""
+    p = pathlib.Path(path).expanduser().resolve()
+    if not p.is_file():
+        raise ValueError(f"not a file: {path}")
+    size = p.stat().st_size
+    if size > MAX_FILE_BYTES:
+        raise ValueError(f"file too large ({size // 1_000_000} MB > {MAX_FILE_BYTES // 1_000_000} MB)")
+    blobs: dict[str, bytes] = {}
+    order: list[str] = []
+    with p.open("rb") as f:
+        while True:
+            buf = f.read(CHUNK_SIZE)
+            if not buf:
+                break
+            h = _h(buf)
+            blobs[h] = buf
+            order.append(h)
+    manifest = {"name": p.name, "size": size, "chunk_size": CHUNK_SIZE,
+                "chunks": order, "root": manifest_root(order)}
+    return manifest, blobs
+
+
+_HEX64 = __import__("re").compile(r"^[0-9a-f]{64}$")
+
+
+def chunk_file_encrypted(path: str, seal_fn) -> tuple[dict, dict[str, bytes]]:
+    """Like chunk_file, but each plaintext chunk is encrypted via seal_fn(bytes)->bytes before
+    hashing/storing. The blob address is the hash of the CIPHERTEXT (so content-addressing and
+    integrity still apply); the manifest is flagged `encrypted` and its size is the PLAINTEXT
+    size. The coordinator only ever stores ciphertext it cannot read (D23)."""
+    p = pathlib.Path(path).expanduser().resolve()
+    if not p.is_file():
+        raise ValueError(f"not a file: {path}")
+    size = p.stat().st_size
+    if size > MAX_FILE_BYTES:
+        raise ValueError(f"file too large ({size // 1_000_000} MB > {MAX_FILE_BYTES // 1_000_000} MB)")
+    blobs: dict[str, bytes] = {}
+    order: list[str] = []
+    with p.open("rb") as f:
+        while True:
+            buf = f.read(CHUNK_SIZE)
+            if not buf:
+                break
+            ct = seal_fn(buf)
+            h = _h(ct)
+            blobs[h] = ct
+            order.append(h)
+    manifest = {"name": p.name, "size": size, "chunk_size": CHUNK_SIZE,
+                "chunks": order, "root": manifest_root(order), "encrypted": True}
+    return manifest, blobs
+
+
+def verify_manifest(manifest: dict) -> bool:
+    """The manifest's declared root must match its ordered chunk list, chunk hashes must be
+    well-formed, and size must be a non-negative int (catches a doctored manifest before we
+    fetch anything). An empty file (size 0, no chunks) is valid."""
+    chunks = manifest.get("chunks")
+    size = manifest.get("size")
+    if not isinstance(chunks, list) or not isinstance(size, int) or size < 0:
+        return False
+    if not all(isinstance(h, str) and _HEX64.match(h) for h in chunks):
+        return False
+    return manifest.get("root") == manifest_root(chunks)
+
+
+def reassemble(manifest: dict, fetch_chunk, out_dir: str, decrypt=None) -> pathlib.Path:
+    """Fetch each chunk via fetch_chunk(hash)->bytes, VERIFY each against its hash, and write
+    the file into out_dir only if everything checks out. fetch_chunk failures or any hash
+    mismatch raise — a corrupted/swapped chunk never reaches disk. Path-safe: the manifest
+    name is reduced to a basename inside out_dir. If the manifest is `encrypted`, `decrypt`
+    (bytes->bytes) is applied AFTER the ciphertext hash is verified."""
+    if manifest.get("encrypted") and decrypt is None:
+        raise ValueError("manifest is encrypted but no decrypt key provided")
+    if not verify_manifest(manifest):
+        raise ValueError("manifest root does not match its chunk list")
+    out = pathlib.Path(out_dir).expanduser().resolve()
+    out.mkdir(parents=True, exist_ok=True)
+    name = pathlib.Path(str(manifest.get("name", ""))).name or "deliverable.bin"  # strip traversal
+    dest = (out / name).resolve()
+    # segment-aware containment (not a bare startswith, which sibling-prefixes could fool)
+    if dest != out and out not in dest.parents:
+        raise ValueError("unsafe output path")
+    total = 0
+    encrypted = bool(manifest.get("encrypted"))
+    with dest.open("wb") as f:
+        for h in manifest["chunks"]:
+            data = fetch_chunk(h)
+            if data is None or _h(data) != h:   # verify the (cipher)text against its address
+                raise ValueError(f"chunk {h[:12]}… missing or corrupted — aborting")
+            if encrypted:
+                try:
+                    data = decrypt(data)        # decrypt AFTER integrity check
+                except Exception:
+                    dest.unlink(missing_ok=True)
+                    raise ValueError("decryption failed (wrong key or tampered chunk)")
+            f.write(data)
+            total += len(data)
+    if manifest.get("size") is not None and total != manifest["size"]:
+        dest.unlink(missing_ok=True)
+        raise ValueError("reassembled size mismatch")
+    return dest

council/batch.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""
+council/batch.py — the per-node batch worker for `shard_map` jobs (D13/D15)
+============================================================================
+One big job, split across computers: this node receives its SHARD of the items and
+applies the instruction to each item with its local model. Wall-clock scales with
+items ÷ computers — the honest "divide to save time" the marketplace sells.
+
+With `fetch=True`, items are PUBLIC URLs: the node fetches each one itself (SSRF-guarded,
+one polite request, size-capped) and returns the model's EXTRACTION — never the raw page.
+The D15 bright line: a node only fetches what it could lawfully fetch alone, and only
+value-added output leaves the node. No relaying, no rate-limit evasion.
+
+Per-item failures never kill the shard — the item's output records the error and the
+batch continues (the judge's spot-check and score reflect quality).
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from dataclasses import dataclass
+
+import requests
+
+from council.research import fetch_extract
+from council.sanitize import clean, spotlight
+
+OLLAMA_BASE = "http://localhost:11434"
+_GEN_TIMEOUT = float(os.environ.get("PW_BATCH_GEN_TIMEOUT",
+                                    os.environ.get("PW_OLLAMA_TIMEOUT", "480")))
+
+
+@dataclass
+class BatchWorker:
+    worker_id: str
+    model: str
+    country: str = "local"
+    temperature: float = 0.2
+    ollama_base: str = OLLAMA_BASE
+
+    def _generate(self, prompt: str, num_predict: int = 220) -> tuple[str, int]:
+        r = requests.post(
+            f"{self.ollama_base}/api/generate",
+            json={"model": self.model, "prompt": prompt, "stream": False,
+                  "options": {"temperature": self.temperature, "num_predict": num_predict},
+                  # keep the model warm across this shard's items (R17) — batch loops _generate per item
+                  "keep_alive": os.environ.get("PW_OLLAMA_KEEP_ALIVE", "30m")},
+            timeout=_GEN_TIMEOUT,
+        )
+        r.raise_for_status()
+        d = r.json()
+        return (d.get("response") or "").strip(), (d.get("eval_count") or 0)
+
+    def process(self, instruction: str, shard: list[dict], fetch: bool = False) -> dict:
+        t0 = time.monotonic()
+        # the instruction is the asker's TASK (kept as a directive) but still scrubbed of invisible/
+        # bidi/HTML-comment injection vectors; every per-item value is untrusted DATA → spotlighted.
+        instruction = clean(instruction)
+        results, tokens, ok = [], 0, 0
+        for entry in shard:
+            idx, item = entry.get("i", 0), str(entry.get("item", ""))
+            try:
+                if fetch:
+                    content = fetch_extract(item)
+                    prompt = (f"{instruction}\n\nSOURCE URL: {item}\n"
+                              f"PAGE TEXT (extracted):\n{spotlight(content)}\n\nOUTPUT:")
+                else:
+                    prompt = f"{instruction}\n\nITEM:\n{spotlight(item)}\n\nOUTPUT (concise):"
+                out, tk = self._generate(prompt)
+                tokens += tk
+                ok += 1
+                results.append({"i": idx, "item": item, "output": out})
+            except Exception as e:
+                results.append({"i": idx, "item": item,
+                                "output": f"(error: {type(e).__name__}: {str(e)[:120]})"})
+        elapsed = round(time.monotonic() - t0, 2)
+        return {
+            "text": (f"Processed {ok}/{len(shard)} items in {elapsed}s on this "
+                     f"{self.country} node ({self.model})."),
+            "results": results,
+            "tokens": tokens,
+            "elapsed_s": elapsed,
+        }

council/cli.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""
+council/cli.py — the `pw` command
+==================================
+    pw research "your brief" [--quick|--deep] [--editor api] [--analysts N] [--local|--web]
+    pw serve                       # local research desk at http://127.0.0.1:8770
+    pw library add <path|dir>      # index your own documents (private, local RAG)
+    pw library list|remove|clear
+    pw mcp                         # run as an MCP server (Claude Desktop, Codex, …)
+    pw tasks                       # list open assisted offers you can do (federation)
+    pw accept <id> | deliver <id> <text|@file>
+    pw fetch <job> <dir>           # download + verify a delivered file (asker)
+    pw rate <job> <0-10>           # rate an assisted deliverable → operator reputation (asker)
+    pw fingerprint                 # print your signing key + fingerprint to share (operator)
+    pw trust add <op> <key> | list | remove <op>   # pin operator keys out of band (asker)
+"""
+
+from __future__ import annotations
+
+import sys
+
+
+def main() -> int:
+    args = sys.argv[1:]
+    if not args or args[0] in ("-h", "--help"):
+        print(__doc__.strip())
+        return 0
+    cmd, rest = args[0], args[1:]
+    if cmd == "research":
+        sys.argv = ["pw research"] + rest
+        from council.local import main as research_main
+        return research_main()
+    if cmd == "serve":
+        from council.serve import main as serve_main
+        serve_main()
+        return 0
+    if cmd == "library":
+        sys.argv = ["pw library"] + rest
+        from council.library import main as library_main
+        return library_main()
+    if cmd == "mcp":
+        from council.mcp_server import main as mcp_main
+        mcp_main()
+        return 0
+    if cmd in ("tasks", "accept", "deliver", "fetch", "keygen", "rate", "fingerprint", "trust"):
+        sys.argv = ["pw", cmd] + rest   # operator.main dispatches on argv[1] (the verb)
+        from council.operator import main as operator_main
+        return operator_main()
+    print(f"unknown command: {cmd}\n\n{__doc__.strip()}")
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

council/coordinator.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""
+council/coordinator.py — the orchestrator
+=========================================
+Ties the loop together:
+
+    intake  →  concurrent fan-out to a diverse fleet  →  judge scores + merges  →
+    score-weighted ledger settlement  →  result bundle
+
+The coordinator is the open-source, self-hostable hub. It holds the non-transferable
+ledger and the worker registry; it never holds a tradeable token and never relays raw
+traffic — it routes TASKS and settles CREDIT for delivered answers.
+
+A worker is owned by a participant (its `owner`); credit for that worker's answer flows
+to the owner's account. That is how "your machine helping on my question earns YOU
+credit" works — and how a user who both asks and helps keeps their give/take balanced.
+"""
+
+from __future__ import annotations
+
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+
+from council.judge import Judge, ScoredCandidate
+from council.ledger import InsufficientCredit, Ledger, Receipt
+from council.worker import Answer, PerspectiveWorker
+
+WORKER_POOL = 30.0   # credits split among helpers per job (by score)
+JUDGE_FEE = 5.0      # credits to the judge per job
+
+
+@dataclass
+class CouncilResult:
+    job_id: str
+    question: str
+    asker_id: str
+    answers: list[Answer]
+    scored: list[ScoredCandidate]
+    merged_answer: str
+    receipt: Receipt
+    elapsed_s: float
+    owner_of: dict = field(default_factory=dict)  # worker_id -> owner_id
+
+    def best_single(self) -> Answer:
+        best_wid = max(self.scored, key=lambda s: s.score).worker_id
+        return next(a for a in self.answers if a.worker_id == best_wid)
+
+    def score_for(self, worker_id: str) -> float:
+        return next((s.score for s in self.scored if s.worker_id == worker_id), 0.0)
+
+
+@dataclass
+class Council:
+    ledger: Ledger
+    judge: Judge
+    judge_owner_id: str = "judge_node"
+    worker_pool: float = WORKER_POOL
+    judge_fee: float = JUDGE_FEE
+    _job_seq: int = 0
+
+    def run(
+        self,
+        asker_id: str,
+        question: str,
+        fleet: list[PerspectiveWorker],
+        owner_of: dict | None = None,
+    ) -> CouncilResult:
+        owner_of = dict(owner_of or {})
+        for w in fleet:
+            owner_of.setdefault(w.worker_id, w.worker_id)
+
+        # Make sure every participant has an account.
+        self.ledger.open_account(asker_id)
+        self.ledger.open_account(self.judge_owner_id)
+        for w in fleet:
+            self.ledger.open_account(owner_of[w.worker_id])
+
+        # Affordability gate — the give/take rule in action.
+        cost = self.ledger.quote(self.worker_pool, self.judge_fee)
+        if not self.ledger.can_afford(asker_id, cost):
+            raise InsufficientCredit(
+                f"{asker_id} cannot afford {cost:.1f} credits "
+                f"(balance {self.ledger.balance(asker_id):.1f}). Help on a job first."
+            )
+
+        self._job_seq += 1
+        job_id = f"job{self._job_seq:03d}"
+        t0 = time.monotonic()
+
+        # Concurrent fan-out — every perspective answers in parallel.
+        answers: list[Answer] = []
+        with ThreadPoolExecutor(max_workers=len(fleet)) as pool:
+            futures = {pool.submit(w.answer, question): w for w in fleet}
+            for fut in as_completed(futures):
+                try:
+                    answers.append(fut.result())
+                except Exception as exc:
+                    w = futures[fut]
+                    print(f"  [coordinator] worker {w.worker_id} failed: {exc}")
+        # Stable order by worker_id for reproducible display.
+        answers.sort(key=lambda a: a.worker_id)
+
+        # Judge: score (blind) then merge.
+        scored = self.judge.score(question, answers)
+        merged = self.judge.merge(question, answers)
+
+        # Aggregate scores by OWNER (a user running >1 model sums their scores).
+        score_by_owner: dict[str, float] = {}
+        for sc in scored:
+            owner = owner_of[sc.worker_id]
+            score_by_owner[owner] = score_by_owner.get(owner, 0.0) + sc.score
+
+        receipt = self.ledger.settle_job(
+            job_id=job_id,
+            asker_id=asker_id,
+            score_by_worker=score_by_owner,
+            worker_pool=self.worker_pool,
+            judge_id=self.judge_owner_id,
+            judge_fee=self.judge_fee,
+        )
+
+        return CouncilResult(
+            job_id=job_id,
+            question=question,
+            asker_id=asker_id,
+            answers=answers,
+            scored=scored,
+            merged_answer=merged,
+            receipt=receipt,
+            elapsed_s=time.monotonic() - t0,
+            owner_of=owner_of,
+        )

council/crypto.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""
+council/crypto.py — optional cryptographic delivery (D23, FEDERATION_V2 step 2)
+================================================================================
+Two real guarantees layered on top of content-addressed delivery (D22), both OPTIONAL
+(the [crypto] extra; everything degrades gracefully to D22 integrity when PyNaCl is absent):
+
+  • SIGNING (Ed25519): an operator signs its deliverable with a private key the coordinator
+    never sees; the asker verifies against the operator's published public key. Detects any
+    post-delivery tampering of the content and binds the deliverable to the operator's key.
+    (Honest limit: a hostile coordinator that swaps content+key+sig together needs an
+    out-of-band key check / PKI to defeat — documented, not papered over.)
+  • ENCRYPTION (SealedBox / X25519): the asker publishes a public key with the job; the
+    operator seals the payload to it; only the asker's private key can open it. The
+    coordinator relays ciphertext it genuinely cannot read — a real confidentiality
+    guarantee even against a hostile coordinator.
+
+Keys are base64 strings so they travel as plain JSON. Helpers persist a keypair per identity
+in ~/.passiveworkers so sign/verify and seal/unseal survive across CLI invocations.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import pathlib
+
+try:
+    from nacl.public import PrivateKey, PublicKey, SealedBox
+    from nacl.signing import SigningKey, VerifyKey
+    from nacl.exceptions import BadSignatureError, CryptoError
+    HAVE_CRYPTO = True
+except Exception:                       # pragma: no cover - optional dependency
+    HAVE_CRYPTO = False
+
+
+def available() -> bool:
+    return HAVE_CRYPTO
+
+
+def _require() -> None:
+    if not HAVE_CRYPTO:
+        raise RuntimeError("the crypto extra is not installed: pip install 'passiveworkers[crypto]'")
+
+
+def _b64e(b: bytes) -> str:
+    return base64.b64encode(b).decode()
+
+
+def _b64d(s: str) -> bytes:
+    return base64.b64decode(s.encode(), validate=True)   # reject whitespace/non-alphabet
+
+
+# ------------------------------------------------------------------ keypairs
+def new_signing_keypair() -> tuple[str, str]:
+    """(private_b64, public_b64) for Ed25519 signing."""
+    _require()
+    sk = SigningKey.generate()
+    return _b64e(bytes(sk)), _b64e(bytes(sk.verify_key))
+
+
+def new_box_keypair() -> tuple[str, str]:
+    """(private_b64, public_b64) for X25519 sealed-box encryption."""
+    _require()
+    sk = PrivateKey.generate()
+    return _b64e(bytes(sk)), _b64e(bytes(sk.public_key))
+
+
+def load_or_create(path: pathlib.Path, kind: str) -> dict:
+    """Persist+reuse a keypair of `kind` ('sign' or 'box') at `path`. Returns {priv, pub}."""
+    if path.exists():
+        try:
+            d = json.loads(path.read_text()).get(kind)
+            if d and d.get("priv") and d.get("pub"):
+                return d
+        except Exception:
+            pass
+    priv, pub = new_signing_keypair() if kind == "sign" else new_box_keypair()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    allk = {}
+    if path.exists():
+        try:
+            allk = json.loads(path.read_text())
+        except Exception:
+            allk = {}
+    allk[kind] = {"priv": priv, "pub": pub}
+    # write with owner-only perms FROM CREATION (no world-readable window before chmod)
+    blob = json.dumps(allk).encode()
+    try:
+        fd = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+        with os.fdopen(fd, "wb") as f:
+            f.write(blob)
+        os.chmod(path, 0o600)
+    except Exception:
+        path.write_text(json.dumps(allk))   # fallback (e.g. platforms without mode support)
+        try:
+            os.chmod(path, 0o600)            # a private key must never be world-readable
+        except Exception:
+            pass
+    return allk[kind]
+
+
+# ------------------------------------------------------------------ signing
+def sign(priv_b64: str, data: bytes) -> str:
+    """Detached Ed25519 signature (b64) over `data`."""
+    _require()
+    return _b64e(SigningKey(_b64d(priv_b64)).sign(data).signature)
+
+
+def verify(pub_b64: str, data: bytes, sig_b64: str) -> bool:
+    """True iff `sig` is a valid signature of `data` under `pub`. Never raises (returns False
+    on any error, including when the crypto extra is absent)."""
+    if not HAVE_CRYPTO:
+        return False
+    try:
+        VerifyKey(_b64d(pub_b64)).verify(data, _b64d(sig_b64))
+        return True
+    except Exception:
+        return False
+
+
+# ------------------------------------------------------------------ encryption
+def seal(recipient_pub_b64: str, data: bytes) -> bytes:
+    """Anonymous-sender encryption to a recipient public key (X25519 SealedBox)."""
+    _require()
+    return SealedBox(PublicKey(_b64d(recipient_pub_b64))).encrypt(data)
+
+
+def unseal(priv_b64: str, ciphertext: bytes) -> bytes:
+    """Open a sealed box with the recipient private key. Raises on wrong key / tampering."""
+    _require()
+    return SealedBox(PrivateKey(_b64d(priv_b64))).decrypt(ciphertext)