passiveworkers 0.1.0__py3-none-any.whl

council/local.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+"""
+council/local.py — single-player deep research, one command (D16)
+==================================================================
+    python -m council.local "your brief"            # or: pw research "your brief"
+    python -m council.local "brief" --quick|--deep
+    python -m council.local "brief" --editor api    # BYOK frontier editor over LOCAL findings
+
+Everything runs on THIS machine: it plans queries, researches the live web from YOUR
+connection, runs multiple installed Ollama models as independent analysts, and the
+strongest model (the blind editor) compiles a cited markdown report saved to ./reports/.
+Private by construction: no account, no server, no telemetry — only the web searches
+leave the machine. Models hold zero tool privileges (they return text; Python acts).
+
+The network (council/net/) is the optional multiplayer mode — this command needs none of it.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime
+import os
+import pathlib
+import re
+import sys
+import time
+
+import requests
+
+from council.judge import Judge, _extract_json
+from council.researcher import ResearchWorker
+from council.sanitize import sanitize_brief
+from council.worker import Answer
+
+OLLAMA = os.environ.get("PW_OLLAMA_BASE", "http://localhost:11434")
+# families give genuinely different inference trajectories — pick analysts across them
+_FAMILIES = ("qwen", "gemma", "llama", "mistral", "phi", "deepseek", "granite")
+_EXCLUDE = ("embed", "vision", "image")
+
+
+def _make_emit(on_progress):
+    def emit(msg: str) -> None:
+        # stderr, not stdout: keeps the MCP stdio JSON-RPC channel clean (still shown in terminals)
+        print(f"  {msg}", file=sys.stderr, flush=True)
+        if on_progress:
+            try:
+                on_progress(msg)
+            except Exception:
+                pass
+    return emit
+
+
+def detect_models() -> list[dict]:
+    r = requests.get(f"{OLLAMA}/api/tags", timeout=10)
+    r.raise_for_status()
+    models = [m for m in r.json().get("models", [])
+              if not any(x in m["name"].lower() for x in _EXCLUDE)]
+    # On CPU-only / constrained machines big models crawl (3-6 tok/s) — let users cap
+    # the cast by size: PW_MODEL_CAP_GB=3 keeps only models whose weights fit that budget.
+    cap = float(os.environ.get("PW_MODEL_CAP_GB", "0") or 0)
+    if cap > 0:
+        capped = [m for m in models if m.get("size", 0) <= cap * 1e9]
+        models = capped or models[:1]      # never end up with zero models
+    if not models:
+        raise SystemExit("No usable Ollama models found. Try: ollama pull qwen3:14b")
+    return sorted(models, key=lambda m: m.get("size", 0))
+
+
+def pick_cast(models: list[dict], n_analysts: int = 3) -> tuple[list[str], str]:
+    """Analysts = up to N models across different families (diversity reduces correlated
+    error); editor/judge = the largest model installed (the quality anchor)."""
+    editor = models[-1]["name"]
+    by_family: dict[str, str] = {}
+    for m in reversed(models):                    # biggest of each family first
+        fam = next((f for f in _FAMILIES if f in m["name"].lower()), m["name"].split(":")[0])
+        by_family.setdefault(fam, m["name"])
+    analysts = list(by_family.values())[:n_analysts]
+    if not analysts:
+        analysts = [editor]
+    return analysts, editor
+
+
+class _ApiEditor(Judge):
+    """BYOK Tier-3: the same editor prompts, generated by a frontier model the USER chose.
+    Only locally-gathered findings are sent — never raw pages, never local files."""
+
+    def __init__(self, model: str, api_key: str, url: str):
+        super().__init__(model=model)
+        self._key, self._url = api_key, url
+
+    def _generate(self, prompt: str, num_predict=None) -> str:
+        r = requests.post(self._url, headers={"Authorization": f"Bearer {self._key}"},
+                          json={"model": self.model,
+                                "messages": [{"role": "user", "content": prompt}],
+                                "max_tokens": num_predict or 900}, timeout=120)
+        r.raise_for_status()
+        return (r.json()["choices"][0]["message"]["content"] or "").strip()
+
+
+def plan_angles(brief: str, planner_model: str, k: int) -> list[str]:
+    """STORM-lite: one cheap call on the SMALLEST model discovers K distinct perspectives;
+    each analyst then researches the brief through its own angle (question diversity on
+    top of model diversity). Empty list on failure — analysts research angle-less."""
+    if k < 2:
+        return []
+    try:
+        r = requests.post(f"{OLLAMA}/api/generate", json={
+            "model": planner_model,
+            "prompt": ("Identify exactly "
+                       f"{k} DISTINCT angles for researching this brief — different lenses "
+                       "that together cover it (e.g. regulatory, costs/economics, "
+                       "practitioner experience, recent developments — pick what fits THIS "
+                       'brief). Reply STRICT JSON only: ["angle one","angle two",…]\n\n'
+                       f"BRIEF:\n{brief}\n\nJSON:"),
+            "stream": False, "options": {"temperature": 0.3, "num_predict": 160},
+            "keep_alive": os.environ.get("PW_OLLAMA_KEEP_ALIVE", "30m")},  # warm the planner (R17)
+            timeout=float(os.environ.get("PW_OLLAMA_TIMEOUT", "300")))
+        r.raise_for_status()
+        parsed = _extract_json((r.json().get("response") or "").strip())
+        angles = [str(a).strip() for a in parsed if str(a).strip()] if isinstance(parsed, list) else []
+        return angles[:k]
+    except Exception:
+        return []
+
+
+def fix_dangling_citations(text: str) -> str:
+    """Drop [S#]/[L#] markers that don't resolve to a listed source (web [S#] or local
+    document [L#]); the source lists are appended inline so the check is local to the block."""
+    listed = set(re.findall(r"^\[([SL]\d+)\]", text, re.MULTILINE))
+    if not listed:
+        return text
+    return re.sub(r"\[([SL]\d+)(?:,\s*(?:[SL]\d+\s*)+)?\]",
+                  lambda m: m.group(0) if m.group(1) in listed else "", text)
+
+
+def run(brief: str, depth: str = "standard", editor_mode: str = "local",
+        out_dir: str = "reports", n_analysts: int = 3, scope: str = "both",
+        on_progress=None) -> pathlib.Path:
+    t0 = time.monotonic()
+    emit = _make_emit(on_progress)
+    brief = sanitize_brief(brief)                      # the one user input → clean + length-bound
+    if not brief:
+        raise ValueError("empty brief — give something to research")
+    os.environ.setdefault("PW_WEB_BACKEND", "ddgs")   # live web ON — the whole point
+
+    models = detect_models()
+    analysts, editor_model = pick_cast(models, n_analysts)
+    emit(f"🔬 Deep research ({depth}) — analysts: {', '.join(analysts)} · editor: "
+         f"{editor_model if editor_mode == 'local' else editor_mode}")
+
+    # STORM-lite perspective planning: distinct angle per analyst (smallest model plans)
+    angles = plan_angles(brief, models[0]["name"], len(analysts))
+    if angles:
+        emit("angles: " + " | ".join(angles))
+    page_evidence = not float(os.environ.get("PW_MODEL_CAP_GB", "0") or 0)  # CPU-capped → snippets
+
+    contributions, answers = [], []
+    for i, model in enumerate(analysts, 1):
+        angle = angles[i - 1] if i <= len(angles) else ""
+        emit(f"[{i}/{len(analysts)}] {model} researching the live web…"
+             + (f" (angle: {angle})" if angle else ""))
+        t = time.monotonic()
+        rw = ResearchWorker(worker_id=model, model=model, lens="independent analyst",
+                            country=os.environ.get("PW_COUNTRY", "your location"),
+                            depth=depth, angle=angle, page_evidence=page_evidence, scope=scope)
+        out = rw.research(brief)
+        text = fix_dangling_citations(out["text"])
+        nsrc = len(out["research"]["sources"])
+        emit(f"    {nsrc} sources · {len(text.split())} words · {time.monotonic()-t:.0f}s")
+        contributions.append({"country": model.split(":")[0], "model": model,
+                              "lens": "analyst", "text": text,
+                              "research": out["research"]})
+        answers.append(Answer(worker_id=model, model=model, lens="analyst",
+                              country=model.split(":")[0], text=text, tokens=out["tokens"],
+                              elapsed_s=out["elapsed_s"]))
+
+    emit("blind judge + editor compiling the report…")
+    if editor_mode == "local":
+        judge = Judge(model=editor_model)
+    else:
+        key = os.environ.get("OPENROUTER_API_KEY") or os.environ.get("PW_BASELINE_API_KEY")
+        if not key:
+            raise SystemExit("--editor api needs OPENROUTER_API_KEY (or PW_BASELINE_API_KEY)")
+        judge = _ApiEditor(os.environ.get("PW_EDITOR_MODEL", "openai/gpt-5-chat"), key,
+                           os.environ.get("PW_BASELINE_API_URL",
+                                          "https://openrouter.ai/api/v1/chat/completions"))
+    read = judge.deliberate(brief, answers)
+    report = judge.compile_report(brief, contributions, read, local=True)
+
+    out_path = pathlib.Path(out_dir)
+    out_path.mkdir(exist_ok=True)
+    slug = re.sub(r"[^a-z0-9]+", "-", brief.lower())[:60].strip("-")
+    fname = out_path / f"{datetime.date.today().isoformat()}-{slug}.md"
+    n = 1
+    while fname.exists():
+        n += 1
+        fname = out_path / f"{datetime.date.today().isoformat()}-{slug}-{n}.md"
+    fname.write_text(report)
+    mins = (time.monotonic() - t0) / 60
+    total_src = sum(len(c["research"]["sources"]) for c in contributions)
+    emit(f"📄 Report ready in {mins:.1f} min · {len(report.split())} words · "
+         f"{total_src} sources → {fname}")
+    return fname
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(prog="pw research",
+                                description="Local-first deep research with your own models.")
+    p.add_argument("brief", help="what should be researched")
+    p.add_argument("--quick", action="store_true", help="1 research round (fastest)")
+    p.add_argument("--deep", action="store_true", help="extra research rounds")
+    p.add_argument("--editor", choices=["local", "api"], default="local",
+                   help="api = BYOK frontier editor over local findings (OPENROUTER_API_KEY)")
+    p.add_argument("--analysts", type=int, default=3, help="how many local models analyze (1-4)")
+    p.add_argument("--out", default="reports", help="output directory")
+    g = p.add_mutually_exclusive_group()
+    g.add_argument("--local", action="store_true", help="research ONLY your library (no web)")
+    g.add_argument("--web", action="store_true", help="research ONLY the live web (no library)")
+    a = p.parse_args()
+    depth = "quick" if a.quick else "deep" if a.deep else "standard"
+    scope = "local" if a.local else "web" if a.web else "both"
+    run(a.brief, depth=depth, editor_mode=a.editor, out_dir=a.out,
+        n_analysts=max(1, min(4, a.analysts)), scope=scope)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

council/mcp_server.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""
+council/mcp_server.py — Passive Workers as an MCP tool (D19)
+============================================================
+Exposes the local research engine over the Model Context Protocol (stdio), so your OWN
+agentic AI — Claude Desktop, Codex, any MCP client — can call it as a tool. This is the
+project's interop play and the founder's worldview made real: the human's assistant
+orchestrates; our multi-model, live-web + private-library research engine is the capability
+it reaches for. Everything still runs locally; nothing leaves the machine but web searches.
+
+Run:  pw mcp        (or: python -m council.mcp_server)
+
+Claude Desktop config (claude_desktop_config.json):
+    {"mcpServers": {"passive-workers": {"command": "pw", "args": ["mcp"]}}}
+
+Tools:
+  research(brief, depth="quick", analysts=2, scope="both") -> cited markdown report
+  library_search(query, k=5)                               -> your private-document hits
+  library_add(path)                                        -> index a file/dir into the library
+"""
+
+from __future__ import annotations
+
+
+def _normalize_research_args(brief: str, depth: str, analysts, scope):
+    """Validate + clamp research() args at the MCP trust boundary. Returns
+    (brief, depth, analysts, scope, error): on a bad brief, error is a clean string and the rest
+    are unset; otherwise error is "". Every out-of-range value is clamped to a safe default rather
+    than raising — an MCP client never sees a traceback."""
+    from council.sanitize import sanitize_brief
+    brief = sanitize_brief(brief)
+    if not brief:
+        return "", "", 2, "both", "error: empty brief — provide a question to research."
+    depth = depth if depth in ("quick", "standard", "deep") else "quick"
+    scope = scope if scope in ("both", "web", "local") else "both"
+    try:
+        analysts = max(1, min(4, int(analysts)))
+    except (TypeError, ValueError):
+        analysts = 2
+    return brief, depth, analysts, scope, ""
+
+
+def build_server():
+    from mcp.server.fastmcp import FastMCP
+
+    mcp = FastMCP("passive-workers")
+
+    @mcp.tool()
+    def research(brief: str, depth: str = "quick", analysts: int = 2,
+                 scope: str = "both") -> str:
+        """Run multi-model local deep research (live web + your private library) and return a
+        cited markdown report. depth: quick|standard|deep. scope: both|web|local. Takes minutes."""
+        from council.local import run
+        brief, depth, analysts, scope, err = _normalize_research_args(brief, depth, analysts, scope)
+        if err:
+            return err
+        path = run(brief, depth=depth, n_analysts=analysts, scope=scope)
+        return path.read_text()
+
+    @mcp.tool()
+    def library_search(query: str, k: int = 5) -> str:
+        """Search your private document library; returns the top matching passages with sources."""
+        from council.library import Library
+        from council.sanitize import spotlight
+        hits = Library().search(query, k=max(1, min(20, int(k))))
+        if not hits:
+            return "(no matches — the library may be empty; add files with library_add)"
+        # document text is untrusted at this model-facing boundary → sanitize + spotlight
+        return "\n\n".join(f"[{h['title']}] (score {h['score']:.2f})\n{spotlight(h['text'][:800])}"
+                           for h in hits)
+
+    @mcp.tool()
+    def library_add(path: str) -> str:
+        """Index a local file or directory (PDF/docx/txt/md) into your private library."""
+        from council.library import Library
+        n = Library().add(path)
+        return f"Indexed {n} chunks from {path}."
+
+    return mcp
+
+
+def main() -> None:
+    build_server().run()   # stdio transport by default
+
+
+if __name__ == "__main__":
+    main()

council/net/__init__.py

@@ -0,0 +1 @@
+"""Networked Council — coordinator service + worker agent (M2)."""

council/net/agent.py

@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+"""
+council/net/agent.py — the networked worker daemon
+==================================================
+Runs on each contributor machine (this Mac, the VPS, …). It:
+  • registers with the coordinator (declaring its model, lens, country, owner, judge ability),
+  • heartbeats so the coordinator knows it's online and how loaded it is,
+  • polls for tasks, runs them on its LOCAL Ollama, and submits OWNED results,
+  • handles two task types: `answer` (a perspective) and `judge` (score + merge).
+
+It only ever DIALS OUT to the coordinator — no inbound connections to this machine.
+
+Config via env (or flags):
+  PW_COORDINATOR   e.g. http://VPS_IP:8088      (required)
+  PW_TOKEN         shared secret                 (required)
+  PW_OWNER         account that earns credit     (default: hostname)
+  PW_NAME, PW_COUNTRY, PW_ANSWER_MODEL, PW_LENS, PW_CAN_JUDGE, PW_JUDGE_MODEL
+
+Run:  python -m council.net.agent
+"""
+
+from __future__ import annotations
+
+import os
+import platform
+import signal
+import socket
+import sys
+import threading
+import time
+
+import requests
+
+from council.judge import Judge
+from council.sanitize import sanitize_brief
+from council.worker import Answer, PerspectiveWorker
+
+try:
+    import psutil
+except Exception:  # psutil optional
+    psutil = None
+
+
+def _env(k: str, default: str = "") -> str:
+    return os.environ.get(k, default)
+
+
+class Agent:
+    def __init__(self):
+        self.base = _env("PW_COORDINATOR").rstrip("/")
+        self.token = _env("PW_TOKEN", "dev-token")
+        if not self.base:
+            raise SystemExit("set PW_COORDINATOR (e.g. http://127.0.0.1:8088)")
+        host = socket.gethostname()
+        self.machine_id = _env("PW_MACHINE_ID", host)   # processes on one computer share this
+        self.owner = _env("PW_OWNER", host)
+        self.name = _env("PW_NAME", host)
+        self.country = _env("PW_COUNTRY", "local")
+        self.answer_model = _env("PW_ANSWER_MODEL", "gemma3:4b")
+        self.lens = _env("PW_LENS", "neutral")
+        self.can_judge = _env("PW_CAN_JUDGE", "0") in ("1", "true", "True")
+        self.judge_model = _env("PW_JUDGE_MODEL", self.answer_model if self.can_judge else "")
+        self.poll_s = float(_env("PW_POLL", "2"))
+        self.node_id: str | None = None
+        self.node_secret: str | None = None
+        self._running = True
+
+    # ------------------------------------------------------------------ http
+    def _headers(self) -> dict:
+        return {"X-PW-Token": self.token}
+
+    def _node_headers(self) -> dict:
+        h = {"X-PW-Token": self.token}
+        if self.node_secret:
+            h["X-Node-Secret"] = self.node_secret
+        return h
+
+    def _profile(self) -> dict:
+        prof = {"os": platform.system(), "machine": platform.machine()}
+        if psutil:
+            prof["ram_gb"] = round(psutil.virtual_memory().total / 1e9, 1)
+            prof["cores"] = psutil.cpu_count(logical=False) or psutil.cpu_count()
+        try:  # capability matching (D15 v1): which models this node can actually run
+            r = requests.get("http://localhost:11434/api/tags", timeout=5)
+            prof["models"] = sorted(m["name"] for m in r.json().get("models", []))[:40]
+        except Exception:
+            prof["models"] = []
+        return prof
+
+    def register(self) -> None:
+        body = {
+            "name": self.name, "country": self.country, "owner": self.owner,
+            "answer_model": self.answer_model, "lens": self.lens,
+            "can_judge": self.can_judge, "judge_model": self.judge_model,
+            "machine_id": self.machine_id, "profile": self._profile(),
+        }
+        r = requests.post(f"{self.base}/nodes/register", json=body, headers=self._headers(), timeout=15)
+        r.raise_for_status()
+        data = r.json()
+        self.node_id = data["node_id"]
+        self.node_secret = data.get("node_secret")
+        print(f"[agent] registered {self.name} ({self.answer_model}/{self.lens}/{self.country}) "
+              f"judge={self.can_judge} → node {self.node_id[:8]}…  @ {self.base}")
+
+    def heartbeat(self) -> None:
+        load = (psutil.cpu_percent(interval=None) / 100.0) if psutil else 0.0
+        try:
+            r = requests.post(f"{self.base}/nodes/heartbeat", json={"load": load},
+                              headers=self._node_headers(), timeout=10)
+            if r.status_code in (401, 404):  # coordinator restarted / forgot us
+                self.register()
+        except requests.RequestException as exc:
+            print(f"[agent] heartbeat failed: {exc}")
+
+    # ------------------------------------------------------------------ task handlers
+    def _do_answer(self, task: dict) -> dict:
+        payload = task.get("payload") or {}
+        # defense-in-depth (D26): the coordinator is not fully trusted (cf. D25) — re-scrub the
+        # brief/instruction here so a hostile coordinator can't slip a hidden payload into a prompt.
+        question = sanitize_brief(payload.get("question", ""))
+        if payload.get("job_type") == "shard_map":
+            # D13: batch shard — apply the instruction to THIS node's slice of the items.
+            from council.batch import BatchWorker
+            bw = BatchWorker(self.node_id, self.answer_model,
+                             country=task.get("country", self.country))
+            return bw.process(question, payload.get("shard") or [],
+                              fetch=bool(payload.get("fetch")))
+        if payload.get("job_type") == "research_report" \
+                and os.environ.get("PW_WEB_BACKEND", "off") != "off":
+            # D13: async deep-research job — this node's own multi-round, egress-localized
+            # research with citations (council/researcher.py).
+            from council.researcher import ResearchWorker
+            rw = ResearchWorker(self.node_id, self.answer_model,
+                                lens=task.get("lens", self.lens),
+                                country=task.get("country", self.country),
+                                scope="web")  # federation = web only; no operator's private library
+            return rw.research(question)
+        web = None
+        if os.environ.get("PW_WEB_BACKEND", "off") != "off":
+            try:
+                from council.research import search as web   # egress-localized web research
+            except Exception as exc:
+                print(f"[agent] web research unavailable: {exc}")
+                web = None
+        w = PerspectiveWorker(self.node_id, self.answer_model, lens=task.get("lens", self.lens),
+                              country=task.get("country", self.country), web_search=web,
+                              num_predict=int(os.environ.get("PW_NUM_PREDICT", "400")))
+        a = w.answer(question)
+        return {"text": a.text, "tokens": a.tokens, "elapsed_s": round(a.elapsed_s, 2)}
+
+    def _do_judge(self, task: dict) -> dict:
+        payload = task["payload"]
+        question = sanitize_brief(payload.get("question", ""))   # defense-in-depth (D26), see _do_answer
+        answers = [
+            Answer(worker_id=x["worker_id"], model=x.get("model", ""), lens=x.get("lens", ""),
+                   country=x.get("country", ""), text=x["text"], tokens=0, elapsed_s=0.0)
+            for x in payload["answers"]
+        ]
+        judge = Judge(model=self.judge_model or self.answer_model)
+        if payload.get("job_type") == "shard_map":
+            # Batch QA: spot-check sampled outputs per node; the store assembles the merged
+            # deliverable from the shards itself.
+            return judge.spot_check(question, payload["answers"])
+        out = judge.deliberate(question, answers)   # scores + merge + council read
+        if payload.get("job_type") == "research_report":
+            # Editor pass: merged becomes the full cited multi-country report.
+            out["merged"] = judge.compile_report(question, payload["answers"], out)
+        return out
+
+    # ------------------------------------------------------------------ loop
+    def _heartbeat_loop(self) -> None:
+        """Heartbeat in the background so a node stays 'alive' even mid-inference."""
+        while self._running:
+            self.heartbeat()
+            time.sleep(self.poll_s)
+
+    def run(self) -> None:
+        # Never die because the coordinator/tunnel is briefly down at boot — keep trying.
+        while self._running:
+            try:
+                self.register()
+                break
+            except requests.RequestException as exc:
+                print(f"[agent] register failed ({exc}); retrying in 10s…")
+                time.sleep(10)
+        threading.Thread(target=self._heartbeat_loop, daemon=True, name="pw-hb").start()
+        while self._running:
+            try:
+                r = requests.get(f"{self.base}/tasks/next",
+                                 headers=self._node_headers(), timeout=20)
+            except requests.RequestException as exc:
+                print(f"[agent] poll failed: {exc}")
+                time.sleep(self.poll_s)
+                continue
+            if r.status_code == 401:          # secret invalid; the heartbeat loop re-registers
+                time.sleep(self.poll_s)
+                continue
+            if r.status_code == 204 or not r.content:
+                time.sleep(self.poll_s)
+                continue
+            task = r.json()
+            kind = task["type"]
+            print(f"[agent] {kind} task {task['task_id'][:8]}… …")
+            t0 = time.monotonic()
+            try:
+                result = self._do_answer(task) if kind == "answer" else self._do_judge(task)
+            except Exception as exc:
+                print(f"[agent] task {task['task_id'][:8]} FAILED: {exc}")
+                result = {"text": "", "error": str(exc), "scores": {}, "merged": ""}
+            try:
+                requests.post(f"{self.base}/tasks/{task['task_id']}/result", json=result,
+                              headers=self._node_headers(), timeout=30)
+            except requests.RequestException as exc:
+                print(f"[agent] result POST failed (task {task['task_id'][:8]}): {exc}")
+            print(f"[agent] {kind} done in {time.monotonic() - t0:.0f}s")
+
+    def stop(self, *_):
+        print("\n[agent] shutting down…")
+        self._running = False
+
+
+def main() -> int:
+    agent = Agent()
+    signal.signal(signal.SIGINT, agent.stop)
+    signal.signal(signal.SIGTERM, agent.stop)
+    agent.run()
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())