pilot-workers 0.3.0__tar.gz → 0.4.0__tar.gz

{pilot_workers-0.3.0/src/pilot_workers.egg-info → pilot_workers-0.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pilot-workers
-Version: 0.3.0
+Version: 0.4.0
 Summary: Dispatch bounded tasks to isolated LLM workers (GLM, Kimi, and more)
 License-Expression: MIT
 Requires-Python: >=3.11
@@ -69,6 +69,7 @@ pilot-workers <subcommand> [args]
 
   run              Dispatch a task (streaming output).
   dispatch         Deterministic wrapper around run (two-line JSON: started + verdict).
+  fanout           Dispatch several jobs concurrently; stdout = one JSON array of verdicts.
   template         Print the task template for a mode (code|explore|test|review).
   install          Install integrations or runner.
                      install <provider|all> on <host|all>

{pilot_workers-0.3.0 → pilot_workers-0.4.0}/README.md

@@ -54,6 +54,7 @@ pilot-workers <subcommand> [args]
 
   run              Dispatch a task (streaming output).
   dispatch         Deterministic wrapper around run (two-line JSON: started + verdict).
+  fanout           Dispatch several jobs concurrently; stdout = one JSON array of verdicts.
   template         Print the task template for a mode (code|explore|test|review).
   install          Install integrations or runner.
                      install <provider|all> on <host|all>

{pilot_workers-0.3.0 → pilot_workers-0.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pilot-workers"
-version = "0.3.0"
+version = "0.4.0"
 description = "Dispatch bounded tasks to isolated LLM workers (GLM, Kimi, and more)"
 readme = "README.md"
 license = "MIT"

pilot_workers-0.4.0/src/pilot_workers/cli/fanout.py

@@ -0,0 +1,436 @@
+#!/usr/bin/env python3
+"""Concurrent multi-job dispatch with aggregated verdicts.
+
+Takes N pre-written task files, dispatches them concurrently (each via a
+``pilot_workers.cli.dispatch`` child process), re-emits each child's
+``worker_runner.started`` event on stderr, and prints ONE JSON array of
+verdicts on stdout -- one object per job, in the original job order.
+
+Log note: same-provider concurrent jobs interleave that provider's
+``latest.log`` (human log only; per-run jsonl/verdict.json files are still
+separate).
+
+Stdout contract (callers depend on it): exactly one JSON array, nothing
+else. Started events and warnings go to stderr only.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+from dataclasses import dataclass
+from typing import Any
+
+from pilot_workers import policy, providers, runtime
+from pilot_workers.runners import get_runner
+
+
+FANOUT_ERROR_EXIT = 2
+DEFAULT_TIMEOUT_S = 3600
+DEFAULT_IDLE_TIMEOUT_S = 900
+JOBS_FILE_FIELDS = {"provider", "mode", "task_file", "worktree"}
+SUCCESS_VERDICTS = ("completed", "step_capped_partial")
+
+
+@dataclass
+class Job:
+    provider: str
+    mode: str
+    task_file: str
+    worktree: bool = False
+
+    @property
+    def label(self) -> str:
+        return f"{self.provider}:{self.mode}"
+
+
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Dispatch several jobs concurrently; stdout is one JSON array "
+            "of verdicts in the original job order."
+        )
+    )
+    parser.add_argument(
+        "--workdir",
+        default=None,
+        help="Existing project directory passed to every worker.",
+    )
+    parser.add_argument(
+        "--job",
+        action="append",
+        default=[],
+        metavar="PROVIDER:MODE:TASK_FILE",
+        help="One job spec; repeatable. Parsed with a left split (paths may "
+             "contain colons).",
+    )
+    parser.add_argument(
+        "--jobs-file",
+        default=None,
+        metavar="JSON",
+        help="JSON array of job objects: provider, mode, task_file, worktree.",
+    )
+    parser.add_argument(
+        "--providers",
+        default=None,
+        metavar="KEYS",
+        help="Comma-separated provider keys (shorthand; requires --mode and "
+             "--task-file).",
+    )
+    parser.add_argument(
+        "--mode",
+        default=None,
+        help="Worker mode for the --providers shorthand.",
+    )
+    parser.add_argument(
+        "--task-file",
+        default=None,
+        help="Shared task file for the --providers shorthand.",
+    )
+    parser.add_argument(
+        "--max-parallel",
+        type=int,
+        default=None,
+        metavar="N",
+        help="Maximum concurrent jobs (default: number of jobs).",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=DEFAULT_TIMEOUT_S,
+        help=f"Per-job wall-clock limit in seconds (default {DEFAULT_TIMEOUT_S}).",
+    )
+    parser.add_argument(
+        "--idle-timeout",
+        type=int,
+        default=DEFAULT_IDLE_TIMEOUT_S,
+        help=f"Per-job idle limit in seconds (default {DEFAULT_IDLE_TIMEOUT_S}).",
+    )
+    return parser.parse_args(argv)
+
+
+# ---------------------------------------------------------------------------
+# Job list construction / validation
+# ---------------------------------------------------------------------------
+
+
+class _SpecError(RuntimeError):
+    pass
+
+
+def _validate_job(job: Job) -> None:
+    if job.provider not in providers.PROVIDERS:
+        raise _SpecError(f"unknown provider: {job.provider}")
+    if job.mode not in policy.MODE_TO_AGENT:
+        raise _SpecError(f"unknown mode: {job.mode}")
+    if job.mode == "resume":
+        raise _SpecError("resume is not supported in fanout")
+    if not Path(job.task_file).is_file():
+        raise _SpecError(f"task file not found: {job.task_file}")
+
+
+def _parse_job_spec(spec: str) -> Job:
+    parts = spec.split(":", 2)
+    if len(parts) != 3 or not all(parts):
+        raise _SpecError(f"invalid --job spec (expected PROVIDER:MODE:TASK_FILE): {spec}")
+    return Job(provider=parts[0], mode=parts[1], task_file=parts[2])
+
+
+def _load_jobs_file(path_arg: str) -> list[Job]:
+    try:
+        with open(path_arg, "r", encoding="utf-8") as handle:
+            data = json.load(handle)
+    except (OSError, json.JSONDecodeError) as exc:
+        raise _SpecError(f"cannot read jobs file: {exc}")
+    if not isinstance(data, list):
+        raise _SpecError("jobs file must be a JSON array of job objects")
+    jobs: list[Job] = []
+    for entry in data:
+        if not isinstance(entry, dict):
+            raise _SpecError("jobs file entries must be objects")
+        unknown = set(entry) - JOBS_FILE_FIELDS
+        if unknown:
+            raise _SpecError(
+                f"unknown jobs file fields: {', '.join(sorted(unknown))}"
+            )
+        missing = {"provider", "mode", "task_file"} - set(entry)
+        if missing:
+            raise _SpecError(
+                f"jobs file entry missing fields: {', '.join(sorted(missing))}"
+            )
+        jobs.append(
+            Job(
+                provider=str(entry["provider"]),
+                mode=str(entry["mode"]),
+                task_file=str(entry["task_file"]),
+                worktree=bool(entry.get("worktree", False)),
+            )
+        )
+    return jobs
+
+
+def _build_jobs(args: argparse.Namespace) -> list[Job]:
+    shorthand_used = any([args.providers, args.mode, args.task_file])
+    explicit_used = bool(args.job) or args.jobs_file is not None
+    if args.job and args.jobs_file is not None:
+        raise _SpecError("--job and --jobs-file are mutually exclusive")
+    if shorthand_used and explicit_used:
+        raise _SpecError(
+            "--providers/--mode/--task-file are mutually exclusive with "
+            "--job and --jobs-file"
+        )
+    if shorthand_used:
+        if not (args.providers and args.mode and args.task_file):
+            raise _SpecError("--providers requires --mode and --task-file")
+        jobs = [
+            Job(provider=key.strip(), mode=args.mode, task_file=args.task_file)
+            for key in args.providers.split(",")
+            if key.strip()
+        ]
+    elif args.jobs_file is not None:
+        jobs = _load_jobs_file(args.jobs_file)
+    else:
+        jobs = [_parse_job_spec(spec) for spec in args.job]
+    if not jobs:
+        raise _SpecError("at least one job is required")
+    for job in jobs:
+        _validate_job(job)
+    return jobs
+
+
+# ---------------------------------------------------------------------------
+# Safety guards
+# ---------------------------------------------------------------------------
+
+
+def _credential_preflight(jobs: list[Job]) -> None:
+    seen: set[str] = set()
+    for job in jobs:
+        if job.provider in seen:
+            continue
+        seen.add(job.provider)
+        provider = providers.PROVIDERS[job.provider]
+        meta = runtime.credential_metadata(provider, get_runner(provider.runner))
+        if not meta.get("configured"):
+            raise _SpecError(
+                f"credential missing for {job.provider}; "
+                f"run: pilot-workers credentials {job.provider}"
+            )
+
+
+def _workdir_collision_guard(jobs: list[Job]) -> None:
+    code_jobs = [job for job in jobs if job.mode == "code"]
+    if len(code_jobs) >= 2 and not all(job.worktree for job in code_jobs):
+        raise _SpecError(
+            "multiple code jobs share one workdir; use --jobs-file with "
+            'per-job "worktree": true'
+        )
+    test_jobs = [job for job in jobs if job.mode == "test"]
+    if len(test_jobs) >= 2:
+        print(
+            "note: concurrent test jobs share a workdir; caches may collide",
+            file=sys.stderr,
+        )
+
+
+# ---------------------------------------------------------------------------
+# Execution
+# ---------------------------------------------------------------------------
+
+
+def _build_dispatch_command(
+    job: Job, workdir: str, timeout: int, idle_timeout: int
+) -> list[str]:
+    cmd: list[str] = [
+        sys.executable,
+        "-m", "pilot_workers.cli.dispatch",
+        "--provider",
+        job.provider,
+        "--mode",
+        job.mode,
+        "--workdir",
+        workdir,
+        "--task-file",
+        job.task_file,
+    ]
+    if job.worktree:
+        cmd.append("--worktree")
+    cmd.extend(["--timeout", str(timeout)])
+    cmd.extend(["--idle-timeout", str(idle_timeout)])
+    return cmd
+
+
+def _synthesized_verdict(
+    job: Job, exit_code: Any, stderr_tail: str, interrupted: bool = False
+) -> dict[str, Any]:
+    verdict: dict[str, Any] = {
+        "job": job.label,
+        "type": "worker_runner.verdict",
+        "verdict": "error",
+        "synthesized": True,
+        "exit_code": exit_code,
+        "stderr_tail": stderr_tail,
+    }
+    if interrupted:
+        verdict["interrupted"] = True
+    return verdict
+
+
+def _run_job(
+    index: int,
+    job: Job,
+    cmd: list[str],
+    results: list[dict[str, Any] | None],
+    procs: list[Any],
+    stderr_lock: threading.Lock,
+) -> None:
+    """Run one dispatch child; never raises out -- always records a verdict."""
+    proc = None
+    try:
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            bufsize=1,
+        )
+        procs[index] = proc
+
+        stderr_chunks: list[str] = []
+
+        def _drain_stderr() -> None:
+            try:
+                assert proc.stderr is not None
+                stderr_chunks.append(proc.stderr.read())
+            except Exception:
+                pass
+
+        drain = threading.Thread(target=_drain_stderr, daemon=True)
+        drain.start()
+
+        started_seen = False
+        last_line = ""
+        assert proc.stdout is not None
+        for raw_line in proc.stdout:
+            line = raw_line.rstrip("\n")
+            if not line:
+                continue
+            if not started_seen:
+                started_seen = True
+                try:
+                    event = json.loads(line)
+                except json.JSONDecodeError:
+                    event = None
+                if isinstance(event, dict):
+                    with stderr_lock:
+                        sys.stderr.write(
+                            json.dumps({"job": job.label, **event},
+                                       ensure_ascii=False)
+                            + "\n"
+                        )
+                        sys.stderr.flush()
+            last_line = line
+        proc.wait()
+        drain.join(timeout=5)
+        stderr_text = "".join(stderr_chunks)
+
+        verdict: dict[str, Any] | None = None
+        if last_line:
+            try:
+                parsed = json.loads(last_line)
+            except json.JSONDecodeError:
+                parsed = None
+            if isinstance(parsed, dict):
+                parsed["job"] = job.label
+                verdict = parsed
+        if verdict is None:
+            verdict = _synthesized_verdict(
+                job, proc.returncode, stderr_text[-500:]
+            )
+        results[index] = verdict
+    except Exception as exc:  # the array must always be printed
+        results[index] = _synthesized_verdict(
+            job, getattr(proc, "returncode", None), str(exc)[-500:]
+        )
+
+
+def run_fanout(args: argparse.Namespace) -> int:
+    if not args.workdir:
+        print("error: --workdir is required", file=sys.stderr)
+        return FANOUT_ERROR_EXIT
+    try:
+        jobs = _build_jobs(args)
+        _credential_preflight(jobs)
+        _workdir_collision_guard(jobs)
+    except _SpecError as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return FANOUT_ERROR_EXIT
+
+    max_workers = args.max_parallel or len(jobs)
+    results: list[dict[str, Any] | None] = [None] * len(jobs)
+    procs: list[Any] = [None] * len(jobs)
+    stderr_lock = threading.Lock()
+    interrupted = False
+
+    try:
+        with ThreadPoolExecutor(max_workers=max_workers) as pool:
+            futures = [
+                pool.submit(
+                    _run_job,
+                    index,
+                    job,
+                    _build_dispatch_command(
+                        job, args.workdir, args.timeout, args.idle_timeout
+                    ),
+                    results,
+                    procs,
+                    stderr_lock,
+                )
+                for index, job in enumerate(jobs)
+            ]
+            for future in futures:
+                future.result()
+    except KeyboardInterrupt:
+        interrupted = True
+        # Children received SIGINT via the process group and self-terminate;
+        # give them a bounded grace period to reap.
+        deadline = time.monotonic() + runtime.TERMINATE_GRACE_SECONDS + 5
+        for proc in procs:
+            if proc is None:
+                continue
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                break
+            try:
+                proc.wait(timeout=remaining)
+            except Exception:
+                pass
+
+    for index, job in enumerate(jobs):
+        if results[index] is None:
+            results[index] = _synthesized_verdict(job, None, "", interrupted)
+
+    final: list[dict[str, Any]] = [r for r in results if r is not None]
+    sys.stdout.write(json.dumps(final, ensure_ascii=False) + "\n")
+    sys.stdout.flush()
+
+    if interrupted:
+        return 1
+    if all(r.get("verdict") in SUCCESS_VERDICTS for r in final):
+        return 0
+    return 1
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = parse_args(argv)
+    return run_fanout(args)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

{pilot_workers-0.3.0 → pilot_workers-0.4.0}/src/pilot_workers/cli/main.py

@@ -12,6 +12,7 @@ USAGE = """usage: pilot-workers <subcommand> [args]
 subcommands:
   run          Dispatch a bounded task to an isolated LLM worker.
   dispatch     Deterministic outer shell around run (started + verdict JSON).
+  fanout       Dispatch several jobs concurrently; stdout = one JSON array of verdicts.
   template     Print the task template for a mode (code|explore|test|review).
   install      install <provider|all> on <host|all> | install runner <name>.
   uninstall    uninstall <provider|all> on <host|all> | uninstall runner <name>.
@@ -55,6 +56,11 @@ def main(argv: list[str] | None = None) -> int:
 
         return dispatch_main(rest)
 
+    if subcommand == "fanout":
+        from pilot_workers.cli.fanout import main as fanout_main
+
+        return fanout_main(rest)
+
     if subcommand == "template":
         import pilot_workers
 

{pilot_workers-0.3.0 → pilot_workers-0.4.0/src/pilot_workers.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pilot-workers
-Version: 0.3.0
+Version: 0.4.0
 Summary: Dispatch bounded tasks to isolated LLM workers (GLM, Kimi, and more)
 License-Expression: MIT
 Requires-Python: >=3.11
@@ -69,6 +69,7 @@ pilot-workers <subcommand> [args]
 
   run              Dispatch a task (streaming output).
   dispatch         Deterministic wrapper around run (two-line JSON: started + verdict).
+  fanout           Dispatch several jobs concurrently; stdout = one JSON array of verdicts.
   template         Print the task template for a mode (code|explore|test|review).
   install          Install integrations or runner.
                      install <provider|all> on <host|all>

{pilot_workers-0.3.0 → pilot_workers-0.4.0}/src/pilot_workers.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ src/pilot_workers.egg-info/requires.txt
 src/pilot_workers.egg-info/top_level.txt
 src/pilot_workers/cli/__init__.py
 src/pilot_workers/cli/dispatch.py
+src/pilot_workers/cli/fanout.py
 src/pilot_workers/cli/install.py
 src/pilot_workers/cli/main.py
 src/pilot_workers/cli/run.py
@@ -71,6 +72,7 @@ src/pilot_workers/scripts/install_runtime.sh
 tests/test_cli_main.py
 tests/test_credentials.py
 tests/test_dispatch.py
+tests/test_fanout.py
 tests/test_install.py
 tests/test_opencode_runner.py
 tests/test_policy.py

pilot_workers-0.4.0/tests/test_fanout.py

@@ -0,0 +1,346 @@
+"""Offline tests for pilot_workers.cli.fanout."""
+
+from __future__ import annotations
+
+import io
+import json
+
+import pytest
+
+from pilot_workers import runtime
+from pilot_workers.cli import fanout as fanout_mod
+
+
+def _allow_credentials(monkeypatch):
+    monkeypatch.setattr(
+        runtime,
+        "credential_metadata",
+        lambda provider, runner: {"configured": True, "secure_mode": True},
+    )
+
+
+def _deny_credentials(monkeypatch):
+    monkeypatch.setattr(
+        runtime,
+        "credential_metadata",
+        lambda provider, runner: {"configured": False, "secure_mode": False},
+    )
+
+
+class _FakePopen:
+    """Minimal Popen stand-in: canned stdout lines, stderr text, returncode."""
+
+    def __init__(self, cmd, behavior):
+        self.cmd = cmd
+        lines = behavior.get("stdout_lines", [])
+        self.stdout = io.StringIO("".join(line + "\n" for line in lines))
+        self.stderr = io.StringIO(behavior.get("stderr", ""))
+        self._rc = behavior.get("returncode", 0)
+        self.returncode = None
+
+    def wait(self, timeout=None):
+        self.returncode = self._rc
+        return self._rc
+
+    def poll(self):
+        return self.returncode
+
+
+def _install_fake_popen(monkeypatch, behavior_for):
+    """Patch subprocess.Popen; behavior_for(cmd) -> behavior dict.
+
+    Returns the list of commands seen (thread-safe enough for tests).
+    """
+    calls = []
+
+    def fake_popen(cmd, stdout=None, stderr=None, text=None, bufsize=None):
+        calls.append(cmd)
+        return _FakePopen(cmd, behavior_for(cmd))
+
+    monkeypatch.setattr(fanout_mod.subprocess, "Popen", fake_popen)
+    return calls
+
+
+def _provider_of(cmd):
+    return cmd[cmd.index("--provider") + 1]
+
+
+def _verdict_line(verdict, provider="glm"):
+    return json.dumps({
+        "type": "worker_runner.verdict",
+        "provider": provider,
+        "verdict": verdict,
+        "exit_code": 0,
+    })
+
+
+def _started_line(provider="glm"):
+    return json.dumps({
+        "type": "worker_runner.started",
+        "provider": provider,
+        "run_id": "r1",
+    })
+
+
+def _ok_behavior(cmd):
+    provider = _provider_of(cmd)
+    return {
+        "stdout_lines": [_started_line(provider), _verdict_line("completed", provider)],
+        "returncode": 0,
+    }
+
+
+@pytest.fixture
+def task_file(tmp_path):
+    path = tmp_path / "task.md"
+    path.write_text("# task\n", encoding="utf-8")
+    return str(path)
+
+
+@pytest.fixture
+def workdir(tmp_path):
+    return str(tmp_path)
+
+
+# ---------------------------------------------------------------------------
+# Grammar
+# ---------------------------------------------------------------------------
+
+
+def test_job_spec_parses(monkeypatch, capsys, tmp_path, task_file, workdir):
+    _allow_credentials(monkeypatch)
+    _install_fake_popen(monkeypatch, _ok_behavior)
+    rc = fanout_mod.main([
+        "--workdir", workdir, "--job", f"glm:review:{task_file}",
+    ])
+    assert rc == 0
+    results = json.loads(capsys.readouterr().out)
+    assert [r["job"] for r in results] == ["glm:review"]
+
+
+def test_job_spec_path_with_colons_parses_via_left_split(
+    monkeypatch, capsys, tmp_path, workdir
+):
+    colon_path = tmp_path / "a:b.md"
+    colon_path.write_text("# task\n", encoding="utf-8")
+    _allow_credentials(monkeypatch)
+    calls = _install_fake_popen(monkeypatch, _ok_behavior)
+    rc = fanout_mod.main([
+        "--workdir", workdir, "--job", f"glm:review:{colon_path}",
+    ])
+    assert rc == 0
+    assert calls[0][calls[0].index("--task-file") + 1] == str(colon_path)
+
+
+def test_unknown_provider_exits_2(capsys, task_file, workdir):
+    rc = fanout_mod.main([
+        "--workdir", workdir, "--job", f"bogus:review:{task_file}",
+    ])
+    assert rc == 2
+    assert "unknown provider" in capsys.readouterr().err
+
+
+def test_resume_mode_rejected(capsys, task_file, workdir):
+    rc = fanout_mod.main([
+        "--workdir", workdir, "--job", f"glm:resume:{task_file}",
+    ])
+    assert rc == 2
+    assert "resume is not supported in fanout" in capsys.readouterr().err
+
+
+def test_no_jobs_exits_2(capsys, workdir):
+    rc = fanout_mod.main(["--workdir", workdir])
+    assert rc == 2
+
+
+def test_providers_shorthand_expands(monkeypatch, capsys, task_file, workdir):
+    _allow_credentials(monkeypatch)
+    calls = _install_fake_popen(monkeypatch, _ok_behavior)
+    rc = fanout_mod.main([
+        "--workdir", workdir,
+        "--providers", "kimi-k3,glm",
+        "--mode", "review",
+        "--task-file", task_file,
+    ])
+    assert rc == 0
+    results = json.loads(capsys.readouterr().out)
+    assert [r["job"] for r in results] == ["kimi-k3:review", "glm:review"]
+    assert len(calls) == 2
+
+
+def test_job_and_jobs_file_together_exits_2(capsys, tmp_path, task_file, workdir):
+    jobs_file = tmp_path / "jobs.json"
+    jobs_file.write_text("[]", encoding="utf-8")
+    rc = fanout_mod.main([
+        "--workdir", workdir,
+        "--job", f"glm:review:{task_file}",
+        "--jobs-file", str(jobs_file),
+    ])
+    assert rc == 2
+
+
+# ---------------------------------------------------------------------------
+# Safety guards
+# ---------------------------------------------------------------------------
+
+
+def test_credential_preflight_failure_spawns_nothing(
+    monkeypatch, capsys, task_file, workdir
+):
+    _deny_credentials(monkeypatch)
+
+    def forbidden_popen(*args, **kwargs):
+        raise AssertionError("Popen must not be called")
+
+    monkeypatch.setattr(fanout_mod.subprocess, "Popen", forbidden_popen)
+    rc = fanout_mod.main([
+        "--workdir", workdir, "--job", f"glm:review:{task_file}",
+    ])
+    assert rc == 2
+    err = capsys.readouterr().err
+    assert "credential missing for glm" in err
+    assert "pilot-workers credentials glm" in err
+
+
+def _write_jobs_file(tmp_path, jobs):
+    path = tmp_path / "jobs.json"
+    path.write_text(json.dumps(jobs), encoding="utf-8")
+    return str(path)
+
+
+def test_code_jobs_same_workdir_without_worktree_exits_2(
+    monkeypatch, capsys, tmp_path, task_file, workdir
+):
+    _allow_credentials(monkeypatch)
+    _install_fake_popen(monkeypatch, _ok_behavior)
+    jobs_file = _write_jobs_file(tmp_path, [
+        {"provider": "glm", "mode": "code", "task_file": task_file},
+        {"provider": "kimi-k3", "mode": "code", "task_file": task_file},
+    ])
+    rc = fanout_mod.main(["--workdir", workdir, "--jobs-file", jobs_file])
+    assert rc == 2
+    assert "multiple code jobs share one workdir" in capsys.readouterr().err
+
+
+def test_code_jobs_with_worktree_proceed(
+    monkeypatch, capsys, tmp_path, task_file, workdir
+):
+    _allow_credentials(monkeypatch)
+    calls = _install_fake_popen(monkeypatch, _ok_behavior)
+    jobs_file = _write_jobs_file(tmp_path, [
+        {"provider": "glm", "mode": "code", "task_file": task_file,
+         "worktree": True},
+        {"provider": "kimi-k3", "mode": "code", "task_file": task_file,
+         "worktree": True},
+    ])
+    rc = fanout_mod.main(["--workdir", workdir, "--jobs-file", jobs_file])
+    assert rc == 0
+    assert all("--worktree" in cmd for cmd in calls)
+
+
+def test_test_jobs_same_workdir_warn_but_proceed(
+    monkeypatch, capsys, tmp_path, task_file, workdir
+):
+    _allow_credentials(monkeypatch)
+    _install_fake_popen(monkeypatch, _ok_behavior)
+    jobs_file = _write_jobs_file(tmp_path, [
+        {"provider": "glm", "mode": "test", "task_file": task_file},
+        {"provider": "kimi-k3", "mode": "test", "task_file": task_file},
+    ])
+    rc = fanout_mod.main(["--workdir", workdir, "--jobs-file", jobs_file])
+    assert rc == 0
+    assert "concurrent test jobs share a workdir" in capsys.readouterr().err
+
+
+def test_jobs_file_unknown_field_exits_2(capsys, tmp_path, task_file, workdir):
+    jobs_file = _write_jobs_file(tmp_path, [
+        {"provider": "glm", "mode": "review", "task_file": task_file,
+         "bogus": 1},
+    ])
+    rc = fanout_mod.main(["--workdir", workdir, "--jobs-file", jobs_file])
+    assert rc == 2
+
+
+# ---------------------------------------------------------------------------
+# Aggregation / verdicts
+# ---------------------------------------------------------------------------
+
+
+def test_aggregation_order_and_started_reemission(
+    monkeypatch, capsys, task_file, workdir
+):
+    _allow_credentials(monkeypatch)
+    _install_fake_popen(monkeypatch, _ok_behavior)
+    rc = fanout_mod.main([
+        "--workdir", workdir,
+        "--job", f"glm:review:{task_file}",
+        "--job", f"kimi-k3:explore:{task_file}",
+    ])
+    assert rc == 0
+    captured = capsys.readouterr()
+    results = json.loads(captured.out)
+    assert len(results) == 2
+    assert [r["job"] for r in results] == ["glm:review", "kimi-k3:explore"]
+    assert all(r["type"] == "worker_runner.verdict" for r in results)
+    started = [
+        json.loads(line) for line in captured.err.splitlines()
+        if line.startswith("{")
+    ]
+    assert {s["job"] for s in started} == {"glm:review", "kimi-k3:explore"}
+    assert all(s["type"] == "worker_runner.started" for s in started)
+
+
+def test_silent_child_gets_synthesized_verdict(
+    monkeypatch, capsys, task_file, workdir
+):
+    _allow_credentials(monkeypatch)
+
+    def behavior_for(cmd):
+        if _provider_of(cmd) == "glm":
+            return _ok_behavior(cmd)
+        return {"stdout_lines": [], "stderr": "boom " * 200, "returncode": 2}
+
+    _install_fake_popen(monkeypatch, behavior_for)
+    rc = fanout_mod.main([
+        "--workdir", workdir,
+        "--job", f"glm:review:{task_file}",
+        "--job", f"kimi-k3:review:{task_file}",
+    ])
+    assert rc == 1
+    results = json.loads(capsys.readouterr().out)
+    assert results[0]["job"] == "glm:review"
+    assert results[0]["verdict"] == "completed"
+    assert "synthesized" not in results[0]
+    assert results[1]["job"] == "kimi-k3:review"
+    assert results[1]["verdict"] == "error"
+    assert results[1]["synthesized"] is True
+    assert results[1]["exit_code"] == 2
+    assert len(results[1]["stderr_tail"]) <= 500
+    assert "boom" in results[1]["stderr_tail"]
+
+
+@pytest.mark.parametrize("verdict,expected_rc", [
+    ("completed", 0),
+    ("step_capped_partial", 0),
+    ("error", 1),
+    ("empty", 1),
+])
+def test_exit_code_from_verdicts(
+    monkeypatch, capsys, task_file, workdir, verdict, expected_rc
+):
+    _allow_credentials(monkeypatch)
+
+    def behavior_for(cmd):
+        provider = _provider_of(cmd)
+        return {
+            "stdout_lines": [
+                _started_line(provider), _verdict_line(verdict, provider),
+            ],
+            "returncode": 0,
+        }
+
+    _install_fake_popen(monkeypatch, behavior_for)
+    rc = fanout_mod.main([
+        "--workdir", workdir, "--job", f"glm:review:{task_file}",
+    ])
+    assert rc == expected_rc