pdl-taskmaster 2.4.0__py3-none-any.whl

pdl_taskmaster/providers/codex_worker.py

@@ -0,0 +1,328 @@
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from pathlib import Path
+from typing import Any, Callable
+
+from pdl_taskmaster.providers.base import TransportError, WorkerResult
+
+
+class CodexWorker:
+    """LIVE SEMANTIC WORKER backed by the Codex CLI.
+
+    This adapter is a semantic worker only. It is NOT a coding domain
+    execution backend: it does not own PDL wire validity, controller routing,
+    or filesystem artifact production on behalf of the candidate.
+
+    Label: DEVELOPMENT / LIVE DEMONSTRATION WORKER
+    NOT A QUALIFIED R2S MEASUREMENT CONDITION.
+
+    Safety defaults (normal local operation):
+        approval mode   = never
+        sandbox mode    = read-only
+        dangerous bypass = OFF
+
+    The bypass condition is a separate, explicitly opt-in execution
+    environment intended only for externally hardened/disposable runners. It
+    is never used as an automatic fallback.
+    """
+
+    def __init__(
+        self,
+        *,
+        model: str | None = None,
+        workdir: str | Path | None = None,
+        allowed_workdir_root: str | Path | None = None,
+        timeout: float = 600.0,
+        progress_path: str | Path | None = None,
+        on_progress: Callable[[str], None] | None = None,
+        capture_tokens: bool = True,
+        sandbox_mode: str = "read-only",
+        approval_policy: str = "never",
+        allow_bypass: bool = False,
+        json_mode: bool | None = None,
+        config_overrides: list[str] | None = None,
+    ):
+        if sandbox_mode not in {"read-only", "workspace-write"}:
+            raise ValueError(f"unsupported sandbox mode: {sandbox_mode}")
+        if approval_policy not in {"never"}:
+            raise ValueError(f"unsupported approval policy: {approval_policy}")
+        initial_workdir = str(Path(workdir).resolve()) if workdir else str(Path.cwd().resolve())
+        self.model = model
+        self.timeout = timeout
+        self.progress_path = Path(progress_path) if progress_path else None
+        self.on_progress = on_progress
+        self.capture_tokens = capture_tokens
+        self.sandbox_mode = sandbox_mode
+        self.approval_policy = approval_policy
+        self.allow_bypass = allow_bypass
+        self.json_mode = capture_tokens if json_mode is None else json_mode
+        self.config_overrides = list(config_overrides or [])
+        self.worker_profile = "codex"
+        self.effective_model: str | None = self._read_effective_model()
+        self.observed_model: str | None = None
+        self.allowed_workdir_root = str(
+            Path(allowed_workdir_root).resolve() if allowed_workdir_root else Path(initial_workdir).resolve()
+        )
+        self.workdir = initial_workdir
+        self.codex_cli_version = self._detect_version()
+
+    @staticmethod
+    def _detect_version() -> str:
+        try:
+            proc = subprocess.run(
+                ["codex", "--version"],
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=15,
+            )
+            return (proc.stdout or proc.stderr or "").strip() or "unknown"
+        except Exception:
+            return "unknown"
+
+    @staticmethod
+    def _read_effective_model() -> str | None:
+        try:
+            config_path = Path.home() / ".codex" / "config.toml"
+            if not config_path.is_file():
+                return None
+            for line in config_path.read_text(encoding="utf-8").splitlines():
+                stripped = line.strip()
+                if not stripped or stripped.startswith("#"):
+                    continue
+                match = re.match(r'^model\s*=\s*"([^"]+)"', stripped)
+                if match:
+                    return match.group(1)
+        except Exception:
+            return None
+        return None
+
+    @property
+    def workdir(self) -> str:
+        return self._workdir
+
+    @workdir.setter
+    def workdir(self, value: str | Path) -> None:
+        resolved = str(Path(value).resolve())
+        allowed = Path(self.allowed_workdir_root).resolve()
+        candidate = Path(resolved).resolve()
+        if candidate != allowed and not candidate.is_relative_to(allowed):
+            raise ValueError(f"workdir must stay inside allowed root {allowed}: {resolved}")
+        self._workdir = resolved
+
+    @staticmethod
+    def _format_progress(line: str, json_mode: bool) -> str:
+        if not json_mode or not line.startswith("{"):
+            return line
+        try:
+            event = json.loads(line)
+        except Exception:
+            return line
+        event_type = event.get("type") or "event"
+        payload = event.get("payload")
+        if isinstance(payload, dict):
+            message = payload.get("message") or payload.get("text")
+            if isinstance(message, str) and message.strip():
+                message = message.strip().replace("\n", " ")
+                if len(message) > 240:
+                    message = message[:237] + "..."
+                return f"{event_type}: {message}"
+        return event_type
+
+    def call(self, request: Any) -> WorkerResult:
+        prompt = (
+            request.prompt
+            + "\n\nReturn only a JSON object. Do not include markdown fences, commentary, or extra text."
+        )
+        fd, output_path = tempfile.mkstemp(prefix="codex-worker-", suffix=".txt")
+        os.close(fd)
+        output_lines: list[str] = []
+        started = time.perf_counter()
+        try:
+            Path(self.workdir).mkdir(parents=True, exist_ok=True)
+            cmd = [
+                "codex",
+                "exec",
+                "-o",
+                output_path,
+                "--ephemeral",
+                "--skip-git-repo-check",
+                "-C",
+                self.workdir,
+            ]
+            if self.model:
+                cmd += ["-m", self.model]
+            if self.allow_bypass:
+                # Separate hardened/disposable execution condition only.
+                cmd.append("--dangerously-bypass-approvals-and-sandbox")
+            else:
+                cmd += ["--sandbox", self.sandbox_mode, "-c", 'approval_policy="never"']
+                for override in self.config_overrides:
+                    cmd += ["-c", override]
+                if self.json_mode:
+                    cmd.append("--json")
+            cmd.append(prompt)
+            popen_kwargs: dict[str, Any] = {}
+            if sys.platform != "win32":
+                popen_kwargs["start_new_session"] = True
+            proc = subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                **popen_kwargs,
+            )
+
+            def _stream() -> None:
+                assert proc.stdout is not None
+                for line in proc.stdout:
+                    text_line = line.rstrip()
+                    output_lines.append(text_line)
+                    if self.progress_path is not None:
+                        self.progress_path.parent.mkdir(parents=True, exist_ok=True)
+                        with self.progress_path.open("a", encoding="utf-8", newline="\n") as handle:
+                            handle.write(text_line + "\n")
+                    if self.on_progress is not None:
+                        try:
+                            display = self._format_progress(text_line, self.json_mode)
+                            if display.strip():
+                                self.on_progress(display)
+                        except Exception:
+                            pass
+
+            thread = threading.Thread(target=_stream, daemon=True)
+            thread.start()
+            try:
+                proc.wait(timeout=self.timeout)
+            except subprocess.TimeoutExpired as exc:
+                if sys.platform == "win32":
+                    subprocess.run(
+                        ["taskkill", "/PID", str(proc.pid), "/T", "/F"],
+                        capture_output=True,
+                        text=True,
+                        encoding="utf-8",
+                        errors="replace",
+                    )
+                else:
+                    import signal
+                    try:
+                        os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+                    except OSError:
+                        proc.kill()
+                thread.join(timeout=5)
+                raise TransportError(f"Codex worker timed out after {self.timeout}s") from exc
+            thread.join(timeout=5)
+            if proc.returncode != 0:
+                raise TransportError(
+                    f"Codex exec failed rc={proc.returncode}; see progress log for details"
+                )
+            text = Path(output_path).read_text(encoding="utf-8", errors="replace").strip()
+            if not text:
+                raise TransportError("Codex returned an empty worker response")
+
+            latency_ms = (time.perf_counter() - started) * 1000.0
+            usage, usage_source, usage_exact, response_id, observed_model = self._parse_telemetry(output_lines)
+            self.observed_model = observed_model
+            metadata: dict[str, Any] = {
+                "worker": "codex",
+                "model": self.model,
+                "observed_model": observed_model,
+                "mode": "live-demonstration",
+                "not_a_qualified_measurement_condition": True,
+                "sandbox_mode": "bypass" if self.allow_bypass else self.sandbox_mode,
+                "approval_mode": "bypass" if self.allow_bypass else self.approval_policy,
+                "codex_cli_version": self.codex_cli_version,
+                "workdir": self.workdir,
+                "json_mode": self.json_mode,
+                "response_id": response_id,
+                "latency_ms": round(latency_ms, 3),
+                "usage_source": usage_source,
+                "usage_exact": usage_exact,
+                "token_telemetry": "enabled" if self.capture_tokens else "disabled",
+            }
+            if usage:
+                metadata["usage"] = usage
+                metadata["codex_cli_reported_total"] = usage.get("codex_cli_reported_total")
+            return WorkerResult(text, metadata)
+        finally:
+            try:
+                os.unlink(output_path)
+            except OSError:
+                pass
+
+    def _parse_telemetry(self, output_lines: list[str]) -> tuple[dict[str, Any] | None, str | None, bool, str | None, str | None]:
+        usage: dict[str, Any] | None = None
+        usage_source: str | None = None
+        usage_exact = False
+        response_id: str | None = None
+        observed_model: str | None = None
+
+        if self.capture_tokens and self.json_mode:
+            for line in output_lines:
+                if not line.startswith("{"):
+                    continue
+                try:
+                    event = json.loads(line)
+                except Exception:
+                    continue
+                payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
+                candidate: dict[str, Any] | None = None
+                if isinstance(event.get("usage"), dict):
+                    candidate = event["usage"]
+                elif isinstance(payload.get("usage"), dict):
+                    candidate = payload["usage"]
+                if candidate is not None:
+                    structured: dict[str, Any] = {}
+                    for key, value in candidate.items():
+                        if isinstance(value, (int, float)) and key in {
+                            "input_tokens",
+                            "output_tokens",
+                            "total_tokens",
+                            "prompt_tokens",
+                            "completion_tokens",
+                            "reasoning_tokens",
+                            "cached_input_tokens",
+                            "uncached_input_tokens",
+                            "tokens",
+                            "tokens_used",
+                        }:
+                            structured[f"codex_cli_{key}"] = value
+                    if structured:
+                        usage = structured
+                        usage_source = "codex_cli_json"
+                        usage_exact = True
+                if response_id is None:
+                    for key in ("response_id", "id", "session_id"):
+                        if isinstance(event.get(key), str):
+                            response_id = event[key]
+                            break
+                if response_id is None:
+                    for key in ("response_id", "id", "session_id"):
+                        if isinstance(payload.get(key), str):
+                            response_id = payload[key]
+                            break
+                if observed_model is None and isinstance(event.get("model"), str):
+                    observed_model = event["model"]
+                elif observed_model is None and isinstance(payload.get("model"), str):
+                    observed_model = payload["model"]
+
+        if usage_source is None and self.capture_tokens:
+            for output_line in reversed(output_lines):
+                match = re.search(r"tokens used\s*([\d,]+)", output_line, re.IGNORECASE)
+                if match:
+                    usage = {"codex_cli_reported_total": int(match.group(1).replace(",", ""))}
+                    usage_source = "codex_cli_plaintext"
+                    usage_exact = False
+                    break
+        return usage, usage_source, usage_exact, response_id, observed_model

pdl_taskmaster/providers/fixtures.py

@@ -0,0 +1,157 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+import hashlib
+import sys
+import tempfile
+
+from pdl_taskmaster.providers.recorded import RecordedFixtureBuilder, RecordedWorker
+
+
+def _load_jsonl(path: Path) -> list[dict[str, Any]]:
+    return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def _load_candidate(candidate_repo: str | Path) -> str:
+    repo = str(Path(candidate_repo).resolve())
+    if repo not in sys.path:
+        sys.path.insert(0, repo)
+    return repo
+
+
+def _ordered_recorded_calls(workspaces: list[Path]) -> list[dict[str, Any]]:
+    calls: list[dict[str, Any]] = []
+    for workspace in workspaces:
+        workspace_calls: list[dict[str, Any]] = []
+        response_paths = list(workspace.glob("stages/*/output/*/model-response.txt")) + list(
+            workspace.glob("turns/*/stages/*/output/*/model-response.txt")
+        )
+        for response_path in response_paths:
+            projection_path = response_path.parent / "compiled-projection.json"
+            if not projection_path.is_file():
+                continue
+            projection = json.loads(projection_path.read_text(encoding="utf-8"))
+            workspace_calls.append(
+                {
+                    "operation": projection.get("operation", "UNKNOWN"),
+                    "response": response_path.read_text(encoding="utf-8"),
+                    "invocation_id": response_path.parent.name,
+                }
+            )
+        workspace_calls.sort(key=lambda item: item["invocation_id"])
+        calls.extend(workspace_calls)
+    return calls
+
+
+def _resolve_workspaces(eval_root: Path, row: dict[str, Any]) -> list[Path]:
+    base = eval_root / "evidence" / "round4" / "workspaces" / row["run_id"] / row["case_id"]
+    candidates = sorted(base.glob("W-*")) if base.is_dir() else []
+    if not candidates:
+        raise SystemExit(f"cannot resolve workspaces for case {row.get('case_id')} run {row.get('run_id')}")
+
+    def _created_at(workspace: Path) -> str:
+        meta_path = workspace / "workspace.json"
+        if meta_path.is_file():
+            value = json.loads(meta_path.read_text(encoding="utf-8")).get("created_at_utc")
+            if isinstance(value, str):
+                return value
+        return workspace.name
+
+    return sorted(candidates, key=_created_at)
+
+
+def build_recorded_fixture(
+    candidate_repo: str | Path,
+    eval_root: str | Path,
+    evidence_jsonl: str | Path,
+    *,
+    case_ids: list[str] | None = None,
+    run_id: str | None = None,
+) -> RecordedWorker:
+    repo = _load_candidate(candidate_repo)
+    eval_root = Path(eval_root).resolve()
+    rows = _load_jsonl(Path(evidence_jsonl).resolve())
+    if run_id is not None:
+        rows = [row for row in rows if row.get("run_id") == run_id]
+    if case_ids:
+        wanted = set(case_ids)
+        rows = [row for row in rows if row.get("case_id") in wanted]
+    if not rows:
+        raise SystemExit("no evidence rows selected for fixture")
+
+    from pdl_taskmaster.runtime.session_engine import SessionEngine
+
+    builder = RecordedFixtureBuilder()
+    for row in rows:
+        workspaces = _resolve_workspaces(eval_root, row)
+        recorded_calls = _ordered_recorded_calls(workspaces)
+        call_index = 0
+
+        def model_call(request: Any, _recorded=recorded_calls) -> str:
+            nonlocal call_index
+            if call_index >= len(_recorded):
+                raise SystemExit(
+                    f"fixture replay exhausted for case {row.get('case_id')} "
+                    f"at operation {request.operation}"
+                )
+            recorded = _recorded[call_index]
+            call_index += 1
+            if recorded["operation"] != request.operation:
+                raise SystemExit(
+                    f"fixture replay order mismatch: expected {recorded['operation']} "
+                    f"got {request.operation} for case {row.get('case_id')}"
+                )
+            builder.add(
+                request.operation,
+                hashlib.sha256(request.prompt.encode("utf-8")).hexdigest(),
+                recorded["response"],
+                metadata={"source": f"{row['case_id']}:{recorded['invocation_id']}"},
+                prompt_text=request.prompt,
+            )
+            return recorded["response"]
+
+        with tempfile.TemporaryDirectory(prefix="pdl-r2s-fixture-") as tmp:
+            engine = SessionEngine(
+                repo,
+                model_call,
+                higher_priority_constraints="Obey applicable provider/platform safety, privacy, permission, and tool constraints.",
+                available_execution_tools=[],
+                workspace_root=tmp,
+            )
+            for turn in row.get("turns", []):
+                engine.handle_user_message(turn.get("user", ""))
+    return builder.build()
+
+def build_recorded_fixture_from_vendored(
+    candidate_repo: str | Path,
+    fixture_file: str | Path,
+    *,
+    case_ids: list[str] | None = None,
+) -> RecordedWorker:
+    """Build a recorded worker from the vendored standalone fixture.
+
+    Extraction adaptation: the qualified REPL recorded path previously read
+    recorded workspaces from an external evaluation repository. The standalone
+    harness vendors the exact recorded calls (operation, prompt hash, response)
+    so the published harness does not depend on that repository.
+    """
+    _load_candidate(candidate_repo)
+    value = json.loads(Path(fixture_file).read_text(encoding="utf-8"))
+    entries = value.get("entries") or []
+    if case_ids:
+        wanted = set(case_ids)
+        entries = [e for e in entries if (e.get("source") or "").split(":")[0] in wanted]
+    if not entries:
+        raise SystemExit("no vendored fixture entries selected")
+    builder = RecordedFixtureBuilder()
+    for entry in entries:
+        builder.add(
+            entry["operation"],
+            entry["prompt_sha256"],
+            entry["response"],
+            metadata={"source": entry.get("source", "vendored")},
+            prompt_text=entry.get("prompt_text"),
+        )
+    return builder.build()

pdl_taskmaster/providers/live_stub.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import json
+
+from pdl_taskmaster.providers.base import WorkerResult
+
+
+class LiveStubWorker:
+    """Deterministic non-recorded worker for live-capability tests.
+
+    Label: DEVELOPMENT / LIVE DEMONSTRATION WORKER
+    NOT A QUALIFIED R2S MEASUREMENT CONDITION.
+
+    This adapter does not reference case IDs, evidence files, or recorded
+    turn sequences. It is intentionally generic so observation can be tested
+    against arbitrary new sessions.
+    """
+
+    def call(self, request) -> WorkerResult:
+        operation = request.operation
+        if operation == "INTERPRET_ACTIVATION":
+            payload = {"route": "APPLY_PROTOCOL"}
+        elif operation == "BOOTSTRAP_ANALYSIS":
+            payload = {
+                "kind": "ANALYSIS",
+                "task_summary": "The user requests a benign explanation of a topic in plain terms.",
+                "approach_notes": "",
+                "risk_notes": "",
+                "task_entities": [],
+            }
+        elif operation == "DRAFT_PROMPT":
+            payload = {
+                "kind": "PROMPT",
+                "prompt_body": "EXPLAIN the user's requested topic in plain terms.",
+                "approach_handoff": "NONE",
+            }
+        elif operation in {"INTERPRET_PROMPT_REVIEW", "INTERPRET_PLAN_REVIEW"}:
+            payload = {
+                "kind": "REVIEW_FACTS",
+                "task_change_dimensions": [],
+                "approach_change_dimensions": [],
+                "progression_requested": True,
+            }
+        elif operation == "DRAFT_PLAN":
+            payload = {
+                "neutral_plan_body": "1. Define the topic.\n2. Explain the key mechanisms.\n3. Summarize practical implications."
+            }
+        elif operation == "DRAFT_EXECUTE":
+            payload = {
+                "kind": "RESULT",
+                "brief_body": "1. Confirm the file contract.\n2. Use the declared wire format.\n3. Execute and verify.",
+                "execution_entities": [],
+            }
+        elif operation == "EMIT_RESULT_IR":
+            payload = {
+                "result_ir": {
+                    "files": [],
+                    "reconciliation": [],
+                    "open_defects": [],
+                }
+            }
+        elif operation == "EXECUTE":
+            payload = {"kind": "RESULT", "body": "Completed."}
+        else:
+            raise RuntimeError(f"unhandled live stub operation: {operation}")
+        return WorkerResult(json.dumps(payload, ensure_ascii=False), {"worker": "live-stub", "mode": "live-demonstration"})

pdl_taskmaster/providers/recorded.py

@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+import hashlib
+
+from pdl_taskmaster.providers.base import WorkerResult
+
+
+class ReplayMissError(RuntimeError):
+    pass
+
+
+class AmbiguousFixtureError(RuntimeError):
+    pass
+
+
+@dataclass
+class RecordedEntry:
+    response: str
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class RecordedFixtureBuilder:
+    def __init__(self) -> None:
+        self.entries: dict[tuple[str, str], list[RecordedEntry]] = {}
+        self.order: list[tuple[str, str]] = []
+        self.prompt_examples: dict[tuple[str, str], str] = {}
+
+    def add(
+        self,
+        operation: str,
+        prompt_sha256: str,
+        response: str,
+        metadata: dict[str, Any] | None = None,
+        *,
+        source: str = "fixture",
+        prompt_text: str | None = None,
+    ) -> None:
+        key = (operation, prompt_sha256)
+        self.entries.setdefault(key, []).append(
+            RecordedEntry(response, {"source": source, **(metadata or {})})
+        )
+        if prompt_text is not None:
+            self.prompt_examples.setdefault(key, prompt_text)
+        if key not in self.order:
+            self.order.append(key)
+
+    def build(self, selected_fixture_id: str | None = None) -> "RecordedWorker":
+        mapping: dict[tuple[str, str], RecordedEntry] = {}
+        for key, entries in self.entries.items():
+            unique = {entry.response: entry for entry in entries}
+            if len(unique) == 1:
+                mapping[key] = next(iter(unique.values()))
+                continue
+            if selected_fixture_id is None:
+                raise AmbiguousFixtureError(
+                    f"ambiguous replay mapping for {key[0]} prompt={key[1][:12]}; "
+                    "provide an explicit fixture/run ID"
+                )
+            selected = [
+                entry
+                for entry in entries
+                if entry.metadata.get("source") == selected_fixture_id
+            ]
+            if len(selected) != 1:
+                raise AmbiguousFixtureError(
+                    f"fixture ID {selected_fixture_id!r} does not resolve exactly one response "
+                    f"for {key[0]} prompt={key[1][:12]}"
+                )
+            mapping[key] = selected[0]
+        return RecordedWorker(mapping, list(self.order), dict(self.prompt_examples), dict(self.entries))
+
+
+class RecordedWorker:
+    """Deterministic replay worker.
+
+    This is a qualification/test implementation of WorkerAdapter, not the
+    general-purpose worker contract.
+    """
+
+    def __init__(
+        self,
+        mapping: dict[tuple[str, str], RecordedEntry],
+        order: list[tuple[str, str]] | None = None,
+        prompt_examples: dict[tuple[str, str], str] | None = None,
+        entries: dict[tuple[str, str], list[RecordedEntry]] | None = None,
+    ):
+        self.mapping = mapping
+        self.order = list(order or [])
+        self.prompt_examples = prompt_examples or {}
+        self.entries = entries or {}
+
+    def call(self, request: Any) -> WorkerResult:
+        key = (request.operation, hashlib.sha256(request.prompt.encode("utf-8")).hexdigest())
+        entry = self.mapping.get(key)
+        if entry is None:
+            raise ReplayMissError(
+                f"no recorded response for operation={request.operation} prompt_sha256={key[1][:12]}"
+            )
+        return WorkerResult(entry.response, dict(entry.metadata))
+
+    def to_manifest(self) -> list[dict[str, str]]:
+        rows: list[dict[str, str]] = []
+        for key, entry in self.mapping.items():
+            operation, prompt_sha256 = key
+            source = str(entry.metadata.get("source", "unknown"))
+            fixture_id = hashlib.sha256(
+                f"{source}|{operation}|{prompt_sha256}".encode("utf-8")
+            ).hexdigest()[:16]
+            rows.append(
+                {
+                    "source_evidence_run": source,
+                    "operation": operation,
+                    "prompt_sha256": prompt_sha256,
+                    "raw_response_sha256": hashlib.sha256(entry.response.encode("utf-8")).hexdigest(),
+                    "fixture_id": fixture_id,
+                }
+            )
+        return rows

pdl_taskmaster/py.typed

@@ -0,0 +1 @@
+# Marker file for PEP 561.