pdl-taskmaster 2.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pdl_taskmaster/__init__.py +6 -0
- pdl_taskmaster/contracts/AUTHORITY_MAP.json +19 -0
- pdl_taskmaster/contracts/CALIBRATION.json +7 -0
- pdl_taskmaster/contracts/CONTRACT_MANIFEST.json +231 -0
- pdl_taskmaster/contracts/EXECUTION_CONTRACT.json +615 -0
- pdl_taskmaster/contracts/IMPLEMENTATION_BOUNDARY.json +32 -0
- pdl_taskmaster/contracts/VERIFICATION_CONTRACT.json +379 -0
- pdl_taskmaster/contracts/standards/ARTIFACT_STANDARD.md +7 -0
- pdl_taskmaster/contracts/standards/AUTHORITY_STANDARD.md +17 -0
- pdl_taskmaster/contracts/standards/CONFORMANCE_STANDARD.md +15 -0
- pdl_taskmaster/contracts/standards/CONTEXT_STANDARD.md +15 -0
- pdl_taskmaster/contracts/standards/EXECUTION_STANDARD.md +14 -0
- pdl_taskmaster/contracts/standards/HANDOFF_STANDARD.md +21 -0
- pdl_taskmaster/contracts/standards/INSTALLATION_STANDARD.md +17 -0
- pdl_taskmaster/contracts/standards/PDL_STANDARD.md +19 -0
- pdl_taskmaster/contracts/standards/PROMPT_STANDARD.md +13 -0
- pdl_taskmaster/contracts/standards/PROTOCOL_STANDARD.md +17 -0
- pdl_taskmaster/contracts/standards/RESPONSE_PLAN_STANDARD.md +24 -0
- pdl_taskmaster/contracts/standards/RESULT_STANDARD.md +35 -0
- pdl_taskmaster/contracts/standards/REVIEW_STANDARD.md +31 -0
- pdl_taskmaster/contracts/standards/SEMANTIC_INPUT_STANDARD.md +15 -0
- pdl_taskmaster/contracts/standards/TASK_SEMANTICS_STANDARD.md +9 -0
- pdl_taskmaster/controller/__init__.py +0 -0
- pdl_taskmaster/controller/mechanical_controller.py +542 -0
- pdl_taskmaster/controller/schemas/activation_decision.schema.json +34 -0
- pdl_taskmaster/controller/schemas/artifact_body.schema.json +14 -0
- pdl_taskmaster/controller/schemas/artifact_review_facts.schema.json +91 -0
- pdl_taskmaster/controller/schemas/bootstrap_analysis.schema.json +60 -0
- pdl_taskmaster/controller/schemas/execution_draft.schema.json +47 -0
- pdl_taskmaster/controller/schemas/execution_input_decision.schema.json +43 -0
- pdl_taskmaster/controller/schemas/execution_outcome.schema.json +201 -0
- pdl_taskmaster/controller/schemas/plan_artifact.schema.json +14 -0
- pdl_taskmaster/controller/schemas/prompt_artifact.schema.json +63 -0
- pdl_taskmaster/controller/schemas/protocol_discussion.schema.json +8 -0
- pdl_taskmaster/controller/schemas/result_ir_repair.schema.json +140 -0
- pdl_taskmaster/eval/__init__.py +1 -0
- pdl_taskmaster/eval/build_adversarial_battery.py +351 -0
- pdl_taskmaster/eval/compare_eval_runs.py +251 -0
- pdl_taskmaster/eval/control_api_call.py +428 -0
- pdl_taskmaster/eval/export_rlcd_dataset.py +799 -0
- pdl_taskmaster/eval/fidelity_scan.py +268 -0
- pdl_taskmaster/eval/gen_fidelity_battery.py +457 -0
- pdl_taskmaster/eval/leak_scan.py +260 -0
- pdl_taskmaster/eval/rerecord_fixtures.py +139 -0
- pdl_taskmaster/eval/run_canary_shape_probe.py +229 -0
- pdl_taskmaster/eval/run_dual_gate_probe.py +404 -0
- pdl_taskmaster/eval/run_qualified_batch.py +841 -0
- pdl_taskmaster/eval/summarize_run.py +194 -0
- pdl_taskmaster/host/__init__.py +1 -0
- pdl_taskmaster/host/app.py +164 -0
- pdl_taskmaster/host/cli.py +100 -0
- pdl_taskmaster/host/repl.py +866 -0
- pdl_taskmaster/observation/__init__.py +1 -0
- pdl_taskmaster/observation/cli.py +47 -0
- pdl_taskmaster/observation/observed_session.py +219 -0
- pdl_taskmaster/observation/parsers.py +66 -0
- pdl_taskmaster/observation/records.py +58 -0
- pdl_taskmaster/observation/session.py +3 -0
- pdl_taskmaster/observation/sink.py +3 -0
- pdl_taskmaster/observation/sinks.py +20 -0
- pdl_taskmaster/providers/README.md +66 -0
- pdl_taskmaster/providers/__init__.py +1 -0
- pdl_taskmaster/providers/api_worker.py +411 -0
- pdl_taskmaster/providers/base.py +26 -0
- pdl_taskmaster/providers/codex_worker.py +328 -0
- pdl_taskmaster/providers/fixtures.py +157 -0
- pdl_taskmaster/providers/live_stub.py +66 -0
- pdl_taskmaster/providers/recorded.py +120 -0
- pdl_taskmaster/py.typed +1 -0
- pdl_taskmaster/runtime/__init__.py +0 -0
- pdl_taskmaster/runtime/context_compiler.py +151 -0
- pdl_taskmaster/runtime/model_classification.py +290 -0
- pdl_taskmaster/runtime/normative_store.py +157 -0
- pdl_taskmaster/runtime/operation_bridge.py +336 -0
- pdl_taskmaster/runtime/presentation.py +18 -0
- pdl_taskmaster/runtime/quarantine.py +91 -0
- pdl_taskmaster/runtime/result_ir.py +401 -0
- pdl_taskmaster/runtime/semantic_observer.py +40 -0
- pdl_taskmaster/runtime/session_engine.py +1168 -0
- pdl_taskmaster/runtime/standard_registry.py +55 -0
- pdl_taskmaster/runtime/wire_payloads.py +462 -0
- pdl_taskmaster/runtime/worker-bootstrap.txt +4 -0
- pdl_taskmaster/runtime/workspace.py +602 -0
- pdl_taskmaster/tracking/__init__.py +1 -0
- pdl_taskmaster/tracking/log_live_session.py +98 -0
- pdl_taskmaster/tracking/mlflow_sink.py +58 -0
- pdl_taskmaster/verify/verify_repl_baseline.py +299 -0
- pdl_taskmaster-2.4.0.dist-info/METADATA +155 -0
- pdl_taskmaster-2.4.0.dist-info/RECORD +92 -0
- pdl_taskmaster-2.4.0.dist-info/WHEEL +4 -0
- pdl_taskmaster-2.4.0.dist-info/entry_points.txt +2 -0
- pdl_taskmaster-2.4.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import sys
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
ROOT = Path(__file__).resolve().parents[3]
|
|
11
|
+
if str(ROOT / "src") not in sys.path:
|
|
12
|
+
sys.path.insert(0, str(ROOT / "src"))
|
|
13
|
+
if str(ROOT) not in sys.path:
|
|
14
|
+
sys.path.insert(0, str(ROOT))
|
|
15
|
+
|
|
16
|
+
import mlflow
|
|
17
|
+
from pdl_taskmaster.tracking.mlflow_sink import log_experiment_run
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _default_tracking_db() -> Path:
|
|
21
|
+
"""Resolve the MLflow SQLite tracking store.
|
|
22
|
+
|
|
23
|
+
MLflow telemetry is post-hoc and non-authoritative; the store lives in the
|
|
24
|
+
external archive by default (repo-restructure-plan §3.1):
|
|
25
|
+
1. PDLT_MLFLOW_DB environment variable (explicit db path override).
|
|
26
|
+
2. Sibling archive: <parent>/PDL-Standard-Archive/mlflow/mlflow.db.
|
|
27
|
+
3. Legacy repo-root mlflow.db fallback (isolated test trees, archive absent).
|
|
28
|
+
"""
|
|
29
|
+
env = os.environ.get("PDLT_MLFLOW_DB", "").strip()
|
|
30
|
+
if env:
|
|
31
|
+
return Path(env)
|
|
32
|
+
archive = ROOT.parent / "PDL-Standard-Archive" / "mlflow" / "mlflow.db"
|
|
33
|
+
if (ROOT.parent / "PDL-Standard-Archive").is_dir():
|
|
34
|
+
return archive
|
|
35
|
+
return ROOT / "mlflow.db"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def main() -> int:
|
|
39
|
+
parser = argparse.ArgumentParser(description="Log one live PDLt session to MLflow")
|
|
40
|
+
parser.add_argument("--session-dir", type=Path, required=True)
|
|
41
|
+
parser.add_argument("--run-name", default=None)
|
|
42
|
+
parser.add_argument("--worker-profile", default="api", help="worker identity to record (api|codex|recorded)")
|
|
43
|
+
args = parser.parse_args()
|
|
44
|
+
|
|
45
|
+
session_dir = args.session_dir.resolve()
|
|
46
|
+
session_json = session_dir / "session.json"
|
|
47
|
+
if not session_json.is_file():
|
|
48
|
+
raise SystemExit(f"session.json missing: {session_json}")
|
|
49
|
+
session_data = json.loads(session_json.read_text(encoding="utf-8"))
|
|
50
|
+
session_id = session_dir.name
|
|
51
|
+
|
|
52
|
+
observation_files = sorted((session_dir / "observations").glob("*.jsonl")) if (session_dir / "observations").is_dir() else []
|
|
53
|
+
observation_records = 0
|
|
54
|
+
final_stage = None
|
|
55
|
+
total_tokens = 0
|
|
56
|
+
for obs in observation_files:
|
|
57
|
+
rows = [json.loads(line) for line in obs.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
58
|
+
observation_records += len(rows)
|
|
59
|
+
if rows:
|
|
60
|
+
final_stage = ((rows[-1].get("controller_after") or {}).get("controller_state") or {}).get("stage")
|
|
61
|
+
for row in rows:
|
|
62
|
+
for call in row.get("calls", []):
|
|
63
|
+
usage = call.get("usage") or {}
|
|
64
|
+
total_tokens += int(usage.get("total_tokens") or 0)
|
|
65
|
+
|
|
66
|
+
transcript = session_dir / "transcript.log"
|
|
67
|
+
artifacts = [str(session_json)]
|
|
68
|
+
if transcript.is_file():
|
|
69
|
+
artifacts.append(str(transcript))
|
|
70
|
+
artifacts.extend(str(path) for path in observation_files)
|
|
71
|
+
worker_progress = session_dir / "worker-progress.log"
|
|
72
|
+
if worker_progress.is_file():
|
|
73
|
+
artifacts.append(str(worker_progress))
|
|
74
|
+
|
|
75
|
+
mlflow.set_tracking_uri(f"sqlite:///{_default_tracking_db().as_posix()}")
|
|
76
|
+
mlflow.set_experiment("PDL-R2S")
|
|
77
|
+
run_id = log_experiment_run(
|
|
78
|
+
args.run_name or f"live-session-{session_id}",
|
|
79
|
+
params={
|
|
80
|
+
"run_type": "live_session",
|
|
81
|
+
"session_id": session_id,
|
|
82
|
+
"candidate_repository": "PDL-Standard-R2S",
|
|
83
|
+
"candidate_commit": "6df3bf5733cd3fcd16b1560ff7a80810c96bbe6c",
|
|
84
|
+
"worker_profile": args.worker_profile,
|
|
85
|
+
"observation_jsonl_path": ",".join(str(path) for path in observation_files),
|
|
86
|
+
"transcript_path": str(transcript) if transcript.is_file() else "none",
|
|
87
|
+
"workspace_path": str(session_data.get("workspace_path", "none")),
|
|
88
|
+
"final_stage": final_stage or "unknown",
|
|
89
|
+
},
|
|
90
|
+
metrics={"observation_records": float(observation_records), "total_tokens": float(total_tokens)},
|
|
91
|
+
artifacts=artifacts,
|
|
92
|
+
)
|
|
93
|
+
print(f"LIVE_SESSION_MLFLOW_RUN {run_id} session={session_id} records={observation_records}")
|
|
94
|
+
return 0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == "__main__":
|
|
98
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
import mlflow
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class MlflowRunLogger:
|
|
10
|
+
"""Tracking-only run logger.
|
|
11
|
+
|
|
12
|
+
This logger lives outside the observed semantic path. It indexes evidence
|
|
13
|
+
after the fact.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(self, run_name: str, params: dict[str, Any] | None = None):
|
|
17
|
+
self._mlflow = mlflow
|
|
18
|
+
self._run = mlflow.start_run(run_name=run_name)
|
|
19
|
+
if params:
|
|
20
|
+
mlflow.log_params(params)
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def run_id(self) -> str:
|
|
24
|
+
return self._run.info.run_id
|
|
25
|
+
|
|
26
|
+
def log_params(self, params: dict[str, Any]) -> None:
|
|
27
|
+
self._mlflow.log_params(params)
|
|
28
|
+
|
|
29
|
+
def log_metrics(self, metrics: dict[str, float]) -> None:
|
|
30
|
+
self._mlflow.log_metrics(metrics)
|
|
31
|
+
|
|
32
|
+
def log_artifact(self, path: str | Path) -> None:
|
|
33
|
+
self._mlflow.log_artifact(str(path))
|
|
34
|
+
|
|
35
|
+
def finish(self) -> None:
|
|
36
|
+
self._mlflow.end_run()
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def log_experiment_run(
|
|
40
|
+
run_name: str,
|
|
41
|
+
*,
|
|
42
|
+
params: dict[str, Any],
|
|
43
|
+
metrics: dict[str, float],
|
|
44
|
+
artifacts: list[str | Path],
|
|
45
|
+
) -> str:
|
|
46
|
+
"""Log one tracking-only run.
|
|
47
|
+
|
|
48
|
+
This function is never called from the observed semantic path.
|
|
49
|
+
"""
|
|
50
|
+
with mlflow.start_run(run_name=run_name) as run:
|
|
51
|
+
mlflow.log_params(params)
|
|
52
|
+
mlflow.log_metrics(metrics)
|
|
53
|
+
for artifact in artifacts:
|
|
54
|
+
path = Path(artifact)
|
|
55
|
+
if not path.is_file():
|
|
56
|
+
raise FileNotFoundError(f"required evidence pointer missing: {path}")
|
|
57
|
+
mlflow.log_artifact(str(path))
|
|
58
|
+
return run.info.run_id
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
"""One deterministic verification entry point for the standalone REPL harness.
|
|
4
|
+
|
|
5
|
+
Checks: required runtime/instructional files, import integrity, fixture
|
|
6
|
+
integrity, external-source-repo leakage, fresh-workspace lifecycle, result
|
|
7
|
+
delivery, and resume. All workspaces are created in temporary directories.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import sys
|
|
14
|
+
import tempfile
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
ROOT = Path(__file__).resolve().parents[3]
|
|
19
|
+
if str(ROOT) not in sys.path:
|
|
20
|
+
sys.path.insert(0, str(ROOT))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
_SRC = "PDL-Standard-R2S"
|
|
24
|
+
SOURCE_REPO_MARKERS = (
|
|
25
|
+
"Desktop" + "\\Frameworks\\" + _SRC,
|
|
26
|
+
"Desktop" + "\\Frameworks\\" + _SRC + "-Lab",
|
|
27
|
+
"Desktop" + "/Frameworks/" + _SRC,
|
|
28
|
+
"Desktop" + "/Frameworks/" + _SRC + "-Lab",
|
|
29
|
+
)
|
|
30
|
+
FORBIDDEN_TERMS = (
|
|
31
|
+
"phase6ev",
|
|
32
|
+
"cohort",
|
|
33
|
+
"recruitment",
|
|
34
|
+
"guardian",
|
|
35
|
+
"recurrence",
|
|
36
|
+
"OPEN_AUTHORIZATION",
|
|
37
|
+
"current-phase-docs",
|
|
38
|
+
"architecture-decisions",
|
|
39
|
+
)
|
|
40
|
+
SECRET_PATTERNS = (
|
|
41
|
+
re.compile(r"\bsk-[A-Za-z0-9]{16,}\b"),
|
|
42
|
+
re.compile(r"\b(?:api[_-]?key|openai_api_key|openrouter_api_key)\s*[:=]\s*['\"][^'\"]{8,}['\"]", re.I),
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
REQUIRED_FILES = (
|
|
46
|
+
"src/pdl_taskmaster/controller/mechanical_controller.py",
|
|
47
|
+
"src/pdl_taskmaster/runtime/session_engine.py",
|
|
48
|
+
"src/pdl_taskmaster/runtime/workspace.py",
|
|
49
|
+
"src/pdl_taskmaster/runtime/normative_store.py",
|
|
50
|
+
"src/pdl_taskmaster/runtime/operation_bridge.py",
|
|
51
|
+
"src/pdl_taskmaster/runtime/context_compiler.py",
|
|
52
|
+
"src/pdl_taskmaster/runtime/standard_registry.py",
|
|
53
|
+
"src/pdl_taskmaster/runtime/presentation.py",
|
|
54
|
+
"src/pdl_taskmaster/runtime/semantic_observer.py",
|
|
55
|
+
"contracts/CONTRACT_MANIFEST.json",
|
|
56
|
+
"contracts/EXECUTION_CONTRACT.json",
|
|
57
|
+
"confirm-with-pseudocode/SKILL.md",
|
|
58
|
+
"src/pdl_taskmaster/host/repl.py",
|
|
59
|
+
"src/pdl_taskmaster/host/app.py",
|
|
60
|
+
"src/pdl_taskmaster/observation/session.py",
|
|
61
|
+
"src/pdl_taskmaster/observation/records.py",
|
|
62
|
+
"src/pdl_taskmaster/observation/sinks.py",
|
|
63
|
+
"src/pdl_taskmaster/providers/base.py",
|
|
64
|
+
"src/pdl_taskmaster/providers/recorded.py",
|
|
65
|
+
"src/pdl_taskmaster/providers/fixtures.py",
|
|
66
|
+
"src/pdl_taskmaster/providers/codex_worker.py",
|
|
67
|
+
"src/pdl_taskmaster/providers/live_stub.py",
|
|
68
|
+
"src/pdl_taskmaster/tracking/mlflow_sink.py",
|
|
69
|
+
"src/pdl_taskmaster/tracking/log_live_session.py",
|
|
70
|
+
"src/pdl_taskmaster/eval/run_qualified_batch.py",
|
|
71
|
+
"src/pdl_taskmaster/eval/leak_scan.py",
|
|
72
|
+
"src/pdl_taskmaster/eval/fidelity_scan.py",
|
|
73
|
+
"README.md",
|
|
74
|
+
"SOURCE_PROVENANCE.json",
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _fixture_dir() -> Path:
|
|
79
|
+
"""Resolve the recorded fixture directory.
|
|
80
|
+
|
|
81
|
+
Precedence (mirrors the Brain/Hands pattern of S1's NormativeStore):
|
|
82
|
+
1. PDLT_FIXTURES_PATH environment variable (points at the r4-recorded-worker dir).
|
|
83
|
+
2. tests/fixtures vendored location (self-contained offline suite).
|
|
84
|
+
3. Repository-relative vendored location (compat with pre-restructure checkouts).
|
|
85
|
+
4. Sibling PDL-Standard-Archive external fixtures.
|
|
86
|
+
"""
|
|
87
|
+
env = os.environ.get("PDLT_FIXTURES_PATH", "").strip()
|
|
88
|
+
if env:
|
|
89
|
+
return Path(env)
|
|
90
|
+
vendored = ROOT / "tests" / "fixtures"
|
|
91
|
+
if vendored.is_dir() and (vendored / "recorded-cases.json").is_file():
|
|
92
|
+
return vendored
|
|
93
|
+
local = ROOT / "fixtures" / "r4-recorded-worker"
|
|
94
|
+
if local.is_dir():
|
|
95
|
+
return local
|
|
96
|
+
archive = ROOT.parent / "PDL-Standard-Archive" / "fixtures-r4-recorded-worker"
|
|
97
|
+
if archive.is_dir():
|
|
98
|
+
return archive
|
|
99
|
+
return vendored
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _failures() -> list[str]:
|
|
104
|
+
problems: list[str] = []
|
|
105
|
+
|
|
106
|
+
# 1. Required runtime/instructional files.
|
|
107
|
+
for relative in REQUIRED_FILES:
|
|
108
|
+
if not (ROOT / relative).is_file():
|
|
109
|
+
problems.append(f"missing_required_file:{relative}")
|
|
110
|
+
|
|
111
|
+
# 1b. Zero-template invariant (S2 / ADR-0008): the workspace template must NOT exist.
|
|
112
|
+
if (ROOT / "workspace-template").exists():
|
|
113
|
+
problems.append("zero_template_violation:workspace-template directory must be absent")
|
|
114
|
+
|
|
115
|
+
# 1c. Clause-coverage invariant (D28): every clause defined in the normative
|
|
116
|
+
# standards must be either bound in a contract or explicitly registered as
|
|
117
|
+
# unbound. A clause added to a standards file without a contract binding
|
|
118
|
+
# fails the verifier exactly like a code regression.
|
|
119
|
+
defined: set[str] = set()
|
|
120
|
+
for std_file in (ROOT / "contracts" / "standards").glob("*.md"):
|
|
121
|
+
for m in re.finditer(r"\*\*([A-Z]+-\d+)(?:\s*[—-])", std_file.read_text(encoding="utf-8")):
|
|
122
|
+
defined.add(m.group(1))
|
|
123
|
+
bound: set[str] = set()
|
|
124
|
+
for contract_name in ("EXECUTION_CONTRACT.json", "VERIFICATION_CONTRACT.json"):
|
|
125
|
+
contract_data = json.loads((ROOT / "contracts" / contract_name).read_text(encoding="utf-8"))
|
|
126
|
+
|
|
127
|
+
def _walk_requirements(node: object) -> None:
|
|
128
|
+
if isinstance(node, dict):
|
|
129
|
+
for key, value in node.items():
|
|
130
|
+
if key == "requirements" and isinstance(value, list):
|
|
131
|
+
bound.update(x for x in value if isinstance(x, str) and re.match(r"^[A-Z]+-\d+$", x))
|
|
132
|
+
_walk_requirements(value)
|
|
133
|
+
elif isinstance(node, list):
|
|
134
|
+
for item in node:
|
|
135
|
+
_walk_requirements(item)
|
|
136
|
+
|
|
137
|
+
_walk_requirements(contract_data)
|
|
138
|
+
for clause in sorted(defined - bound):
|
|
139
|
+
problems.append(f"clause_coverage_uncovered:{clause} defined in standards but not bound in any contract")
|
|
140
|
+
|
|
141
|
+
# 1d. Subprocess script targets referenced by the REPL must exist.
|
|
142
|
+
repl_path = ROOT / "src" / "pdl_taskmaster" / "host" / "repl.py"
|
|
143
|
+
if repl_path.is_file():
|
|
144
|
+
repl_text = repl_path.read_text(encoding="utf-8")
|
|
145
|
+
for match in re.findall(r'-m",\s*"([^"]+)"', repl_text):
|
|
146
|
+
try:
|
|
147
|
+
__import__(match)
|
|
148
|
+
except ImportError:
|
|
149
|
+
problems.append(f"missing_repl_subprocess_module:{match}")
|
|
150
|
+
|
|
151
|
+
# 2. Broken imports.
|
|
152
|
+
try:
|
|
153
|
+
from pdl_taskmaster.host.app import PDLtHost # noqa: F401
|
|
154
|
+
from pdl_taskmaster.host.repl import main as repl_main # noqa: F401
|
|
155
|
+
from pdl_taskmaster.runtime.session_engine import SessionEngine # noqa: F401
|
|
156
|
+
from pdl_taskmaster.providers.fixtures import build_recorded_fixture_from_vendored # noqa: F401
|
|
157
|
+
except Exception as exc:
|
|
158
|
+
problems.append(f"import_failure:{exc}")
|
|
159
|
+
|
|
160
|
+
# 3. External-source-repo leakage.
|
|
161
|
+
for path in ROOT.rglob("*"):
|
|
162
|
+
if not path.is_file() or ".git" in path.parts or "__pycache__" in path.parts:
|
|
163
|
+
continue
|
|
164
|
+
if path.name == "SOURCE_PROVENANCE.json":
|
|
165
|
+
continue # provenance must record source repository paths by design
|
|
166
|
+
try:
|
|
167
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
168
|
+
except OSError:
|
|
169
|
+
continue
|
|
170
|
+
for marker in SOURCE_REPO_MARKERS:
|
|
171
|
+
if marker in text:
|
|
172
|
+
problems.append(f"source_repo_path_leak:{path.relative_to(ROOT).as_posix()}")
|
|
173
|
+
break
|
|
174
|
+
for env_name in ("PYTHONPATH", "PYTHONHOME"):
|
|
175
|
+
value = os.environ.get(env_name, "")
|
|
176
|
+
for root in ("PDL-Standard-R2S", "PDL-Standard-R2S-Lab"):
|
|
177
|
+
if root in value:
|
|
178
|
+
problems.append(f"environment_leak:{env_name}={value}")
|
|
179
|
+
|
|
180
|
+
# 4. Fixture integrity (externalized: PDLT_FIXTURES_PATH -> repo-relative fallback).
|
|
181
|
+
fixture_dir = _fixture_dir()
|
|
182
|
+
if not fixture_dir.is_dir():
|
|
183
|
+
problems.append(
|
|
184
|
+
"fixture_dir_missing:" + str(fixture_dir) + " (set PDLT_FIXTURES_PATH to the r4-recorded-worker archive directory)"
|
|
185
|
+
)
|
|
186
|
+
fixture = {}
|
|
187
|
+
manifest = {}
|
|
188
|
+
else:
|
|
189
|
+
fixture_path = fixture_dir / "recorded-cases.json"
|
|
190
|
+
try:
|
|
191
|
+
fixture = json.loads(fixture_path.read_text(encoding="utf-8"))
|
|
192
|
+
except Exception as exc:
|
|
193
|
+
problems.append(f"fixture_load_failure:{exc}")
|
|
194
|
+
fixture = {}
|
|
195
|
+
manifest = json.loads((fixture_dir / "FIXTURE_MANIFEST.json").read_text(encoding="utf-8"))
|
|
196
|
+
manifest_prompts = {entry["source_evidence_run"]: entry["prompt_sha256"] for entry in manifest.get("entries", [])}
|
|
197
|
+
for entry in fixture.get("entries", []):
|
|
198
|
+
expected = manifest_prompts.get(entry.get("source"))
|
|
199
|
+
if expected is not None and entry.get("prompt_sha256") != expected:
|
|
200
|
+
problems.append(f"fixture_hash_mismatch:{entry.get('source')}")
|
|
201
|
+
|
|
202
|
+
# 5. Forbidden terms / secrets (classification; absolute source paths above are fatal).
|
|
203
|
+
for path in ROOT.rglob("*"):
|
|
204
|
+
if not path.is_file() or ".git" in path.parts or "__pycache__" in path.parts:
|
|
205
|
+
continue
|
|
206
|
+
if path.name == "verify_repl_baseline.py":
|
|
207
|
+
continue # scanner's own source contains forbidden terms by definition
|
|
208
|
+
try:
|
|
209
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
210
|
+
except OSError:
|
|
211
|
+
continue
|
|
212
|
+
for pattern in SECRET_PATTERNS:
|
|
213
|
+
if pattern.search(text):
|
|
214
|
+
problems.append(f"possible_secret:{path.relative_to(ROOT).as_posix()}")
|
|
215
|
+
for term in FORBIDDEN_TERMS:
|
|
216
|
+
if term in text:
|
|
217
|
+
print(f"WARN forbidden-term {term!r} in {path.relative_to(ROOT).as_posix()}")
|
|
218
|
+
|
|
219
|
+
return problems
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _fresh_workspace_lifecycle() -> dict[str, object]:
|
|
223
|
+
from pdl_taskmaster.host.app import PDLtHost
|
|
224
|
+
from pdl_taskmaster.providers.fixtures import build_recorded_fixture_from_vendored
|
|
225
|
+
|
|
226
|
+
fixture_file = _fixture_dir() / "recorded-cases.json"
|
|
227
|
+
fixture = json.loads(fixture_file.read_text(encoding="utf-8"))
|
|
228
|
+
turns = fixture["case_turns"]["G06"]
|
|
229
|
+
|
|
230
|
+
with tempfile.TemporaryDirectory(prefix="repl-baseline-") as tmp:
|
|
231
|
+
tmp_root = Path(tmp)
|
|
232
|
+
worker = build_recorded_fixture_from_vendored(ROOT, fixture_file, case_ids=["G06"])
|
|
233
|
+
host = PDLtHost(
|
|
234
|
+
ROOT,
|
|
235
|
+
worker=worker,
|
|
236
|
+
workspace_root=tmp_root / "workspaces",
|
|
237
|
+
run_id="baseline",
|
|
238
|
+
observation_dir=tmp_root / "observations",
|
|
239
|
+
).start()
|
|
240
|
+
stages: list[str] = []
|
|
241
|
+
try:
|
|
242
|
+
for turn in turns:
|
|
243
|
+
result = host.handle(turn)
|
|
244
|
+
if result.state_after is not None:
|
|
245
|
+
stage = (result.state_after.get("controller_state") or {}).get("stage")
|
|
246
|
+
if stage:
|
|
247
|
+
stages.append(str(stage))
|
|
248
|
+
status = host.status()
|
|
249
|
+
workspace_path = Path(status["workspace_path"])
|
|
250
|
+
final_stage = (status.get("controller_state") or {}).get("stage")
|
|
251
|
+
from pdl_taskmaster.runtime.workspace import WorkspaceRun as _WR
|
|
252
|
+
_ws = _WR.open(ROOT, workspace_path)
|
|
253
|
+
_stages = _ws.stages_root()
|
|
254
|
+
result_file = _stages / "50_execution" / "output" / "current.json"
|
|
255
|
+
stage_dirs = sorted(p.name for p in _stages.iterdir() if p.is_dir())
|
|
256
|
+
fresh_ok = "10_prompt" in stage_dirs and "30_plan" in stage_dirs and "50_execution" in stage_dirs
|
|
257
|
+
if final_stage != "CLOSED_SUCCESS":
|
|
258
|
+
raise AssertionError(f"final stage {final_stage!r} != CLOSED_SUCCESS")
|
|
259
|
+
if not result_file.is_file():
|
|
260
|
+
raise AssertionError("execution result file missing")
|
|
261
|
+
if not fresh_ok:
|
|
262
|
+
raise AssertionError("fresh workspace did not materialize stage directories")
|
|
263
|
+
|
|
264
|
+
# Resume the newly-created workspace.
|
|
265
|
+
worker2 = build_recorded_fixture_from_vendored(ROOT, fixture_file, case_ids=["G06"])
|
|
266
|
+
resumed = PDLtHost(
|
|
267
|
+
ROOT,
|
|
268
|
+
worker=worker2,
|
|
269
|
+
restore_path=workspace_path,
|
|
270
|
+
run_id="baseline-resume",
|
|
271
|
+
observation_dir=tmp_root / "observations-resume",
|
|
272
|
+
).start()
|
|
273
|
+
resume_stage = (resumed.status().get("controller_state") or {}).get("stage")
|
|
274
|
+
if resume_stage != "CLOSED_SUCCESS":
|
|
275
|
+
raise AssertionError(f"resume stage {resume_stage!r} != CLOSED_SUCCESS")
|
|
276
|
+
resumed.close()
|
|
277
|
+
finally:
|
|
278
|
+
host.close()
|
|
279
|
+
return {"stages": stages, "final_stage": final_stage, "resume_stage": resume_stage, "fresh_workspace": fresh_ok}
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def main() -> int:
|
|
283
|
+
problems = _failures()
|
|
284
|
+
lifecycle: dict[str, object] = {}
|
|
285
|
+
if not problems:
|
|
286
|
+
try:
|
|
287
|
+
lifecycle = _fresh_workspace_lifecycle()
|
|
288
|
+
except Exception as exc:
|
|
289
|
+
problems.append(f"lifecycle_failure:{exc}")
|
|
290
|
+
for problem in problems:
|
|
291
|
+
print(f"FAIL {problem}")
|
|
292
|
+
print("REPL BASELINE VERIFICATION", "PASS" if not problems else "FAIL")
|
|
293
|
+
if lifecycle:
|
|
294
|
+
print(json.dumps(lifecycle, indent=2, sort_keys=True))
|
|
295
|
+
return 0 if not problems else 1
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
if __name__ == "__main__":
|
|
299
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pdl-taskmaster
|
|
3
|
+
Version: 2.4.0
|
|
4
|
+
Summary: Deterministic protocol governance and bounded AI agent harness
|
|
5
|
+
Author: PDL-Standard Contributors
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
16
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Requires-Dist: pydantic<3.0.0,>=2.5.0
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: anyio>=4.0.0; extra == 'dev'
|
|
21
|
+
Requires-Dist: pytest-xdist>=3.5.0; extra == 'dev'
|
|
22
|
+
Requires-Dist: pytest>=8.0.0; extra == 'dev'
|
|
23
|
+
Provides-Extra: tracking
|
|
24
|
+
Requires-Dist: mlflow>=2.0.0; extra == 'tracking'
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# PDL Taskmaster
|
|
28
|
+
*(PDL-Standard-REPL-Harness)*
|
|
29
|
+
|
|
30
|
+
A REPL harness that makes a language model interpret your request — in short, readable pseudocode — and wait for you to confirm it before anything runs. The same mechanism that catches a misread request also blocks prompt injection: anything quoted or pasted into a task is treated as data at read time and never promoted to an instruction.
|
|
31
|
+
|
|
32
|
+
**Why this exists:** [`docs/architecture/framing.md`](docs/architecture/framing.md) — the full case, including a live boundary test where the identical model, on one unprotected API call, leaked 2 of 5 injected probes while the protocol held 0 of 5. For the full mechanism, design history, and evidence: [`docs/architecture/whitepaper.md`](docs/architecture/whitepaper.md).
|
|
33
|
+
|
|
34
|
+
## Repository layout
|
|
35
|
+
|
|
36
|
+
```text
|
|
37
|
+
contracts/ versioned standards, contracts, and schemas (normative source)
|
|
38
|
+
confirm-with-pseudocode/ bootstrap skill entrypoint used by the protocol
|
|
39
|
+
src/pdl_taskmaster/ clean Python package root
|
|
40
|
+
host/ REPL host, umbrella CLI (pdlt), and application wiring
|
|
41
|
+
runtime/ session engine, context compiler, 4-tier standards store, workspace
|
|
42
|
+
controller/ deterministic mechanical controller
|
|
43
|
+
providers/ model-worker boundary: api / recorded / codex
|
|
44
|
+
observation/ JSONL telemetry sinks
|
|
45
|
+
tracking/ optional MLflow session logger (post-hoc, non-authoritative)
|
|
46
|
+
eval/ adversarial battery, fidelity scoring, batch runner
|
|
47
|
+
verify/ deterministic baseline verification entry point
|
|
48
|
+
contracts/ bundled package data contracts (Tier 4 normative store)
|
|
49
|
+
tests/ pytest suite and vendored self-contained test fixtures
|
|
50
|
+
fixtures/ vendored recorded cases and adversarial/fidelity batteries
|
|
51
|
+
docs/ documentation
|
|
52
|
+
architecture/ framing.md, whitepaper.md, protocol specs
|
|
53
|
+
operations/ eval-metrics.md, efficiency-report.md
|
|
54
|
+
governance/ roadmap.md, experiment-log.md (full decision history)
|
|
55
|
+
adr/ trd/ background architecture-decision records (history, not required reading)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The package ships with self-contained test fixtures vendored under `tests/fixtures/`, enabling 100% offline testing. Historical run ledgers and large evaluation archives are kept in external archives outside this repository and resolve via `PDLT_FIXTURES_PATH` (fixtures) and `PDLT_RUNS_ROOT` (evaluation runs).
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
Requirements: Python 3.11+, Pydantic v2 (`pydantic>=2.5.0,<3.0.0`).
|
|
63
|
+
|
|
64
|
+
```powershell
|
|
65
|
+
# From PyPI (once published)
|
|
66
|
+
pip install pdl-taskmaster
|
|
67
|
+
# or using uv:
|
|
68
|
+
uv add pdl-taskmaster
|
|
69
|
+
|
|
70
|
+
# Development / editable installation:
|
|
71
|
+
cd PDL-Standard-REPL-Harness
|
|
72
|
+
pip install -e ".[test]"
|
|
73
|
+
# or with optional MLflow tracking:
|
|
74
|
+
pip install -e ".[test,tracking]"
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Run every command from this repository root or anywhere when installed as a package.
|
|
78
|
+
|
|
79
|
+
## Umbrella CLI (`pdlt`)
|
|
80
|
+
|
|
81
|
+
When installed, the `pdlt` command line tool provides the complete entry point:
|
|
82
|
+
|
|
83
|
+
```powershell
|
|
84
|
+
pdlt --help # View CLI command options and REPL flags
|
|
85
|
+
pdlt version # Print package version, protocol spec, and manifest hash
|
|
86
|
+
pdlt init --global # Initialize standards in ~/.pdlt/versions/v2/contracts/
|
|
87
|
+
pdlt init --local # Seed standards into ./contracts/ in current directory
|
|
88
|
+
pdlt verify # Run deterministic baseline verification gate
|
|
89
|
+
pdlt # Launch interactive REPL (default)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
## Deterministic baseline verification
|
|
93
|
+
|
|
94
|
+
```powershell
|
|
95
|
+
pdlt verify # Deterministic baseline verifier
|
|
96
|
+
pytest # Run complete test suite (offline, 87 passed)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
The verifier checks required runtime/instructional files, imports, the zero-template workspace invariant, fixture hashes, REPL subprocess-script presence, source-repository isolation, a fresh-workspace lifecycle test (prompt → plan → execute → result), and resuming that same workspace. All verifier workspaces are temporary.
|
|
100
|
+
|
|
101
|
+
## Interactive REPL (recorded / deterministic)
|
|
102
|
+
|
|
103
|
+
```powershell
|
|
104
|
+
pdlt --worker recorded --case-ids G06 --new-session
|
|
105
|
+
# or: python -m pdl_taskmaster.host.repl --worker recorded --case-ids G06 --new-session
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
Recorded mode is **exact, deterministic replay**: it only responds to the exact interaction sequences captured in the fixture (`G06`: full lifecycle; `A02`: prompt revision). Use `--quit` to exit — there's no `/exit` command.
|
|
109
|
+
|
|
110
|
+
Commands: `/help`, `/status`, `/session`, `/new`, `/resume`, `/mlflow [on|off]`, `/tokens [on|off]`, `/timeout [seconds]`, `/model [name]`, `/worker [api|codex|recorded]`, `/config` (codex only), `/sandbox` (codex only), `/workdir [path]`, `/transcript [path]`, `/quit`.
|
|
111
|
+
|
|
112
|
+
Fast-path review commands: `/confirm`, `/revise <feedback>`, `/stop`.
|
|
113
|
+
|
|
114
|
+
## Live model worker (default)
|
|
115
|
+
|
|
116
|
+
```powershell
|
|
117
|
+
# Default worker is 'api' with model 'z-ai/glm-4.7'
|
|
118
|
+
pdlt --new-session
|
|
119
|
+
# or: python -m pdl_taskmaster.host.repl --candidate-repo . --new-session
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
> **System 1 / System 2 Architecture Note**: Production default is `--worker api` (System 2, e.g. `z-ai/glm-4.7`). Local fast System 1 classification models are undergoing contrastive RLCD fine-tuning (ADR-0012) and fail-closed to System 2 via the confidence ladder if threshold ($0.85$) or top-2 margin ($0.40$) are unmet.
|
|
123
|
+
|
|
124
|
+
`--worker api` sends the compiled interpretation/plan directly to an OpenAI-compatible `/responses` endpoint (instructions bundled in `src/pdl_taskmaster/runtime/worker-bootstrap.txt`), with no tool definitions, sandbox, or agentic system prompt attached — deliberately; see the "instruction-lightness" finding in the whitepaper (§3).
|
|
125
|
+
|
|
126
|
+
### Efficiency flags (`--worker api`)
|
|
127
|
+
|
|
128
|
+
- `--api-reasoning-effort low|medium|high` — reasoning budget override, applied globally.
|
|
129
|
+
- `--api-reasoning-operation OP=EFFORT` (repeatable) — per-step override; sensible per-model defaults are already wired in (see whitepaper §4).
|
|
130
|
+
- `--api-model-operation OP=MODEL` (repeatable) — per-step model override, e.g. keep interpretation on a frontier model and route everything else to a cheaper instruction-following one.
|
|
131
|
+
- `--render-compact` / `--render-pretty` — how the interpretation/plan is serialized on the wire (compact is the default for `--worker api`).
|
|
132
|
+
- `--cache-order-render` — opt-in reordering for provider-side prefix caching.
|
|
133
|
+
|
|
134
|
+
Per-call cost data, and which optimizations were tried and rejected: [`docs/operations/efficiency-report.md`](docs/operations/efficiency-report.md).
|
|
135
|
+
|
|
136
|
+
## Adversarial evaluation
|
|
137
|
+
|
|
138
|
+
```powershell
|
|
139
|
+
python -m pdl_taskmaster.eval.run_qualified_batch --case-id BND-00 --stub --trials 1
|
|
140
|
+
python -m pdl_taskmaster.eval.run_qualified_batch --case-id BND-00 --trials 1 --model z-ai/glm-4.7
|
|
141
|
+
python -m pdl_taskmaster.eval.compare_eval_runs <summary.json>
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
The battery manifest and prior run ledgers resolve via `PDLT_RUNS_ROOT` (defaults to a sibling archive directory not included in this repository — point it at your own results directory to run the battery from scratch). Measured baselines so far: [`docs/operations/eval-metrics.md`](docs/operations/eval-metrics.md).
|
|
145
|
+
|
|
146
|
+
## Notes
|
|
147
|
+
|
|
148
|
+
- Run artifacts (`runs/`, `mlruns/`) and external evaluation archives are **not** runtime dependencies and are kept outside the repository. The optional MLflow tracking store follows the same pattern (override with `PDLT_MLFLOW_DB`; falls back to a repo-root `mlflow.db` if unset).
|
|
149
|
+
- Workspaces are zero-template and dynamic: a fresh run scaffolds only `state/`, `events/`, `stages/`, `shared/`, and stage folders materialize as they're needed (see whitepaper §5).
|
|
150
|
+
- `SOURCE_PROVENANCE.json` records per-file provenance (source repo, commit, hashes, byte-identical vs. adapted) from the original extraction.
|
|
151
|
+
- Internal architecture/technical decision records referenced elsewhere in these docs (`docs/adr/`, `docs/trd/`) are background only and not required reading to use the harness.
|
|
152
|
+
|
|
153
|
+
## Optional MLflow logging
|
|
154
|
+
|
|
155
|
+
MLflow logging is post-hoc and non-authoritative; the protocol never requires it. Enable it by answering `y` to the startup prompt, launching with `--mlflow`, or toggling with `/mlflow on`. When enabled, closed sessions are logged via `pdl_taskmaster.tracking.log_live_session` to a local SQLite store (default location follows the archive-resolution pattern described above) under experiment `PDL-R2S`, printing a `LIVE_SESSION_MLFLOW_RUN` line per logged session. Only sessions that completed at least one protocol turn are logged. Requires `pip install "pdl-taskmaster[tracking]"`.
|