nlp-refinement-cli 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. nlp_refinement/__init__.py +1 -0
  2. nlp_refinement/diagnostic_doe_support.py +84 -0
  3. nlp_refinement/doe_receipt_retention.py +81 -0
  4. nlp_refinement/format_contracts.py +157 -0
  5. nlp_refinement/host_client.py +505 -0
  6. nlp_refinement/host_diagnostics.py +430 -0
  7. nlp_refinement/host_execution_request.py +15 -0
  8. nlp_refinement/host_executor.py +104 -0
  9. nlp_refinement/host_refine.py +82 -0
  10. nlp_refinement/m8_doe_fixtures.py +28 -0
  11. nlp_refinement/m8_review_coordinator.py +370 -0
  12. nlp_refinement/patch_proposal_adapter.py +303 -0
  13. nlp_refinement/refine.py +372 -0
  14. nlp_refinement/repair_feasibility.py +327 -0
  15. nlp_refinement/resources/__init__.py +0 -0
  16. nlp_refinement/resources/prompts/patch-proposal-phase.v5.md +51 -0
  17. nlp_refinement/resources/prompts/review-phase.v5.md +57 -0
  18. nlp_refinement/resources/schemas/v5/acceptance-trace.schema.json +17 -0
  19. nlp_refinement/resources/schemas/v5/artifact.schema.json +17 -0
  20. nlp_refinement/resources/schemas/v5/finding.schema.json +27 -0
  21. nlp_refinement/resources/schemas/v5/patch-proposal-result.schema.json +66 -0
  22. nlp_refinement/resources/schemas/v5/phase-envelope.schema.json +18 -0
  23. nlp_refinement/resources/schemas/v5/retention-policy.schema.json +16 -0
  24. nlp_refinement/resources/schemas/v5/review-result.schema.json +37 -0
  25. nlp_refinement/resources/schemas/v5/run.schema.json +22 -0
  26. nlp_refinement/review_adapter.py +203 -0
  27. nlp_refinement/sensitive_data.py +98 -0
  28. nlp_refinement/state_store.py +1025 -0
  29. nlp_refinement_cli-0.9.0.dist-info/METADATA +300 -0
  30. nlp_refinement_cli-0.9.0.dist-info/RECORD +35 -0
  31. nlp_refinement_cli-0.9.0.dist-info/WHEEL +5 -0
  32. nlp_refinement_cli-0.9.0.dist-info/entry_points.txt +4 -0
  33. nlp_refinement_cli-0.9.0.dist-info/licenses/LICENSE +201 -0
  34. nlp_refinement_cli-0.9.0.dist-info/licenses/NOTICE +4 -0
  35. nlp_refinement_cli-0.9.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,22 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "title": "NLP-refinement Run v5",
4
+ "type": "object",
5
+ "required": ["schema_version", "run_id", "project_key", "target_snapshot_hash", "current_snapshot_hash", "authorization_mode", "state", "outcome", "revision", "repair_cycle_count", "max_repair_cycles", "open_high_risk_assumptions", "retention_policy_version"],
6
+ "properties": {
7
+ "schema_version": {"const": "5.0"},
8
+ "run_id": {"type": "string", "pattern": "^run_[a-z0-9]{12,64}$"},
9
+ "project_key": {"type": "string", "pattern": "^[a-f0-9]{64}$"},
10
+ "target_snapshot_hash": {"type": "string", "pattern": "^[a-f0-9]{64}$"},
11
+ "current_snapshot_hash": {"type": "string", "pattern": "^[a-f0-9]{64}$"},
12
+ "authorization_mode": {"type": "string", "enum": ["review-only", "bounded-repair", "repair-and-add-tests"]},
13
+ "state": {"type": "string", "enum": ["DRAFT", "READY", "REVIEWING", "TRIAGING", "REPAIRING", "VERIFYING", "REREVIEWING", "INTEGRITY_CHECK", "ROLLING_BACK", "SNAPSHOT_STALE", "STALE", "PASS", "PASS_WITH_ACCEPTED_RISK", "NEEDS_USER_DECISION", "NO_MATERIAL_DELTA", "BLOCKED", "CANCELLED"]},
14
+ "outcome": {"type": "string", "enum": ["IN_PROGRESS", "PASS", "PASS_WITH_ACCEPTED_RISK", "NEEDS_USER_DECISION", "NO_MATERIAL_DELTA", "SNAPSHOT_STALE", "BLOCKED", "CANCELLED", "STALE"]},
15
+ "revision": {"type": "integer", "minimum": 0},
16
+ "repair_cycle_count": {"type": "integer", "minimum": 0},
17
+ "max_repair_cycles": {"type": "integer", "minimum": 1, "maximum": 5},
18
+ "open_high_risk_assumptions": {"type": "integer", "minimum": 0},
19
+ "retention_policy_version": {"const": "5.0"}
20
+ },
21
+ "additionalProperties": false
22
+ }
@@ -0,0 +1,203 @@
1
+ """M3 adapter for accepting an externally produced, structured review result."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ import secrets
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from nlp_refinement.state_store import (
12
+ StateStore,
13
+ validate_finding_assessment,
14
+ validate_finding_classification,
15
+ )
16
+
17
+
18
+ FINGERPRINT = re.compile(r"[a-f0-9]{64}")
19
+ SEVERITIES = {"P0", "P1", "P2", "P3"}
20
+ GATES = {"blocking", "needs-user-decision", "recordable-risk"}
21
+ REQUIRED_RESULT = {"schema_version", "phase_name", "reviewer_id", "summary", "findings"}
22
+ REQUIRED_REVIEW_FINDING = {
23
+ "fingerprint", "origin_severity", "effective_gate", "invariant_id", "last_probe",
24
+ "statement", "evidence", "recommendation", "subject_kind", "evidence_plane", "category",
25
+ "evidence_confidence", "impact_scope", "claim_status",
26
+ }
27
+
28
+
29
+ class ReviewInputError(ValueError):
30
+ """Raised when external review data is not valid for the M3 protocol."""
31
+
32
+
33
+ def load_review_result(path: Path) -> dict[str, Any]:
34
+ try:
35
+ loaded = json.loads(path.read_text(encoding="utf-8"))
36
+ except json.JSONDecodeError as error:
37
+ raise ReviewInputError(f"review result is not valid JSON: {error.msg}") from error
38
+ if not isinstance(loaded, dict):
39
+ raise ReviewInputError("review result must be a JSON object")
40
+ validate_review_result(loaded)
41
+ return loaded
42
+
43
+
44
+ def validate_review_result(result: dict[str, Any]) -> None:
45
+ if set(result) != REQUIRED_RESULT:
46
+ raise ReviewInputError("review result has unexpected or missing fields")
47
+ if result["schema_version"] != "5.0" or result["phase_name"] != "review":
48
+ raise ReviewInputError("review result has an unsupported schema or phase")
49
+ for field in ("reviewer_id", "summary"):
50
+ if not isinstance(result[field], str) or not result[field].strip():
51
+ raise ReviewInputError(f"review result {field} must be a non-empty string")
52
+ if not isinstance(result["findings"], list):
53
+ raise ReviewInputError("review result findings must be an array")
54
+ fingerprints: set[str] = set()
55
+ for finding in result["findings"]:
56
+ if not isinstance(finding, dict) or set(finding) != REQUIRED_REVIEW_FINDING:
57
+ raise ReviewInputError("review finding has unexpected or missing fields")
58
+ for field in (
59
+ "fingerprint", "origin_severity", "effective_gate", "invariant_id", "last_probe",
60
+ "statement", "evidence", "recommendation", "subject_kind", "evidence_plane", "category",
61
+ "evidence_confidence", "impact_scope", "claim_status",
62
+ ):
63
+ if not isinstance(finding[field], str) or not finding[field].strip():
64
+ raise ReviewInputError(f"review finding {field} must be a non-empty string")
65
+ if FINGERPRINT.fullmatch(finding["fingerprint"]) is None:
66
+ raise ReviewInputError("review finding fingerprint must be lowercase SHA-256 hex")
67
+ if finding["origin_severity"] not in SEVERITIES or finding["effective_gate"] not in GATES:
68
+ raise ReviewInputError("review finding has an unsupported severity or gate")
69
+ try:
70
+ validate_finding_classification(
71
+ finding["subject_kind"], finding["evidence_plane"], finding["category"]
72
+ )
73
+ validate_finding_assessment(
74
+ finding["origin_severity"],
75
+ finding["effective_gate"],
76
+ finding["evidence_plane"],
77
+ finding["evidence_confidence"],
78
+ finding["impact_scope"],
79
+ finding["claim_status"],
80
+ )
81
+ except ValueError as error:
82
+ raise ReviewInputError(str(error)) from error
83
+ if finding["fingerprint"] in fingerprints:
84
+ raise ReviewInputError("review result contains a duplicate fingerprint")
85
+ fingerprints.add(finding["fingerprint"])
86
+
87
+
88
+ def _next_finding_id(findings: list[dict[str, Any]]) -> str:
89
+ highest = max((int(finding["finding_id"].removeprefix("F-")) for finding in findings), default=0)
90
+ return f"F-{highest + 1:03d}"
91
+
92
+
93
+ def _assert_compatible_duplicate(existing: dict[str, Any], incoming: dict[str, Any]) -> None:
94
+ for field in ("origin_severity", "effective_gate", "invariant_id"):
95
+ if existing[field] != incoming[field]:
96
+ raise ReviewInputError(f"duplicate fingerprint conflicts with immutable finding fields: {field}")
97
+ for field in (
98
+ "subject_kind", "evidence_plane", "category",
99
+ "evidence_confidence", "impact_scope", "claim_status",
100
+ ):
101
+ if existing[field] != incoming[field]:
102
+ raise ReviewInputError(f"duplicate fingerprint conflicts with immutable classification: {field}")
103
+ if existing["status"] != "open":
104
+ raise ReviewInputError("M3 can deduplicate only an open finding")
105
+
106
+
107
+ def ingest_review(
108
+ store: StateStore,
109
+ run_id: str,
110
+ review_result: dict[str, Any],
111
+ observed_snapshot_hash: str,
112
+ redaction_status: str,
113
+ now: int,
114
+ lease_token: str | None = None,
115
+ ) -> dict[str, Any]:
116
+ """Validate, append, and triage a review result without invoking a model."""
117
+ validate_review_result(review_result)
118
+ run = store.get_run(run_id)
119
+ if run["authorization_mode"] != "review-only":
120
+ raise ReviewInputError("M3 accepts only review-only runs")
121
+ if run["state"] != "READY":
122
+ raise ReviewInputError(f"M3 review requires READY state, found {run['state']}")
123
+
124
+ owned_lease = lease_token is None
125
+ token = lease_token or store.acquire_lease(run_id, f"m3-review-{secrets.token_hex(8)}", now=now)["token"]
126
+ try:
127
+ run = store.get_run(run_id)
128
+ review_body = json.dumps(review_result, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
129
+ if observed_snapshot_hash != run["current_snapshot_hash"]:
130
+ applied = store.apply_review_phase(
131
+ run_id,
132
+ expected_revision=run["revision"],
133
+ lease_token=token,
134
+ observed_snapshot_hash=observed_snapshot_hash,
135
+ review_body=review_body,
136
+ redaction_status=redaction_status,
137
+ planned_findings=[],
138
+ now=now,
139
+ )
140
+ return {**applied, "no_material_delta_candidates": []}
141
+ existing_findings = store.list_findings(run_id)
142
+ existing_by_fingerprint = {finding["fingerprint"]: finding for finding in existing_findings}
143
+ for incoming in review_result["findings"]:
144
+ existing = existing_by_fingerprint.get(incoming["fingerprint"])
145
+ if existing is not None:
146
+ _assert_compatible_duplicate(existing, incoming)
147
+
148
+ planned_findings: list[dict[str, Any]] = []
149
+ no_material_delta_candidates: list[str] = []
150
+ for incoming in review_result["findings"]:
151
+ existing = existing_by_fingerprint.get(incoming["fingerprint"])
152
+ if existing is None:
153
+ finding = {
154
+ "schema_version": "5.0",
155
+ "finding_id": _next_finding_id(existing_findings),
156
+ "run_id": run_id,
157
+ "fingerprint": incoming["fingerprint"],
158
+ "origin_severity": incoming["origin_severity"],
159
+ "effective_gate": incoming["effective_gate"],
160
+ "status": "open",
161
+ "invariant_id": incoming["invariant_id"],
162
+ "evidence_refs": [],
163
+ "repeat_count": 0,
164
+ "last_probe": incoming["last_probe"],
165
+ "accepted_risk_decision_ref": "none",
166
+ "subject_kind": incoming["subject_kind"],
167
+ "evidence_plane": incoming["evidence_plane"],
168
+ "category": incoming["category"],
169
+ "evidence_confidence": incoming["evidence_confidence"],
170
+ "impact_scope": incoming["impact_scope"],
171
+ "claim_status": incoming["claim_status"],
172
+ }
173
+ existing_findings.append(finding)
174
+ existing_by_fingerprint[finding["fingerprint"]] = finding
175
+ planned_findings.append({"action": "new", "finding": finding})
176
+ continue
177
+
178
+ same_probe = existing["last_probe"] == incoming["last_probe"]
179
+ duplicate = dict(existing)
180
+ duplicate["repeat_count"] = existing["repeat_count"] + 1
181
+ duplicate["last_probe"] = incoming["last_probe"]
182
+ existing_by_fingerprint[duplicate["fingerprint"]] = duplicate
183
+ planned_findings.append({"action": "duplicate", "finding": duplicate})
184
+ if same_probe and duplicate["repeat_count"] >= 2:
185
+ no_material_delta_candidates.append(duplicate["finding_id"])
186
+
187
+ applied = store.apply_review_phase(
188
+ run_id,
189
+ expected_revision=run["revision"],
190
+ lease_token=token,
191
+ observed_snapshot_hash=observed_snapshot_hash,
192
+ review_body=review_body,
193
+ redaction_status=redaction_status,
194
+ planned_findings=planned_findings,
195
+ now=now,
196
+ )
197
+ return {
198
+ **applied,
199
+ "no_material_delta_candidates": no_material_delta_candidates,
200
+ }
201
+ finally:
202
+ if owned_lease:
203
+ store.release_lease(run_id, token, now=now)
@@ -0,0 +1,98 @@
1
+ """Narrow secret guards and model-input redaction for review-only flows."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from dataclasses import dataclass
6
+ import re
7
+
8
+
9
+ REDACTION_DETECTOR_VERSION = "1"
10
+
11
+
12
+ class RedactionRequiredError(ValueError):
13
+ """A likely secret cannot be safely transformed without user input."""
14
+
15
+
16
+ @dataclass(frozen=True)
17
+ class ModelInputRedaction:
18
+ """An in-memory model view and content-free replacement summary."""
19
+
20
+ text: str
21
+ replacement_counts: dict[str, int]
22
+
23
+
24
+ _PRIVATE_KEY_BLOCK = re.compile(
25
+ r"-----BEGIN(?: [A-Z0-9]+)? PRIVATE KEY-----.*?-----END(?: [A-Z0-9]+)? PRIVATE KEY-----",
26
+ re.DOTALL,
27
+ )
28
+ _INCOMPLETE_PRIVATE_KEY = re.compile(r"-----BEGIN(?: [A-Z0-9]+)? PRIVATE KEY-----")
29
+ _REDACTABLE_PATTERNS = (
30
+ ("private-key", _PRIVATE_KEY_BLOCK),
31
+ ("bearer-token", re.compile(r"(?i)\bbearer\s+[A-Za-z0-9._~+/-]{20,}")),
32
+ ("aws-access-key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
33
+ ("github-token", re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b")),
34
+ ("api-key", re.compile(r"\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b")),
35
+ )
36
+ _FUZZY_ASSIGNMENT = re.compile(
37
+ r"(?im)^\s*(?:export\s+)?[\"']?(?:password|passwd|secret|token|api[_-]?key|apikey|authorization)[\"']?\s*[:=]\s*(?P<value>[^#\r\n]+?)\s*$"
38
+ )
39
+
40
+
41
+ def redact_for_model(value: str) -> ModelInputRedaction:
42
+ """Return a safe-to-send view or require a user-produced redacted copy.
43
+
44
+ Only complete, high-confidence values are replaced. The placeholder map is
45
+ local to this call and is deliberately not returned or persisted.
46
+ """
47
+ text = value
48
+ replacement_counts: dict[str, int] = {}
49
+ placeholder_ids: dict[tuple[str, str], int] = {}
50
+
51
+ def replace(label: str, match: re.Match[str]) -> str:
52
+ original = match.group(0)
53
+ key = (label, original)
54
+ placeholder_id = placeholder_ids.setdefault(key, sum(1 for known_label, _ in placeholder_ids if known_label == label) + 1)
55
+ replacement_counts[label] = replacement_counts.get(label, 0) + 1
56
+ return f"<REDACTED:{label}:{placeholder_id}>"
57
+
58
+ for label, pattern in _REDACTABLE_PATTERNS:
59
+ text = pattern.sub(lambda match, label=label: replace(label, match), text)
60
+
61
+ if _INCOMPLETE_PRIVATE_KEY.search(text):
62
+ raise RedactionRequiredError("incomplete-private-key")
63
+ if _fuzzy_assignment_present(text):
64
+ raise RedactionRequiredError("ambiguous-secret-assignment")
65
+ return ModelInputRedaction(text=text, replacement_counts=dict(sorted(replacement_counts.items())))
66
+
67
+
68
+ def likely_secret_kind(value: str) -> str | None:
69
+ """Return a stable category, never echo the matched value.
70
+
71
+ This is a narrow rejection guard for persistence, not proof that other
72
+ content is safe. Unlike model-input redaction, it treats incomplete and
73
+ ambiguous values as unsafe to persist too.
74
+ """
75
+ for label, pattern in _REDACTABLE_PATTERNS:
76
+ if pattern.search(value):
77
+ return label
78
+ if _INCOMPLETE_PRIVATE_KEY.search(value):
79
+ return "incomplete-private-key"
80
+ if _fuzzy_assignment_present(value):
81
+ return "ambiguous-secret-assignment"
82
+ return None
83
+
84
+
85
+ def _fuzzy_assignment_present(value: str) -> bool:
86
+ for match in _FUZZY_ASSIGNMENT.finditer(value):
87
+ candidate = match.group("value").strip().rstrip(",").strip().strip("\"'")
88
+ normalized = candidate.lower()
89
+ if candidate.startswith("<REDACTED:"):
90
+ continue
91
+ if normalized.startswith("bearer <redacted:"):
92
+ continue
93
+ if candidate.startswith("${") and candidate.endswith("}"):
94
+ continue
95
+ if re.fullmatch(r"\$[A-Za-z_][A-Za-z0-9_]*", candidate):
96
+ continue
97
+ return True
98
+ return False