dataforge-07 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataforge/__init__.py +204 -0
- dataforge/__main__.py +5 -0
- dataforge/agent/__init__.py +16 -0
- dataforge/agent/providers.py +259 -0
- dataforge/agent/scratchpad.py +183 -0
- dataforge/agent/tool_actions.py +343 -0
- dataforge/bench/__init__.py +31 -0
- dataforge/bench/core.py +426 -0
- dataforge/bench/groq_client.py +386 -0
- dataforge/bench/methods.py +443 -0
- dataforge/bench/report.py +309 -0
- dataforge/bench/runner.py +247 -0
- dataforge/causal/__init__.py +21 -0
- dataforge/causal/dag.py +174 -0
- dataforge/causal/pc.py +232 -0
- dataforge/causal/root_cause.py +193 -0
- dataforge/cli/__init__.py +50 -0
- dataforge/cli/audit.py +70 -0
- dataforge/cli/bench.py +154 -0
- dataforge/cli/common.py +267 -0
- dataforge/cli/constraints.py +407 -0
- dataforge/cli/profile.py +147 -0
- dataforge/cli/release.py +166 -0
- dataforge/cli/repair.py +407 -0
- dataforge/cli/revert.py +139 -0
- dataforge/cli/watch.py +144 -0
- dataforge/datasets/__init__.py +25 -0
- dataforge/datasets/embedded/hospital/clean.csv +11 -0
- dataforge/datasets/embedded/hospital/dirty.csv +11 -0
- dataforge/datasets/real_world.py +290 -0
- dataforge/datasets/registry.py +103 -0
- dataforge/detectors/__init__.py +80 -0
- dataforge/detectors/base.py +145 -0
- dataforge/detectors/decimal_shift.py +166 -0
- dataforge/detectors/fd_violation.py +157 -0
- dataforge/detectors/type_mismatch.py +173 -0
- dataforge/engine/__init__.py +39 -0
- dataforge/engine/repair.py +905 -0
- dataforge/env/__init__.py +22 -0
- dataforge/env/environment.py +883 -0
- dataforge/env/observation.py +61 -0
- dataforge/env/openenv_core.py +161 -0
- dataforge/env/reward.py +128 -0
- dataforge/env/server.py +176 -0
- dataforge/evaluation_contract.py +76 -0
- dataforge/fixtures/hospital_10rows.csv +11 -0
- dataforge/fixtures/hospital_schema.yaml +17 -0
- dataforge/http/__init__.py +1 -0
- dataforge/http/problem.py +103 -0
- dataforge/integrations/__init__.py +1 -0
- dataforge/integrations/dbt.py +164 -0
- dataforge/observability.py +76 -0
- dataforge/py.typed +1 -0
- dataforge/release/__init__.py +1 -0
- dataforge/release/doctor.py +367 -0
- dataforge/release/full_vision.py +702 -0
- dataforge/release/gate.py +861 -0
- dataforge/release/playground_check.py +411 -0
- dataforge/repair_contract.py +468 -0
- dataforge/repairers/__init__.py +88 -0
- dataforge/repairers/base.py +77 -0
- dataforge/repairers/decimal_shift.py +43 -0
- dataforge/repairers/fd_violation.py +225 -0
- dataforge/repairers/type_mismatch.py +73 -0
- dataforge/safety/__init__.py +5 -0
- dataforge/safety/adversarial/attack_01_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_02_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_03_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_04_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_05_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_06_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_07_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_08_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_09_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_10_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_11_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_12_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_13_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_14_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_15_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_16_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_17_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_18_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_19_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_20_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_21_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_22_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_23_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_24_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_25_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_26_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_27_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_28_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_29_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_30_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_31_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_32_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_33_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_34_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_35_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_36_row_delete.yaml +11 -0
- dataforge/safety/adversarial/attack_37_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_38_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_39_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_40_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_41_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_42_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_43_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_44_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_45_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_46_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_47_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_48_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_49_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_50_row_delete.yaml +7 -0
- dataforge/safety/constitution.py +307 -0
- dataforge/safety/constitutions/default.yaml +40 -0
- dataforge/safety/filter.py +134 -0
- dataforge/schema_inference.py +620 -0
- dataforge/stores/__init__.py +46 -0
- dataforge/stores/base.py +73 -0
- dataforge/stores/cloud.py +78 -0
- dataforge/stores/csv.py +94 -0
- dataforge/stores/duckdb.py +313 -0
- dataforge/stores/patch_plan.py +178 -0
- dataforge/stores/registry.py +82 -0
- dataforge/stores/repair.py +121 -0
- dataforge/stores/revert.py +22 -0
- dataforge/stores/sql.py +27 -0
- dataforge/table.py +228 -0
- dataforge/transactions/__init__.py +34 -0
- dataforge/transactions/files.py +96 -0
- dataforge/transactions/log.py +613 -0
- dataforge/transactions/revert.py +102 -0
- dataforge/transactions/txn.py +104 -0
- dataforge/ui/__init__.py +1 -0
- dataforge/ui/profile_view.py +136 -0
- dataforge/ui/repair_diff.py +91 -0
- dataforge/verifier/__init__.py +55 -0
- dataforge/verifier/constraint_ir.py +155 -0
- dataforge/verifier/explain.py +47 -0
- dataforge/verifier/gate.py +5 -0
- dataforge/verifier/schema.py +111 -0
- dataforge/verifier/smt.py +433 -0
- dataforge_07-0.1.0.dist-info/METADATA +436 -0
- dataforge_07-0.1.0.dist-info/RECORD +150 -0
- dataforge_07-0.1.0.dist-info/WHEEL +5 -0
- dataforge_07-0.1.0.dist-info/entry_points.txt +3 -0
- dataforge_07-0.1.0.dist-info/licenses/LICENSE +176 -0
- dataforge_07-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,861 @@
|
|
|
1
|
+
"""Canonical local release gate for DataForge.
|
|
2
|
+
|
|
3
|
+
The release gate is intentionally heavier than the release doctor: it proves a
|
|
4
|
+
fresh user can install the built wheel from a local wheelhouse and run the core
|
|
5
|
+
CLI lifecycle without relying on the source checkout.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import tarfile
|
|
16
|
+
import tempfile
|
|
17
|
+
import zipfile
|
|
18
|
+
from dataclasses import asdict, dataclass, field
|
|
19
|
+
from hashlib import sha256
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
SCHEMA_VERSION = "release_gate_report_v1"
|
|
24
|
+
PACKAGE_NAME = "dataforge_07"
|
|
25
|
+
CLI_NAMES = ("dataforge", "dataforge15")
|
|
26
|
+
REQUIRED_WHEEL_MEMBERS = frozenset(
|
|
27
|
+
{
|
|
28
|
+
"dataforge/py.typed",
|
|
29
|
+
"dataforge/cli/__init__.py",
|
|
30
|
+
"dataforge/cli/constraints.py",
|
|
31
|
+
"dataforge/cli/repair.py",
|
|
32
|
+
"dataforge/cli/revert.py",
|
|
33
|
+
"dataforge/cli/audit.py",
|
|
34
|
+
"dataforge/fixtures/hospital_10rows.csv",
|
|
35
|
+
"dataforge/fixtures/hospital_schema.yaml",
|
|
36
|
+
"dataforge/safety/constitutions/default.yaml",
|
|
37
|
+
"dataforge/stores/cloud.py",
|
|
38
|
+
"dataforge/stores/duckdb.py",
|
|
39
|
+
"dataforge/stores/patch_plan.py",
|
|
40
|
+
"dataforge/stores/registry.py",
|
|
41
|
+
"dataforge/stores/repair.py",
|
|
42
|
+
"dataforge/transactions/log.py",
|
|
43
|
+
"dataforge/transactions/revert.py",
|
|
44
|
+
}
|
|
45
|
+
)
|
|
46
|
+
REJECTED_WHEEL_PREFIXES = (
|
|
47
|
+
"tests/",
|
|
48
|
+
"data_quality_env/",
|
|
49
|
+
".github/",
|
|
50
|
+
".hf-space",
|
|
51
|
+
"build/",
|
|
52
|
+
"dist/",
|
|
53
|
+
"node_modules/",
|
|
54
|
+
)
|
|
55
|
+
REQUIRED_SDIST_MEMBERS = frozenset(
|
|
56
|
+
{
|
|
57
|
+
"PKG-INFO",
|
|
58
|
+
"README.md",
|
|
59
|
+
"LICENSE",
|
|
60
|
+
"MANIFEST.in",
|
|
61
|
+
"pyproject.toml",
|
|
62
|
+
"dataforge/__init__.py",
|
|
63
|
+
"dataforge/py.typed",
|
|
64
|
+
"dataforge/cli/profile.py",
|
|
65
|
+
"dataforge/cli/constraints.py",
|
|
66
|
+
"dataforge/cli/repair.py",
|
|
67
|
+
"dataforge/stores/cloud.py",
|
|
68
|
+
"dataforge/stores/duckdb.py",
|
|
69
|
+
"dataforge/stores/patch_plan.py",
|
|
70
|
+
"dataforge/stores/registry.py",
|
|
71
|
+
"dataforge/stores/repair.py",
|
|
72
|
+
"dataforge/fixtures/hospital_10rows.csv",
|
|
73
|
+
"dataforge/fixtures/hospital_schema.yaml",
|
|
74
|
+
}
|
|
75
|
+
)
|
|
76
|
+
ALLOWED_SDIST_TOP_LEVEL = {
|
|
77
|
+
"PKG-INFO",
|
|
78
|
+
"README.md",
|
|
79
|
+
"LICENSE",
|
|
80
|
+
"MANIFEST.in",
|
|
81
|
+
"pyproject.toml",
|
|
82
|
+
"setup.cfg",
|
|
83
|
+
"dataforge",
|
|
84
|
+
f"{PACKAGE_NAME}.egg-info",
|
|
85
|
+
}
|
|
86
|
+
ALLOWED_SDIST_EGG_INFO = {
|
|
87
|
+
"PKG-INFO",
|
|
88
|
+
"SOURCES.txt",
|
|
89
|
+
"dependency_links.txt",
|
|
90
|
+
"entry_points.txt",
|
|
91
|
+
"requires.txt",
|
|
92
|
+
"top_level.txt",
|
|
93
|
+
}
|
|
94
|
+
REJECTED_SDIST_PREFIXES = (
|
|
95
|
+
"tests/",
|
|
96
|
+
"data_quality_env/",
|
|
97
|
+
".github/",
|
|
98
|
+
".hf-space",
|
|
99
|
+
"build/",
|
|
100
|
+
"dist/",
|
|
101
|
+
"node_modules/",
|
|
102
|
+
"logs/",
|
|
103
|
+
"eval/",
|
|
104
|
+
"training/",
|
|
105
|
+
"playground/",
|
|
106
|
+
"playground-model/",
|
|
107
|
+
"dataforge-mcp/",
|
|
108
|
+
"benchmark_results/",
|
|
109
|
+
"datasets/",
|
|
110
|
+
"scripts/",
|
|
111
|
+
"docs/",
|
|
112
|
+
)
|
|
113
|
+
REJECTED_ROOT_LEGACY_FILES = {
|
|
114
|
+
"analyze_trajectory.py",
|
|
115
|
+
"benchmark.py",
|
|
116
|
+
"client.py",
|
|
117
|
+
"compat.py",
|
|
118
|
+
"generate_datasets.py",
|
|
119
|
+
"heuristic_baseline.py",
|
|
120
|
+
"inference.py",
|
|
121
|
+
"models.py",
|
|
122
|
+
"random_baseline.py",
|
|
123
|
+
"run_baseline.py",
|
|
124
|
+
"test_env.py",
|
|
125
|
+
"verify_all_fields.py",
|
|
126
|
+
"verify_nuclear.py",
|
|
127
|
+
"verify_score_range.py",
|
|
128
|
+
}
|
|
129
|
+
REJECTED_WHEEL_PARTS = {
|
|
130
|
+
"__pycache__",
|
|
131
|
+
".pytest_cache",
|
|
132
|
+
".mypy_cache",
|
|
133
|
+
".ruff_cache",
|
|
134
|
+
".hypothesis",
|
|
135
|
+
}
|
|
136
|
+
REJECTED_SDIST_PARTS = REJECTED_WHEEL_PARTS | {
|
|
137
|
+
".benchmarks",
|
|
138
|
+
".claude",
|
|
139
|
+
".idea",
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass(frozen=True)
|
|
144
|
+
class ReleaseGateStep:
|
|
145
|
+
"""One release-gate check or command result."""
|
|
146
|
+
|
|
147
|
+
name: str
|
|
148
|
+
ok: bool
|
|
149
|
+
detail: str
|
|
150
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
@dataclass(frozen=True)
|
|
154
|
+
class ReleaseGateReport:
|
|
155
|
+
"""Machine-readable proof emitted by ``dataforge release gate``."""
|
|
156
|
+
|
|
157
|
+
ok: bool
|
|
158
|
+
steps: list[ReleaseGateStep]
|
|
159
|
+
schema_version: str = SCHEMA_VERSION
|
|
160
|
+
artifact_sha256: dict[str, str] = field(default_factory=dict)
|
|
161
|
+
offline_install: bool = True
|
|
162
|
+
secrets_printed: bool = False
|
|
163
|
+
|
|
164
|
+
def to_dict(self) -> dict[str, Any]:
|
|
165
|
+
"""Return a JSON-serializable report."""
|
|
166
|
+
return asdict(self)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class ReleaseGateError(RuntimeError):
|
|
170
|
+
"""Raised when the release gate cannot be completed."""
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _project_root() -> Path:
|
|
174
|
+
"""Return the repository root for the DataForge package."""
|
|
175
|
+
return Path(__file__).resolve().parents[2]
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _python_in_venv(venv_path: Path) -> Path:
|
|
179
|
+
"""Return the Python executable path inside a venv."""
|
|
180
|
+
if os.name == "nt":
|
|
181
|
+
return venv_path / "Scripts" / "python.exe"
|
|
182
|
+
return venv_path / "bin" / "python"
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _script_in_venv(venv_path: Path, name: str) -> Path:
|
|
186
|
+
"""Return an installed console-script path inside a venv."""
|
|
187
|
+
suffix = ".exe" if os.name == "nt" else ""
|
|
188
|
+
directory = "Scripts" if os.name == "nt" else "bin"
|
|
189
|
+
return venv_path / directory / f"{name}{suffix}"
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _command_text(command: list[str | os.PathLike[str]]) -> str:
|
|
193
|
+
"""Return a readable command string without shell quoting semantics."""
|
|
194
|
+
return " ".join(str(part) for part in command)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _tail(text: str, *, limit: int = 4000) -> str:
|
|
198
|
+
"""Return the tail of command output for compact reports."""
|
|
199
|
+
if len(text) <= limit:
|
|
200
|
+
return text
|
|
201
|
+
return text[-limit:]
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _output_text(value: str | bytes | None) -> str:
|
|
205
|
+
"""Normalize subprocess timeout output to text."""
|
|
206
|
+
if value is None:
|
|
207
|
+
return ""
|
|
208
|
+
if isinstance(value, bytes):
|
|
209
|
+
return value.decode("utf-8", errors="replace")
|
|
210
|
+
return value
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _gate_env() -> dict[str, str]:
|
|
214
|
+
"""Return a command environment that avoids source-checkout import leakage."""
|
|
215
|
+
env = os.environ.copy()
|
|
216
|
+
env.pop("PYTHONPATH", None)
|
|
217
|
+
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
|
|
218
|
+
env.setdefault("PYTHONIOENCODING", "utf-8")
|
|
219
|
+
return env
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _run_command(
|
|
223
|
+
name: str,
|
|
224
|
+
command: list[str | os.PathLike[str]],
|
|
225
|
+
*,
|
|
226
|
+
cwd: Path,
|
|
227
|
+
timeout_seconds: int = 120,
|
|
228
|
+
) -> tuple[ReleaseGateStep, subprocess.CompletedProcess[str]]:
|
|
229
|
+
"""Run a command and convert the result into a release-gate step."""
|
|
230
|
+
try:
|
|
231
|
+
result = subprocess.run(
|
|
232
|
+
[str(part) for part in command],
|
|
233
|
+
cwd=cwd,
|
|
234
|
+
capture_output=True,
|
|
235
|
+
text=True,
|
|
236
|
+
encoding="utf-8",
|
|
237
|
+
errors="replace",
|
|
238
|
+
timeout=timeout_seconds,
|
|
239
|
+
check=False,
|
|
240
|
+
env=_gate_env(),
|
|
241
|
+
)
|
|
242
|
+
except FileNotFoundError as exc:
|
|
243
|
+
step = ReleaseGateStep(
|
|
244
|
+
name=name,
|
|
245
|
+
ok=False,
|
|
246
|
+
detail=f"Command not found: {exc}",
|
|
247
|
+
metadata={"command": _command_text(command)},
|
|
248
|
+
)
|
|
249
|
+
return step, subprocess.CompletedProcess([str(part) for part in command], 127)
|
|
250
|
+
except subprocess.TimeoutExpired as exc:
|
|
251
|
+
step = ReleaseGateStep(
|
|
252
|
+
name=name,
|
|
253
|
+
ok=False,
|
|
254
|
+
detail=f"Timed out after {timeout_seconds}s.",
|
|
255
|
+
metadata={
|
|
256
|
+
"command": _command_text(command),
|
|
257
|
+
"stdout_tail": _tail(_output_text(exc.stdout)),
|
|
258
|
+
"stderr_tail": _tail(_output_text(exc.stderr)),
|
|
259
|
+
},
|
|
260
|
+
)
|
|
261
|
+
return step, subprocess.CompletedProcess([str(part) for part in command], 124)
|
|
262
|
+
|
|
263
|
+
ok = result.returncode == 0
|
|
264
|
+
step = ReleaseGateStep(
|
|
265
|
+
name=name,
|
|
266
|
+
ok=ok,
|
|
267
|
+
detail="passed" if ok else f"exited {result.returncode}",
|
|
268
|
+
metadata={
|
|
269
|
+
"command": _command_text(command),
|
|
270
|
+
"returncode": result.returncode,
|
|
271
|
+
"stdout_tail": _tail(result.stdout),
|
|
272
|
+
"stderr_tail": _tail(result.stderr),
|
|
273
|
+
},
|
|
274
|
+
)
|
|
275
|
+
return step, result
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _file_sha256(path: Path) -> str:
|
|
279
|
+
"""Return a SHA-256 digest for a file."""
|
|
280
|
+
digest = sha256()
|
|
281
|
+
with path.open("rb") as handle:
|
|
282
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
283
|
+
digest.update(chunk)
|
|
284
|
+
return digest.hexdigest()
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def _find_single_artifact(directory: Path, pattern: str) -> Path:
|
|
288
|
+
"""Return the unique newest artifact matching ``pattern``."""
|
|
289
|
+
matches = sorted(directory.glob(pattern), key=lambda path: path.name)
|
|
290
|
+
if not matches:
|
|
291
|
+
raise ReleaseGateError(f"No artifact matched {pattern!r} in {directory}.")
|
|
292
|
+
if len(matches) > 1:
|
|
293
|
+
raise ReleaseGateError(f"Expected one artifact matching {pattern!r}, found {len(matches)}.")
|
|
294
|
+
return matches[0]
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _audit_wheel_contents(wheel_path: Path) -> ReleaseGateStep:
|
|
298
|
+
"""Verify the wheel contains only the public package boundary."""
|
|
299
|
+
errors: list[str] = []
|
|
300
|
+
with zipfile.ZipFile(wheel_path) as archive:
|
|
301
|
+
members = sorted(info.filename for info in archive.infolist() if not info.is_dir())
|
|
302
|
+
|
|
303
|
+
for member in members:
|
|
304
|
+
parts = set(Path(member).parts)
|
|
305
|
+
if any(member.startswith(prefix) for prefix in REJECTED_WHEEL_PREFIXES):
|
|
306
|
+
errors.append(f"Rejected path prefix: {member}")
|
|
307
|
+
if parts & REJECTED_WHEEL_PARTS:
|
|
308
|
+
errors.append(f"Rejected generated/cache path: {member}")
|
|
309
|
+
if member.endswith((".pyc", ".pyo")):
|
|
310
|
+
errors.append(f"Rejected bytecode file: {member}")
|
|
311
|
+
if not (
|
|
312
|
+
member.startswith("dataforge/")
|
|
313
|
+
or (member.startswith(f"{PACKAGE_NAME}-") and ".dist-info/" in member)
|
|
314
|
+
):
|
|
315
|
+
errors.append(f"Unexpected top-level wheel member: {member}")
|
|
316
|
+
|
|
317
|
+
missing = sorted(REQUIRED_WHEEL_MEMBERS - set(members))
|
|
318
|
+
errors.extend(f"Missing required wheel member: {member}" for member in missing)
|
|
319
|
+
metadata = {
|
|
320
|
+
"wheel": str(wheel_path),
|
|
321
|
+
"member_count": len(members),
|
|
322
|
+
"required_count": len(REQUIRED_WHEEL_MEMBERS),
|
|
323
|
+
"missing": missing,
|
|
324
|
+
}
|
|
325
|
+
return ReleaseGateStep(
|
|
326
|
+
name="wheel_contents_audit",
|
|
327
|
+
ok=not errors,
|
|
328
|
+
detail="wheel contains only the allowed DataForge package surface"
|
|
329
|
+
if not errors
|
|
330
|
+
else "wheel contents audit failed",
|
|
331
|
+
metadata={**metadata, "errors": errors},
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _strip_sdist_root(member: str) -> str:
|
|
336
|
+
"""Return an sdist member path without the archive root directory."""
|
|
337
|
+
parts = member.split("/", 1)
|
|
338
|
+
if len(parts) == 2 and parts[0].startswith(f"{PACKAGE_NAME}-"):
|
|
339
|
+
return parts[1]
|
|
340
|
+
return member
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _audit_sdist_contents(sdist_path: Path) -> ReleaseGateStep:
|
|
344
|
+
"""Verify the source distribution contains only release-owned core assets."""
|
|
345
|
+
errors: list[str] = []
|
|
346
|
+
with tarfile.open(sdist_path, "r:gz") as archive:
|
|
347
|
+
members = sorted(
|
|
348
|
+
_strip_sdist_root(info.name) for info in archive.getmembers() if info.isfile()
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
for member in members:
|
|
352
|
+
parts = set(Path(member).parts)
|
|
353
|
+
top_level = member.split("/", 1)[0]
|
|
354
|
+
if any(member.startswith(prefix) for prefix in REJECTED_SDIST_PREFIXES):
|
|
355
|
+
errors.append(f"Rejected path prefix: {member}")
|
|
356
|
+
if parts & REJECTED_SDIST_PARTS:
|
|
357
|
+
errors.append(f"Rejected generated/cache path: {member}")
|
|
358
|
+
if member in REJECTED_ROOT_LEGACY_FILES:
|
|
359
|
+
errors.append(f"Rejected legacy root wrapper: {member}")
|
|
360
|
+
if member.endswith((".pyc", ".pyo", ".ipynb")):
|
|
361
|
+
errors.append(f"Rejected generated or notebook file: {member}")
|
|
362
|
+
if top_level not in ALLOWED_SDIST_TOP_LEVEL:
|
|
363
|
+
errors.append(f"Unexpected top-level sdist member: {member}")
|
|
364
|
+
if member.startswith(f"{PACKAGE_NAME}.egg-info/"):
|
|
365
|
+
egg_info_name = member.split("/", 1)[1]
|
|
366
|
+
if egg_info_name not in ALLOWED_SDIST_EGG_INFO:
|
|
367
|
+
errors.append(f"Unexpected egg-info member: {member}")
|
|
368
|
+
|
|
369
|
+
missing = sorted(REQUIRED_SDIST_MEMBERS - set(members))
|
|
370
|
+
errors.extend(f"Missing required sdist member: {member}" for member in missing)
|
|
371
|
+
metadata = {
|
|
372
|
+
"sdist": str(sdist_path),
|
|
373
|
+
"member_count": len(members),
|
|
374
|
+
"required_count": len(REQUIRED_SDIST_MEMBERS),
|
|
375
|
+
"missing": missing,
|
|
376
|
+
}
|
|
377
|
+
return ReleaseGateStep(
|
|
378
|
+
name="sdist_contents_audit",
|
|
379
|
+
ok=not errors,
|
|
380
|
+
detail="sdist contains only the allowed DataForge core source surface"
|
|
381
|
+
if not errors
|
|
382
|
+
else "sdist contents audit failed",
|
|
383
|
+
metadata={**metadata, "errors": errors},
|
|
384
|
+
)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _json_step(
|
|
388
|
+
name: str,
|
|
389
|
+
result: subprocess.CompletedProcess[str],
|
|
390
|
+
*,
|
|
391
|
+
expected: dict[str, Any],
|
|
392
|
+
) -> ReleaseGateStep:
|
|
393
|
+
"""Validate a command emitted JSON containing expected key/value pairs."""
|
|
394
|
+
try:
|
|
395
|
+
payload = json.loads(result.stdout)
|
|
396
|
+
except json.JSONDecodeError as exc:
|
|
397
|
+
return ReleaseGateStep(
|
|
398
|
+
name=name,
|
|
399
|
+
ok=False,
|
|
400
|
+
detail=f"Could not parse JSON output: {exc}",
|
|
401
|
+
metadata={"stdout_tail": _tail(result.stdout), "stderr_tail": _tail(result.stderr)},
|
|
402
|
+
)
|
|
403
|
+
errors = [
|
|
404
|
+
f"Expected {key}={value!r}, found {payload.get(key)!r}"
|
|
405
|
+
for key, value in expected.items()
|
|
406
|
+
if payload.get(key) != value
|
|
407
|
+
]
|
|
408
|
+
return ReleaseGateStep(
|
|
409
|
+
name=name,
|
|
410
|
+
ok=not errors,
|
|
411
|
+
detail="JSON contract matched" if not errors else "JSON contract mismatch",
|
|
412
|
+
metadata={"expected": expected, "errors": errors, "keys": sorted(payload)},
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _warehouse_dry_run_contract_step(result: subprocess.CompletedProcess[str]) -> ReleaseGateStep:
|
|
417
|
+
"""Validate the installed warehouse dry-run contract without credentials."""
|
|
418
|
+
try:
|
|
419
|
+
payload = json.loads(result.stdout)
|
|
420
|
+
except json.JSONDecodeError as exc:
|
|
421
|
+
return ReleaseGateStep(
|
|
422
|
+
name="smoke_warehouse_cloud_dry_run_contract",
|
|
423
|
+
ok=False,
|
|
424
|
+
detail=f"Could not parse JSON output: {exc}",
|
|
425
|
+
metadata={"stdout_tail": _tail(result.stdout), "stderr_tail": _tail(result.stderr)},
|
|
426
|
+
)
|
|
427
|
+
|
|
428
|
+
plan = payload.get("patch_plan")
|
|
429
|
+
errors: list[str] = []
|
|
430
|
+
if payload.get("schema_version") != "table_store_repair_result_v1":
|
|
431
|
+
errors.append("warehouse dry-run result schema_version mismatch")
|
|
432
|
+
if payload.get("backend") != "snowflake":
|
|
433
|
+
errors.append(f"expected backend='snowflake', found {payload.get('backend')!r}")
|
|
434
|
+
if payload.get("mode") != "dry_run":
|
|
435
|
+
errors.append(f"expected mode='dry_run', found {payload.get('mode')!r}")
|
|
436
|
+
if not isinstance(plan, dict):
|
|
437
|
+
errors.append("patch_plan is missing or is not an object")
|
|
438
|
+
else:
|
|
439
|
+
if plan.get("schema_version") != "patch_plan_v1":
|
|
440
|
+
errors.append("patch_plan schema_version mismatch")
|
|
441
|
+
if plan.get("apply_supported") is not False:
|
|
442
|
+
errors.append("cloud warehouse dry-run must not claim apply support")
|
|
443
|
+
if plan.get("reversible") is not False:
|
|
444
|
+
errors.append("cloud warehouse dry-run must not claim reversibility")
|
|
445
|
+
if plan.get("operations") != []:
|
|
446
|
+
errors.append("cloud warehouse dry-run must not emit write operations")
|
|
447
|
+
return ReleaseGateStep(
|
|
448
|
+
name="smoke_warehouse_cloud_dry_run_contract",
|
|
449
|
+
ok=not errors,
|
|
450
|
+
detail="warehouse dry-run contract matched"
|
|
451
|
+
if not errors
|
|
452
|
+
else "warehouse dry-run contract mismatch",
|
|
453
|
+
metadata={
|
|
454
|
+
"errors": errors,
|
|
455
|
+
"top_level_keys": sorted(payload) if isinstance(payload, dict) else [],
|
|
456
|
+
"patch_plan_keys": sorted(plan) if isinstance(plan, dict) else [],
|
|
457
|
+
},
|
|
458
|
+
)
|
|
459
|
+
|
|
460
|
+
|
|
461
|
+
def _copy_packaged_fixtures(venv_python: Path, smoke_dir: Path, *, cwd: Path) -> ReleaseGateStep:
|
|
462
|
+
"""Copy demo fixtures from the installed wheel into a mutable smoke directory."""
|
|
463
|
+
script = (
|
|
464
|
+
"from importlib.resources import files\n"
|
|
465
|
+
"from pathlib import Path\n"
|
|
466
|
+
"import shutil, sys\n"
|
|
467
|
+
"target = Path(sys.argv[1])\n"
|
|
468
|
+
"target.mkdir(parents=True, exist_ok=True)\n"
|
|
469
|
+
"fixtures = files('dataforge') / 'fixtures'\n"
|
|
470
|
+
"shutil.copyfile(fixtures / 'hospital_10rows.csv', target / 'hospital_10rows.csv')\n"
|
|
471
|
+
"shutil.copyfile(fixtures / 'hospital_schema.yaml', target / 'hospital_schema.yaml')\n"
|
|
472
|
+
)
|
|
473
|
+
step, _result = _run_command(
|
|
474
|
+
"copy_packaged_fixtures",
|
|
475
|
+
[venv_python, "-c", script, smoke_dir],
|
|
476
|
+
cwd=cwd,
|
|
477
|
+
timeout_seconds=30,
|
|
478
|
+
)
|
|
479
|
+
return step
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def _append_step(steps: list[ReleaseGateStep], step: ReleaseGateStep) -> bool:
|
|
483
|
+
"""Append a step and return whether the gate can continue."""
|
|
484
|
+
steps.append(step)
|
|
485
|
+
return step.ok
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def run_release_gate(*, keep_artifacts: bool = False) -> ReleaseGateReport:
|
|
489
|
+
"""Run the complete local release gate."""
|
|
490
|
+
project_root = _project_root()
|
|
491
|
+
steps: list[ReleaseGateStep] = []
|
|
492
|
+
artifact_hashes: dict[str, str] = {}
|
|
493
|
+
|
|
494
|
+
with tempfile.TemporaryDirectory(prefix="dataforge-release-gate-") as temp_name:
|
|
495
|
+
temp_root = Path(temp_name)
|
|
496
|
+
dist_dir = temp_root / "dist"
|
|
497
|
+
wheelhouse_dir = temp_root / "wheelhouse"
|
|
498
|
+
smoke_dir = temp_root / "smoke"
|
|
499
|
+
venv_path = temp_root / ".venv"
|
|
500
|
+
dist_dir.mkdir()
|
|
501
|
+
wheelhouse_dir.mkdir()
|
|
502
|
+
smoke_dir.mkdir()
|
|
503
|
+
|
|
504
|
+
preflight_commands: list[tuple[str, list[str | os.PathLike[str]], int]] = [
|
|
505
|
+
(
|
|
506
|
+
"release_doctor_core",
|
|
507
|
+
[sys.executable, "-m", "dataforge.release.doctor", "--core", "--json"],
|
|
508
|
+
60,
|
|
509
|
+
),
|
|
510
|
+
(
|
|
511
|
+
"readme_truth",
|
|
512
|
+
[sys.executable, "scripts/ci/readme_truth.py"],
|
|
513
|
+
60,
|
|
514
|
+
),
|
|
515
|
+
]
|
|
516
|
+
for name, command, timeout_seconds in preflight_commands:
|
|
517
|
+
step, _result = _run_command(
|
|
518
|
+
name,
|
|
519
|
+
command,
|
|
520
|
+
cwd=project_root,
|
|
521
|
+
timeout_seconds=timeout_seconds,
|
|
522
|
+
)
|
|
523
|
+
if not _append_step(steps, step):
|
|
524
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
525
|
+
|
|
526
|
+
step, _result = _run_command(
|
|
527
|
+
"build_sdist_and_wheel",
|
|
528
|
+
[sys.executable, "-m", "build", "--sdist", "--wheel", "--outdir", dist_dir],
|
|
529
|
+
cwd=project_root,
|
|
530
|
+
timeout_seconds=180,
|
|
531
|
+
)
|
|
532
|
+
if not _append_step(steps, step):
|
|
533
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
534
|
+
|
|
535
|
+
try:
|
|
536
|
+
wheel_path = _find_single_artifact(dist_dir, f"{PACKAGE_NAME}-*.whl")
|
|
537
|
+
sdist_path = _find_single_artifact(dist_dir, f"{PACKAGE_NAME}-*.tar.gz")
|
|
538
|
+
except ReleaseGateError as exc:
|
|
539
|
+
steps.append(
|
|
540
|
+
ReleaseGateStep(
|
|
541
|
+
name="find_built_artifacts",
|
|
542
|
+
ok=False,
|
|
543
|
+
detail=str(exc),
|
|
544
|
+
metadata={"dist_dir": str(dist_dir)},
|
|
545
|
+
)
|
|
546
|
+
)
|
|
547
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
548
|
+
|
|
549
|
+
artifact_hashes[wheel_path.name] = _file_sha256(wheel_path)
|
|
550
|
+
artifact_hashes[sdist_path.name] = _file_sha256(sdist_path)
|
|
551
|
+
if not _append_step(steps, _audit_wheel_contents(wheel_path)):
|
|
552
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
553
|
+
if not _append_step(steps, _audit_sdist_contents(sdist_path)):
|
|
554
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
555
|
+
|
|
556
|
+
shutil.copy2(wheel_path, wheelhouse_dir / wheel_path.name)
|
|
557
|
+
step, _result = _run_command(
|
|
558
|
+
"build_dependency_wheelhouse",
|
|
559
|
+
[sys.executable, "-m", "pip", "download", "--dest", wheelhouse_dir, wheel_path],
|
|
560
|
+
cwd=project_root,
|
|
561
|
+
timeout_seconds=180,
|
|
562
|
+
)
|
|
563
|
+
if not _append_step(steps, step):
|
|
564
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
565
|
+
|
|
566
|
+
step, _result = _run_command(
|
|
567
|
+
"create_clean_venv",
|
|
568
|
+
[sys.executable, "-m", "venv", venv_path],
|
|
569
|
+
cwd=temp_root,
|
|
570
|
+
timeout_seconds=90,
|
|
571
|
+
)
|
|
572
|
+
if not _append_step(steps, step):
|
|
573
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
574
|
+
|
|
575
|
+
venv_python = _python_in_venv(venv_path)
|
|
576
|
+
dataforge = _script_in_venv(venv_path, "dataforge")
|
|
577
|
+
dataforge15 = _script_in_venv(venv_path, "dataforge15")
|
|
578
|
+
step, _result = _run_command(
|
|
579
|
+
"offline_wheel_install",
|
|
580
|
+
[
|
|
581
|
+
venv_python,
|
|
582
|
+
"-m",
|
|
583
|
+
"pip",
|
|
584
|
+
"install",
|
|
585
|
+
"--no-index",
|
|
586
|
+
"--find-links",
|
|
587
|
+
wheelhouse_dir,
|
|
588
|
+
PACKAGE_NAME,
|
|
589
|
+
],
|
|
590
|
+
cwd=temp_root,
|
|
591
|
+
timeout_seconds=180,
|
|
592
|
+
)
|
|
593
|
+
if not _append_step(steps, step):
|
|
594
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
595
|
+
|
|
596
|
+
import_script = (
|
|
597
|
+
"import importlib.util\n"
|
|
598
|
+
"from pathlib import Path\n"
|
|
599
|
+
f"project_root = Path({str(project_root)!r}).resolve()\n"
|
|
600
|
+
"dataforge_spec = importlib.util.find_spec('dataforge')\n"
|
|
601
|
+
"assert dataforge_spec is not None\n"
|
|
602
|
+
"origin = Path(str(dataforge_spec.origin)).resolve()\n"
|
|
603
|
+
"assert project_root not in (origin, *origin.parents), origin\n"
|
|
604
|
+
"assert importlib.util.find_spec('dataforge') is not None\n"
|
|
605
|
+
"assert importlib.util.find_spec('data_quality_env') is None\n"
|
|
606
|
+
)
|
|
607
|
+
step, _result = _run_command(
|
|
608
|
+
"installed_import_boundaries",
|
|
609
|
+
[venv_python, "-c", import_script],
|
|
610
|
+
cwd=temp_root,
|
|
611
|
+
timeout_seconds=30,
|
|
612
|
+
)
|
|
613
|
+
if not _append_step(steps, step):
|
|
614
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
615
|
+
|
|
616
|
+
for cli_name, cli_path in (("dataforge", dataforge), ("dataforge15", dataforge15)):
|
|
617
|
+
step, _result = _run_command(
|
|
618
|
+
f"{cli_name}_cli_alias",
|
|
619
|
+
[cli_path, "--version"],
|
|
620
|
+
cwd=temp_root,
|
|
621
|
+
timeout_seconds=30,
|
|
622
|
+
)
|
|
623
|
+
if not _append_step(steps, step):
|
|
624
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
625
|
+
|
|
626
|
+
step, _result = _run_command(
|
|
627
|
+
"installed_release_doctor_core",
|
|
628
|
+
[dataforge, "release", "doctor", "--core", "--json"],
|
|
629
|
+
cwd=temp_root,
|
|
630
|
+
timeout_seconds=60,
|
|
631
|
+
)
|
|
632
|
+
if not _append_step(steps, step):
|
|
633
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
634
|
+
|
|
635
|
+
if not _append_step(steps, _copy_packaged_fixtures(venv_python, smoke_dir, cwd=temp_root)):
|
|
636
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
637
|
+
|
|
638
|
+
step, _result = _run_command(
|
|
639
|
+
"packaged_fixture_alias_profile",
|
|
640
|
+
[
|
|
641
|
+
dataforge,
|
|
642
|
+
"profile",
|
|
643
|
+
"fixtures/hospital_10rows.csv",
|
|
644
|
+
"--schema",
|
|
645
|
+
"fixtures/hospital_schema.yaml",
|
|
646
|
+
"--json",
|
|
647
|
+
],
|
|
648
|
+
cwd=temp_root,
|
|
649
|
+
timeout_seconds=30,
|
|
650
|
+
)
|
|
651
|
+
if not _append_step(steps, step):
|
|
652
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
653
|
+
|
|
654
|
+
source_path = smoke_dir / "hospital_10rows.csv"
|
|
655
|
+
schema_path = smoke_dir / "hospital_schema.yaml"
|
|
656
|
+
constraints_path = smoke_dir / "constraints.json"
|
|
657
|
+
original_sha256 = _file_sha256(source_path)
|
|
658
|
+
lifecycle_commands: list[tuple[str, list[str | os.PathLike[str]]]] = [
|
|
659
|
+
(
|
|
660
|
+
"smoke_profile",
|
|
661
|
+
[dataforge, "profile", source_path, "--schema", schema_path, "--json"],
|
|
662
|
+
),
|
|
663
|
+
(
|
|
664
|
+
"smoke_profile_constraints_artifact",
|
|
665
|
+
[
|
|
666
|
+
dataforge,
|
|
667
|
+
"profile",
|
|
668
|
+
source_path,
|
|
669
|
+
"--constraints-out",
|
|
670
|
+
constraints_path,
|
|
671
|
+
"--json",
|
|
672
|
+
],
|
|
673
|
+
),
|
|
674
|
+
(
|
|
675
|
+
"smoke_constraints_review_no_tui",
|
|
676
|
+
[dataforge, "constraints", "review", constraints_path, "--no-tui", "--json"],
|
|
677
|
+
),
|
|
678
|
+
(
|
|
679
|
+
"smoke_repair_dry_run",
|
|
680
|
+
[
|
|
681
|
+
dataforge,
|
|
682
|
+
"repair",
|
|
683
|
+
source_path,
|
|
684
|
+
"--schema",
|
|
685
|
+
schema_path,
|
|
686
|
+
"--dry-run",
|
|
687
|
+
"--json",
|
|
688
|
+
],
|
|
689
|
+
),
|
|
690
|
+
(
|
|
691
|
+
"smoke_watch_once",
|
|
692
|
+
[dataforge, "watch", source_path, "--schema", schema_path, "--once", "--json"],
|
|
693
|
+
),
|
|
694
|
+
]
|
|
695
|
+
for name, command in lifecycle_commands:
|
|
696
|
+
step, _result = _run_command(name, command, cwd=temp_root, timeout_seconds=30)
|
|
697
|
+
if not _append_step(steps, step):
|
|
698
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
699
|
+
|
|
700
|
+
step, warehouse_dry_run_result = _run_command(
|
|
701
|
+
"smoke_warehouse_cloud_dry_run",
|
|
702
|
+
[
|
|
703
|
+
dataforge,
|
|
704
|
+
"repair",
|
|
705
|
+
"warehouse://snowflake?relation=PUBLIC.HOSPITALS&row_id=id",
|
|
706
|
+
"--dry-run",
|
|
707
|
+
"--json",
|
|
708
|
+
],
|
|
709
|
+
cwd=temp_root,
|
|
710
|
+
timeout_seconds=30,
|
|
711
|
+
)
|
|
712
|
+
if not _append_step(steps, step):
|
|
713
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
714
|
+
if not _append_step(steps, _warehouse_dry_run_contract_step(warehouse_dry_run_result)):
|
|
715
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
716
|
+
|
|
717
|
+
step, apply_result = _run_command(
|
|
718
|
+
"smoke_repair_apply",
|
|
719
|
+
[dataforge, "repair", source_path, "--schema", schema_path, "--apply", "--json"],
|
|
720
|
+
cwd=temp_root,
|
|
721
|
+
timeout_seconds=30,
|
|
722
|
+
)
|
|
723
|
+
if not _append_step(steps, step):
|
|
724
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
725
|
+
try:
|
|
726
|
+
apply_payload = json.loads(apply_result.stdout)
|
|
727
|
+
txn_id = str(apply_payload["receipt"]["txn_id"])
|
|
728
|
+
except (KeyError, TypeError, json.JSONDecodeError) as exc:
|
|
729
|
+
steps.append(
|
|
730
|
+
ReleaseGateStep(
|
|
731
|
+
name="smoke_apply_receipt_contract",
|
|
732
|
+
ok=False,
|
|
733
|
+
detail=f"Could not read txn_id from apply receipt: {exc}",
|
|
734
|
+
metadata={"stdout_tail": _tail(apply_result.stdout)},
|
|
735
|
+
)
|
|
736
|
+
)
|
|
737
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
738
|
+
|
|
739
|
+
audit_command: list[str | os.PathLike[str]] = [
|
|
740
|
+
dataforge,
|
|
741
|
+
"audit",
|
|
742
|
+
txn_id,
|
|
743
|
+
"--search-root",
|
|
744
|
+
smoke_dir,
|
|
745
|
+
"--json",
|
|
746
|
+
]
|
|
747
|
+
step, audit_result = _run_command(
|
|
748
|
+
"smoke_audit_applied",
|
|
749
|
+
audit_command,
|
|
750
|
+
cwd=temp_root,
|
|
751
|
+
timeout_seconds=30,
|
|
752
|
+
)
|
|
753
|
+
if not _append_step(steps, step):
|
|
754
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
755
|
+
if not _append_step(
|
|
756
|
+
steps,
|
|
757
|
+
_json_step(
|
|
758
|
+
"smoke_audit_applied_contract", audit_result, expected={"verdict": "verified"}
|
|
759
|
+
),
|
|
760
|
+
):
|
|
761
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
762
|
+
|
|
763
|
+
step, revert_result = _run_command(
|
|
764
|
+
"smoke_revert",
|
|
765
|
+
[dataforge, "revert", txn_id, "--search-root", smoke_dir, "--json"],
|
|
766
|
+
cwd=temp_root,
|
|
767
|
+
timeout_seconds=30,
|
|
768
|
+
)
|
|
769
|
+
if not _append_step(steps, step):
|
|
770
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
771
|
+
if not _append_step(
|
|
772
|
+
steps,
|
|
773
|
+
_json_step("smoke_revert_contract", revert_result, expected={"txn_id": txn_id}),
|
|
774
|
+
):
|
|
775
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
776
|
+
|
|
777
|
+
final_sha256 = _file_sha256(source_path)
|
|
778
|
+
byte_identity_errors: list[str] = []
|
|
779
|
+
if final_sha256 != original_sha256:
|
|
780
|
+
byte_identity_errors.append(
|
|
781
|
+
f"Expected reverted SHA-256 {original_sha256}, found {final_sha256}."
|
|
782
|
+
)
|
|
783
|
+
if not _append_step(
|
|
784
|
+
steps,
|
|
785
|
+
ReleaseGateStep(
|
|
786
|
+
name="smoke_revert_byte_identity",
|
|
787
|
+
ok=not byte_identity_errors,
|
|
788
|
+
detail="reverted file bytes match original bytes"
|
|
789
|
+
if not byte_identity_errors
|
|
790
|
+
else "reverted bytes differ",
|
|
791
|
+
metadata={
|
|
792
|
+
"original_sha256": original_sha256,
|
|
793
|
+
"final_sha256": final_sha256,
|
|
794
|
+
"errors": byte_identity_errors,
|
|
795
|
+
},
|
|
796
|
+
),
|
|
797
|
+
):
|
|
798
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
799
|
+
|
|
800
|
+
step, final_audit_result = _run_command(
|
|
801
|
+
"smoke_audit_reverted",
|
|
802
|
+
audit_command,
|
|
803
|
+
cwd=temp_root,
|
|
804
|
+
timeout_seconds=30,
|
|
805
|
+
)
|
|
806
|
+
if not _append_step(steps, step):
|
|
807
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
808
|
+
if not _append_step(
|
|
809
|
+
steps,
|
|
810
|
+
_json_step(
|
|
811
|
+
"smoke_audit_reverted_contract",
|
|
812
|
+
final_audit_result,
|
|
813
|
+
expected={"verdict": "verified"},
|
|
814
|
+
),
|
|
815
|
+
):
|
|
816
|
+
return ReleaseGateReport(ok=False, steps=steps, artifact_sha256=artifact_hashes)
|
|
817
|
+
|
|
818
|
+
if keep_artifacts:
|
|
819
|
+
kept_dir = project_root / "dist" / "release-gate-latest"
|
|
820
|
+
kept_dir.parent.mkdir(exist_ok=True)
|
|
821
|
+
if kept_dir.exists():
|
|
822
|
+
shutil.rmtree(kept_dir)
|
|
823
|
+
shutil.copytree(temp_root, kept_dir)
|
|
824
|
+
steps.append(
|
|
825
|
+
ReleaseGateStep(
|
|
826
|
+
name="keep_artifacts",
|
|
827
|
+
ok=True,
|
|
828
|
+
detail="release gate artifacts copied",
|
|
829
|
+
metadata={"path": str(kept_dir)},
|
|
830
|
+
)
|
|
831
|
+
)
|
|
832
|
+
|
|
833
|
+
return ReleaseGateReport(
|
|
834
|
+
ok=all(step.ok for step in steps), steps=steps, artifact_sha256=artifact_hashes
|
|
835
|
+
)
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
def main(argv: list[str] | None = None) -> int:
|
|
839
|
+
"""Script entrypoint used by CI and local release work."""
|
|
840
|
+
import argparse
|
|
841
|
+
|
|
842
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
843
|
+
parser.add_argument("--json", action="store_true", help="Print JSON instead of text.")
|
|
844
|
+
parser.add_argument(
|
|
845
|
+
"--keep-artifacts",
|
|
846
|
+
action="store_true",
|
|
847
|
+
help="Copy the temporary gate workspace to dist/release-gate-latest.",
|
|
848
|
+
)
|
|
849
|
+
args = parser.parse_args(argv)
|
|
850
|
+
report = run_release_gate(keep_artifacts=args.keep_artifacts)
|
|
851
|
+
if args.json:
|
|
852
|
+
print(json.dumps(report.to_dict(), indent=2, sort_keys=True))
|
|
853
|
+
else:
|
|
854
|
+
for step in report.steps:
|
|
855
|
+
status = "ok" if step.ok else "fail"
|
|
856
|
+
print(f"{status:4} {step.name}: {step.detail}")
|
|
857
|
+
return 0 if report.ok else 1
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
if __name__ == "__main__":
|
|
861
|
+
raise SystemExit(main(sys.argv[1:]))
|