dw-harness 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dw_core/__init__.py +11 -0
- dw_core/batch.py +114 -0
- dw_core/check.py +302 -0
- dw_core/cli.py +437 -0
- dw_core/config.py +195 -0
- dw_core/contracts.py +325 -0
- dw_core/data/__init__.py +1 -0
- dw_core/data/dialects/hive.json +8 -0
- dw_core/data/dialects/odps.json +8 -0
- dw_core/data/dialects/spark.json +8 -0
- dw_core/data/dialects/starrocks.json +8 -0
- dw_core/data/template/.claude/agents/data-comparator.md +32 -0
- dw_core/data/template/.claude/agents/data-quality-checker.md +34 -0
- dw_core/data/template/.claude/agents/dw-explorer.md +25 -0
- dw_core/data/template/.claude/agents/sql-validator.md +30 -0
- dw_core/data/template/.claude/dw/harness.json +40 -0
- dw_core/data/template/.claude/dw/mcp/server.py +244 -0
- dw_core/data/template/.claude/dw/policy_exceptions.json +1 -0
- dw_core/data/template/.claude/dw/state.json +8 -0
- dw_core/data/template/.claude/dw/v1_hashes.json +21 -0
- dw_core/data/template/.claude/dw_core/__init__.py +11 -0
- dw_core/data/template/.claude/dw_core/batch.py +114 -0
- dw_core/data/template/.claude/dw_core/check.py +302 -0
- dw_core/data/template/.claude/dw_core/cli.py +437 -0
- dw_core/data/template/.claude/dw_core/config.py +195 -0
- dw_core/data/template/.claude/dw_core/contracts.py +325 -0
- dw_core/data/template/.claude/dw_core/data/__init__.py +1 -0
- dw_core/data/template/.claude/dw_core/data/dialects/hive.json +8 -0
- dw_core/data/template/.claude/dw_core/data/dialects/odps.json +8 -0
- dw_core/data/template/.claude/dw_core/data/dialects/spark.json +8 -0
- dw_core/data/template/.claude/dw_core/data/dialects/starrocks.json +8 -0
- dw_core/data/template/.claude/dw_core/ddl.py +247 -0
- dw_core/data/template/.claude/dw_core/demo.py +303 -0
- dw_core/data/template/.claude/dw_core/doctor.py +193 -0
- dw_core/data/template/.claude/dw_core/dqc.py +90 -0
- dw_core/data/template/.claude/dw_core/evidence.py +178 -0
- dw_core/data/template/.claude/dw_core/guard.py +387 -0
- dw_core/data/template/.claude/dw_core/impact.py +193 -0
- dw_core/data/template/.claude/dw_core/mcp_selftest.py +61 -0
- dw_core/data/template/.claude/dw_core/migrate.py +363 -0
- dw_core/data/template/.claude/dw_core/plan.py +98 -0
- dw_core/data/template/.claude/dw_core/reconcile.py +492 -0
- dw_core/data/template/.claude/dw_core/rules.py +247 -0
- dw_core/data/template/.claude/dw_core/sqlsplit.py +215 -0
- dw_core/data/template/.claude/dw_core/state.py +83 -0
- dw_core/data/template/.claude/hooks/_bootstrap.py +77 -0
- dw_core/data/template/.claude/hooks/guard_shell.py +43 -0
- dw_core/data/template/.claude/hooks/guard_write.py +44 -0
- dw_core/data/template/.claude/hooks/handoff.py +40 -0
- dw_core/data/template/.claude/hooks/post_compact.py +36 -0
- dw_core/data/template/.claude/hooks/session_context.py +27 -0
- dw_core/data/template/.claude/hooks/statusline.py +60 -0
- dw_core/data/template/.claude/hooks/validate_post.py +49 -0
- dw_core/data/template/.claude/output-styles/dw-engineer.md +12 -0
- dw_core/data/template/.claude/rules/etl.md +12 -0
- dw_core/data/template/.claude/rules/sql-style.md +13 -0
- dw_core/data/template/.claude/rules/sr.md +12 -0
- dw_core/data/template/.claude/settings.json +93 -0
- dw_core/data/template/.claude/skills/dw-backfill/SKILL.md +29 -0
- dw_core/data/template/.claude/skills/dw-bootstrap/SKILL.md +32 -0
- dw_core/data/template/.claude/skills/dw-check/SKILL.md +32 -0
- dw_core/data/template/.claude/skills/dw-contract/SKILL.md +35 -0
- dw_core/data/template/.claude/skills/dw-data-comparison/SKILL.md +35 -0
- dw_core/data/template/.claude/skills/dw-dqc/SKILL.md +31 -0
- dw_core/data/template/.claude/skills/dw-etl/SKILL.md +41 -0
- dw_core/data/template/.claude/skills/dw-next/SKILL.md +34 -0
- dw_core/data/template/.claude/skills/dw-performance-optimization/SKILL.md +26 -0
- dw_core/data/template/.claude/skills/dw-requirement-analysis/SKILL.md +29 -0
- dw_core/data/template/.claude/skills/dw-review/SKILL.md +31 -0
- dw_core/data/template/.claude/skills/dw-self-test/SKILL.md +29 -0
- dw_core/data/template/.claude/skills/dw-sql-reference/SKILL.md +44 -0
- dw_core/data/template/.claude/skills/dw-sr/SKILL.md +33 -0
- dw_core/data/template/.claude/skills/dw-technical-design/SKILL.md +33 -0
- dw_core/data/template/AGENTS.md +44 -0
- dw_core/data/template/CLAUDE.md +16 -0
- dw_core/data/template/contracts/contract.example.json +22 -0
- dw_core/ddl.py +247 -0
- dw_core/demo.py +303 -0
- dw_core/doctor.py +193 -0
- dw_core/dqc.py +90 -0
- dw_core/evidence.py +178 -0
- dw_core/guard.py +387 -0
- dw_core/impact.py +193 -0
- dw_core/mcp_selftest.py +61 -0
- dw_core/migrate.py +363 -0
- dw_core/plan.py +98 -0
- dw_core/reconcile.py +492 -0
- dw_core/rules.py +247 -0
- dw_core/sqlsplit.py +215 -0
- dw_core/state.py +83 -0
- dw_harness-2.0.0.dist-info/METADATA +724 -0
- dw_harness-2.0.0.dist-info/RECORD +96 -0
- dw_harness-2.0.0.dist-info/WHEEL +5 -0
- dw_harness-2.0.0.dist-info/entry_points.txt +2 -0
- dw_harness-2.0.0.dist-info/licenses/LICENSE +21 -0
- dw_harness-2.0.0.dist-info/top_level.txt +1 -0
dw_core/__init__.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
"""dw-harness core: shared by CLI, Claude Code hooks, CI and MCP.
|
|
2
|
+
|
|
3
|
+
Standard-library only at this layer. Optional capabilities (YAML contracts,
|
|
4
|
+
deep SQL parsing, database clients) degrade gracefully and are reported as
|
|
5
|
+
"unavailable" instead of raising.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "2.0.0"
|
|
9
|
+
|
|
10
|
+
EVIDENCE_SCHEMA = "dw-evidence/2"
|
|
11
|
+
CONTRACT_SCHEMA = "dw-contract/2"
|
dw_core/batch.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Resolve a Git changeset to independently checked warehouse tables."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import subprocess
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from .contracts import ContractError, load_contract, read_baseline, safe_git_ref
|
|
9
|
+
from .ddl import parse_create_table
|
|
10
|
+
from .guard import _target_table
|
|
11
|
+
from .sqlsplit import scan_sql
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _is_contract(path: Path) -> bool:
|
|
15
|
+
return (path.suffix.lower() in {".json", ".yaml", ".yml"}
|
|
16
|
+
and ("contracts" in path.parent.parts or path.stem.startswith("contract"))
|
|
17
|
+
and "schema" not in path.parent.parts
|
|
18
|
+
and not path.name.endswith((".example.json", ".schema.json")))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _git_paths(root: Path, base: str):
|
|
22
|
+
# literal argument list (no shell); the revision is validated first so a
|
|
23
|
+
# user-controlled value can never be parsed by git as an option
|
|
24
|
+
output = subprocess.run(
|
|
25
|
+
["git", "ls-tree", "-r", "--name-only", "-z", "--", safe_git_ref(base)],
|
|
26
|
+
cwd=root, capture_output=True, check=True, timeout=30)
|
|
27
|
+
return set(output.stdout.decode("utf-8").split("\0"))
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def resolve_table_checks(root: Path, base: str, paths, table: str = ""):
|
|
31
|
+
"""Group changed SQL, contracts and DQC by target table; surface ambiguities."""
|
|
32
|
+
existing = _git_paths(root, base)
|
|
33
|
+
targets = {}
|
|
34
|
+
unresolved = []
|
|
35
|
+
changed_contracts = {}
|
|
36
|
+
candidates = list(dict.fromkeys([
|
|
37
|
+
*(p for p in paths if _is_contract(p.relative_to(root))),
|
|
38
|
+
*(p for p in (root / "contracts").rglob("*") if p.is_file() and _is_contract(p.relative_to(root))),
|
|
39
|
+
*(p for p in root.glob("contract.*") if p.is_file() and _is_contract(p.relative_to(root))),
|
|
40
|
+
]))
|
|
41
|
+
contracts = {}
|
|
42
|
+
for path in candidates:
|
|
43
|
+
rel = path.relative_to(root).as_posix()
|
|
44
|
+
baseline = f"git:{base}:{rel}" if rel in existing else None
|
|
45
|
+
try:
|
|
46
|
+
if path.is_file():
|
|
47
|
+
name = load_contract(path)[0]["table"]
|
|
48
|
+
elif baseline:
|
|
49
|
+
name = read_baseline(baseline, root)[0]["table"]
|
|
50
|
+
else:
|
|
51
|
+
raise ValueError("contract deleted")
|
|
52
|
+
if not name:
|
|
53
|
+
raise ValueError("no target table")
|
|
54
|
+
contracts.setdefault(name, []).append((path, baseline))
|
|
55
|
+
if path in paths:
|
|
56
|
+
changed_contracts.setdefault(name, []).append((path, baseline))
|
|
57
|
+
if not table or table == name:
|
|
58
|
+
targets.setdefault(name, {})
|
|
59
|
+
except (OSError, ValueError, TypeError, KeyError, AttributeError, ContractError,
|
|
60
|
+
subprocess.SubprocessError):
|
|
61
|
+
if path in paths:
|
|
62
|
+
unresolved.append(rel)
|
|
63
|
+
|
|
64
|
+
for path in paths:
|
|
65
|
+
rel = path.relative_to(root).as_posix()
|
|
66
|
+
if not path.is_file():
|
|
67
|
+
if path.suffix.lower() == ".sql" or _is_contract(path) or path.name.startswith("dqc_"):
|
|
68
|
+
unresolved.append(rel + " (deleted)")
|
|
69
|
+
continue
|
|
70
|
+
if path.suffix.lower() == ".sql":
|
|
71
|
+
sql = path.read_text(encoding="utf-8-sig", errors="replace")
|
|
72
|
+
ddls = parse_create_table(sql)
|
|
73
|
+
names = {ddl.table for ddl in ddls}
|
|
74
|
+
names.update(filter(None, (_target_table(st.code) for st in scan_sql(sql)[0])))
|
|
75
|
+
if not names and table:
|
|
76
|
+
names.add(table)
|
|
77
|
+
if not names:
|
|
78
|
+
unresolved.append(rel)
|
|
79
|
+
for name in sorted(names):
|
|
80
|
+
matches = [key for key in contracts if key == name or
|
|
81
|
+
("." not in name and key.rsplit(".", 1)[-1] == name)]
|
|
82
|
+
if len(matches) > 1:
|
|
83
|
+
unresolved.append(rel + " (ambiguous target " + name + ")")
|
|
84
|
+
continue
|
|
85
|
+
name = matches[0] if matches else name
|
|
86
|
+
if table and table != name:
|
|
87
|
+
continue
|
|
88
|
+
options = targets.setdefault(name, {})
|
|
89
|
+
options.setdefault("sql_files", []).append(path)
|
|
90
|
+
if any(ddl.table in {name, name.rsplit(".", 1)[-1]} for ddl in ddls):
|
|
91
|
+
if "ddl_path" in options and options["ddl_path"] != path:
|
|
92
|
+
unresolved.append(rel + " (multiple DDLs for " + name + ")")
|
|
93
|
+
else:
|
|
94
|
+
options["ddl_path"] = path
|
|
95
|
+
elif path.name.startswith("dqc_") and path.suffix.lower() == ".json":
|
|
96
|
+
try:
|
|
97
|
+
raw = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
98
|
+
name = raw.get("table") or raw.get("table_name")
|
|
99
|
+
if not isinstance(name, str) or not name:
|
|
100
|
+
raise ValueError("no target table")
|
|
101
|
+
if not table or table == name:
|
|
102
|
+
targets.setdefault(name, {})["dqc_path"] = path
|
|
103
|
+
except (OSError, ValueError, AttributeError):
|
|
104
|
+
unresolved.append(rel)
|
|
105
|
+
|
|
106
|
+
if table:
|
|
107
|
+
targets.setdefault(table, {})
|
|
108
|
+
for name, options in targets.items():
|
|
109
|
+
choices = changed_contracts.get(name) or contracts.get(name, [])
|
|
110
|
+
if len(choices) == 1:
|
|
111
|
+
options["contract_path"], options["baseline"] = choices[0]
|
|
112
|
+
elif choices:
|
|
113
|
+
unresolved.append(f"{name} (multiple contracts)")
|
|
114
|
+
return dict(sorted(targets.items())), sorted(set(unresolved))
|
dw_core/check.py
ADDED
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
"""dw check: four-state acceptance report (pass / fail / unverified / n-a).
|
|
2
|
+
|
|
3
|
+
Exit codes: 0 policy allows merge, 1 known failure, 2 required evidence
|
|
4
|
+
unverified. Failures outrank missing evidence; local mode may allow
|
|
5
|
+
unverified items but never rewrites their status.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import subprocess
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Dict, List, Optional
|
|
15
|
+
|
|
16
|
+
from . import __version__
|
|
17
|
+
from .config import HarnessConfig
|
|
18
|
+
from .contracts import (ContractError, crosscheck_ddl, diff_contracts, read_baseline,
|
|
19
|
+
load_contract, load_decisions,
|
|
20
|
+
unapproved_breaking)
|
|
21
|
+
from .ddl import parse_create_table
|
|
22
|
+
from .dqc import check_rules
|
|
23
|
+
from .evidence import (evidence_policy, file_digest, load_evidence, path_binding,
|
|
24
|
+
verify_evidence)
|
|
25
|
+
from .rules import lint_sql
|
|
26
|
+
|
|
27
|
+
STATUS_ORDER = {"fail": 0, "unverified": 1, "pass": 2, "not_applicable": 3}
|
|
28
|
+
STATUS_LABEL = {"pass": "PASS", "fail": "FAIL", "unverified": "UNVERIFIED",
|
|
29
|
+
"not_applicable": "N/A"}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class CheckItem:
|
|
34
|
+
name: str
|
|
35
|
+
status: str
|
|
36
|
+
reason: str = ""
|
|
37
|
+
evidence: str = ""
|
|
38
|
+
|
|
39
|
+
def to_dict(self) -> Dict:
|
|
40
|
+
return {"name": self.name, "status": self.status,
|
|
41
|
+
"reason": self.reason, "evidence": self.evidence}
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass
|
|
45
|
+
class CheckReport:
|
|
46
|
+
table: str
|
|
47
|
+
environment: str
|
|
48
|
+
partition: str
|
|
49
|
+
items: List[CheckItem] = field(default_factory=list)
|
|
50
|
+
mode: str = "strict" # strict | local
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def overall(self) -> str:
|
|
54
|
+
worst = "pass"
|
|
55
|
+
for item in self.items:
|
|
56
|
+
if item.status == "not_applicable":
|
|
57
|
+
continue
|
|
58
|
+
if STATUS_ORDER[item.status] < STATUS_ORDER[worst]:
|
|
59
|
+
worst = item.status
|
|
60
|
+
return worst
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def exit_code(self) -> int:
|
|
64
|
+
if any(i.status == "fail" for i in self.items):
|
|
65
|
+
return 1
|
|
66
|
+
if self.mode == "strict" and any(i.status == "unverified" for i in self.items):
|
|
67
|
+
return 2
|
|
68
|
+
return 0
|
|
69
|
+
|
|
70
|
+
def to_dict(self) -> Dict:
|
|
71
|
+
return {
|
|
72
|
+
"schema": "dw-check/1",
|
|
73
|
+
"tool_version": __version__,
|
|
74
|
+
"generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
|
75
|
+
"table": self.table, "environment": self.environment,
|
|
76
|
+
"partition": self.partition, "mode": self.mode,
|
|
77
|
+
"overall": self.overall, "exit_code": self.exit_code,
|
|
78
|
+
"items": [i.to_dict() for i in self.items],
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
def to_markdown(self) -> str:
|
|
82
|
+
lines = [
|
|
83
|
+
f"# dw check — {self.table}",
|
|
84
|
+
"",
|
|
85
|
+
f"- environment: `{self.environment}` | partition: `{self.partition or '-'}` "
|
|
86
|
+
f"| mode: `{self.mode}` | overall: **{STATUS_LABEL[self.overall]}**",
|
|
87
|
+
"",
|
|
88
|
+
"| check | status | reason | evidence |",
|
|
89
|
+
"|---|---|---|---|",
|
|
90
|
+
]
|
|
91
|
+
for item in self.items:
|
|
92
|
+
reason = (item.reason or "-").replace("|", "\\|")
|
|
93
|
+
lines.append(f"| `{item.name}` | {STATUS_LABEL[item.status]} | {reason} | "
|
|
94
|
+
f"{item.evidence or '-'} |")
|
|
95
|
+
lines += ["", f"_dw-harness v{__version__}_"]
|
|
96
|
+
return "\n".join(lines) + "\n"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _item(name: str, status: str, reason: str = "", evidence: str = "") -> CheckItem:
|
|
100
|
+
return CheckItem(name, status, reason, evidence)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def run_checks(config: HarnessConfig, table: str, environment: str = "dev",
|
|
104
|
+
partition: str = "", sql_files: Optional[List[Path]] = None,
|
|
105
|
+
contract_path: Optional[Path] = None, ddl_path: Optional[Path] = None,
|
|
106
|
+
dqc_path: Optional[Path] = None, baseline: Optional[str] = None,
|
|
107
|
+
reconcile_spec: Optional[Path] = None,
|
|
108
|
+
mode: str = "strict") -> CheckReport:
|
|
109
|
+
report = CheckReport(table=table, environment=environment,
|
|
110
|
+
partition=partition, mode=mode)
|
|
111
|
+
root = config.root
|
|
112
|
+
inputs: Dict[str, str] = {"policy": evidence_policy(config), "contract-baseline": "none"}
|
|
113
|
+
|
|
114
|
+
# ---- lint changed SQL ------------------------------------------------
|
|
115
|
+
sql_files = sql_files or []
|
|
116
|
+
if sql_files:
|
|
117
|
+
errors = 0
|
|
118
|
+
details: List[str] = []
|
|
119
|
+
unreadable = []
|
|
120
|
+
for path in sql_files:
|
|
121
|
+
try:
|
|
122
|
+
inputs[f"sql:{path_binding(path, root)}"] = file_digest(path)
|
|
123
|
+
sql = path.read_text(encoding="utf-8-sig")
|
|
124
|
+
except (OSError, UnicodeError):
|
|
125
|
+
unreadable.append(str(path))
|
|
126
|
+
continue
|
|
127
|
+
dialect = config.resolve_dialect(str(path))
|
|
128
|
+
result = lint_sql(sql, dialect, config)
|
|
129
|
+
for issue in result.errors:
|
|
130
|
+
errors += 1
|
|
131
|
+
details.append(f"{path.name}:{issue.line} {issue.rule}")
|
|
132
|
+
if errors:
|
|
133
|
+
report.items.append(_item("lint-sql", "fail",
|
|
134
|
+
f"{errors} error(s): {'; '.join(details[:5])}"))
|
|
135
|
+
elif unreadable:
|
|
136
|
+
report.items.append(_item("lint-sql", "unverified", "cannot read: " + ", ".join(unreadable)))
|
|
137
|
+
else:
|
|
138
|
+
report.items.append(_item("lint-sql", "pass",
|
|
139
|
+
f"{len(sql_files)} file(s) clean (offline rules)"))
|
|
140
|
+
else:
|
|
141
|
+
report.items.append(_item("lint-sql", "not_applicable",
|
|
142
|
+
"no SQL files in this changeset"))
|
|
143
|
+
|
|
144
|
+
# ---- contract ---------------------------------------------------------
|
|
145
|
+
contract_path = contract_path or _find_unique(root, [
|
|
146
|
+
f"contracts/{table}.json", f"contracts/{table}.yaml", f"contracts/{table}.yml",
|
|
147
|
+
"contract.json", "contract.yaml", "contract.yml"])
|
|
148
|
+
contract = None
|
|
149
|
+
if contract_path is None:
|
|
150
|
+
report.items.append(_item("contract-validate", "unverified",
|
|
151
|
+
"no contract found for this table; unmanaged tables need "
|
|
152
|
+
"explicit onboarding (docs/upgrade-from-v1.md)"))
|
|
153
|
+
else:
|
|
154
|
+
try:
|
|
155
|
+
contract, _ = load_contract(contract_path)
|
|
156
|
+
inputs["contract"] = file_digest(contract_path)
|
|
157
|
+
except Exception as exc: # noqa: BLE001 - report, don't crash acceptance
|
|
158
|
+
report.items.append(_item("contract-validate", "fail", f"contract unreadable: {exc}"))
|
|
159
|
+
contract = None
|
|
160
|
+
if contract is not None:
|
|
161
|
+
from .contracts import validate_contract
|
|
162
|
+
problems = validate_contract(contract)
|
|
163
|
+
if contract.get("table", "").lower() != table.lower():
|
|
164
|
+
problems.append(f"contract table {contract.get('table')!r} does not match {table}")
|
|
165
|
+
report.items.append(
|
|
166
|
+
_item("contract-validate", "fail" if problems else "pass",
|
|
167
|
+
"; ".join(problems) if problems else "schema valid",
|
|
168
|
+
evidence=str(contract_path)))
|
|
169
|
+
# DDL cross-check (provable attributes only)
|
|
170
|
+
ddl_path = ddl_path or _find_unique(root, [f"ddl_{table.split('.')[-1]}.sql",
|
|
171
|
+
f"**/ddl_{table.split('.')[-1]}.sql"])
|
|
172
|
+
if ddl_path is None or not ddl_path.is_file():
|
|
173
|
+
report.items.append(_item("ddl-vs-contract", "unverified",
|
|
174
|
+
"no DDL found in changeset"))
|
|
175
|
+
else:
|
|
176
|
+
tables = parse_create_table(ddl_path.read_text(encoding="utf-8-sig", errors="replace"),
|
|
177
|
+
config.resolve_dialect(str(ddl_path)))
|
|
178
|
+
names = {table.lower(), table.rsplit(".", 1)[-1].lower()}
|
|
179
|
+
tables = [ddl for ddl in tables if ddl.table.lower() in names]
|
|
180
|
+
if len(tables) != 1:
|
|
181
|
+
report.items.append(_item("ddl-vs-contract", "unverified",
|
|
182
|
+
"DDL file has no unique CREATE TABLE for the requested table"))
|
|
183
|
+
else:
|
|
184
|
+
inputs[f"ddl:{path_binding(ddl_path, root)}"] = file_digest(ddl_path)
|
|
185
|
+
findings = crosscheck_ddl(contract, tables[0])
|
|
186
|
+
hard = [f for f in findings if f["finding"] != "missing_in_contract"]
|
|
187
|
+
report.items.append(_item(
|
|
188
|
+
"ddl-vs-contract", "fail" if hard else "pass",
|
|
189
|
+
"; ".join(f"{f['field']}: {f['note']}" for f in hard[:5]) if hard
|
|
190
|
+
else "provable attributes consistent; business semantics confirmed in review",
|
|
191
|
+
evidence=str(ddl_path)))
|
|
192
|
+
# contract diff vs baseline + migration decisions
|
|
193
|
+
if baseline is not None:
|
|
194
|
+
try:
|
|
195
|
+
base_raw, baseline_digest = read_baseline(baseline, root)
|
|
196
|
+
if base_raw.get("table") != contract.get("table"):
|
|
197
|
+
raise ValueError("baseline contract belongs to a different table")
|
|
198
|
+
changes = diff_contracts(contract, base_raw)
|
|
199
|
+
pending = unapproved_breaking(changes, contract, base_raw,
|
|
200
|
+
load_decisions(root, contract.get("contract_id", "")))
|
|
201
|
+
inputs["contract-baseline"] = baseline_digest
|
|
202
|
+
if pending:
|
|
203
|
+
report.items.append(_item(
|
|
204
|
+
"contract-diff", "fail",
|
|
205
|
+
f"breaking change(s) without recorded migration decision: "
|
|
206
|
+
f"{'; '.join(p['field'] + ' ' + p['type'] for p in pending[:5])}. "
|
|
207
|
+
f"Record via `dw contract decide` (docs/configuration.md)"))
|
|
208
|
+
else:
|
|
209
|
+
report.items.append(_item("contract-diff", "pass",
|
|
210
|
+
"no unapproved breaking changes vs baseline",
|
|
211
|
+
evidence=str(baseline)))
|
|
212
|
+
except (OSError, ValueError, subprocess.SubprocessError, ContractError, TypeError, AttributeError) as exc:
|
|
213
|
+
inputs["contract-baseline"] = "unreadable"
|
|
214
|
+
report.items.append(_item("contract-diff", "unverified",
|
|
215
|
+
f"baseline unreadable: {exc}"))
|
|
216
|
+
else:
|
|
217
|
+
report.items.append(_item("contract-diff", "not_applicable",
|
|
218
|
+
"no baseline requested for this check"))
|
|
219
|
+
|
|
220
|
+
# ---- DQC ---------------------------------------------------------------
|
|
221
|
+
dqc_path = dqc_path or _find_unique(root, [f"dqc_{table.split('.')[-1]}.json"])
|
|
222
|
+
if contract is None:
|
|
223
|
+
report.items.append(_item("dqc-config", "not_applicable", "no contract to derive rules from"))
|
|
224
|
+
elif dqc_path is None:
|
|
225
|
+
report.items.append(_item("dqc-config", "unverified",
|
|
226
|
+
"no DQC config file; generate via `dw dqc gen`"))
|
|
227
|
+
else:
|
|
228
|
+
try:
|
|
229
|
+
dqc_json = json.loads(dqc_path.read_text(encoding="utf-8-sig"))
|
|
230
|
+
problems = check_rules(dqc_json, contract)
|
|
231
|
+
inputs[f"dqc:{dqc_path.name}"] = file_digest(dqc_path)
|
|
232
|
+
report.items.append(_item("dqc-config", "fail" if problems else "pass",
|
|
233
|
+
"; ".join(problems[:5]) if problems
|
|
234
|
+
else f"{len(dqc_json.get('rules', []))} rules stable vs contract",
|
|
235
|
+
evidence=str(dqc_path)))
|
|
236
|
+
except (json.JSONDecodeError, OSError) as exc:
|
|
237
|
+
report.items.append(_item("dqc-config", "unverified", f"dqc config unreadable: {exc}"))
|
|
238
|
+
|
|
239
|
+
# Establish trust before following any paths carried inside a report.
|
|
240
|
+
from .reconcile import ReconcileError, ReconcileSpec
|
|
241
|
+
ev = load_evidence(root, f"reconcile-{table}")
|
|
242
|
+
ttl = int(config.data.get("evidence", {}).get("freshness_hours", 72))
|
|
243
|
+
fresh = verify_evidence(ev, {}, freshness_hours=ttl, strict=(mode == "strict"))
|
|
244
|
+
if fresh["state"] == "valid":
|
|
245
|
+
try:
|
|
246
|
+
names = [f"reconcile_{table}.json", f"reconcile_{table.rsplit('.', 1)[-1]}.json"]
|
|
247
|
+
spec_path = reconcile_spec or _find_unique(root, names)
|
|
248
|
+
if spec_path is None:
|
|
249
|
+
raise ValueError("no reconciliation spec selected; pass --reconcile-spec")
|
|
250
|
+
raw, spec = ReconcileSpec.load(spec_path)
|
|
251
|
+
requested = {"table": table, "environment": environment, "partition": partition}
|
|
252
|
+
scope = spec.evidence_scope(raw.get("table"))
|
|
253
|
+
if scope != requested or ev.get("scope") != requested:
|
|
254
|
+
raise ValueError("evidence scope does not match target table, environment and partition")
|
|
255
|
+
inputs["reconcile:spec"] = file_digest(spec_path)
|
|
256
|
+
for role, path in spec.file_inputs().items():
|
|
257
|
+
inputs[role] = file_digest(path)
|
|
258
|
+
current = {key: value for key, value in inputs.items()
|
|
259
|
+
if not key.startswith("dqc:")}
|
|
260
|
+
for role, raw_path in (ev.get("binding_paths") or {}).items():
|
|
261
|
+
if role.startswith(("sql:", "ddl:")) and role not in current:
|
|
262
|
+
path = root / Path(raw_path)
|
|
263
|
+
if path_binding(path, root) != role.partition(":")[2]:
|
|
264
|
+
raise ValueError(f"evidence file binding does not match role {role}")
|
|
265
|
+
current[role] = file_digest(path)
|
|
266
|
+
fresh = verify_evidence(ev, current, freshness_hours=ttl, strict=(mode == "strict"))
|
|
267
|
+
except (OSError, TypeError, ValueError, AttributeError, KeyError, ReconcileError) as exc:
|
|
268
|
+
fresh = {"state": "stale", "reason": str(exc)}
|
|
269
|
+
if ev is None:
|
|
270
|
+
report.items.append(_item("reconcile", "unverified",
|
|
271
|
+
"no reconciliation evidence for this table/partition"))
|
|
272
|
+
elif fresh["state"] == "untrusted":
|
|
273
|
+
report.items.append(_item("reconcile", "unverified",
|
|
274
|
+
f"evidence untrusted: {fresh['reason']}; "
|
|
275
|
+
f"re-run `dw reconcile --save-evidence` (deterministic producer)"))
|
|
276
|
+
elif fresh["state"] != "valid":
|
|
277
|
+
report.items.append(_item("reconcile", "unverified",
|
|
278
|
+
f"evidence stale: {fresh['reason']}; re-run `dw reconcile`"))
|
|
279
|
+
elif ev.get("result_summary", {}).get("status") == "pass":
|
|
280
|
+
report.items.append(_item("reconcile", "pass",
|
|
281
|
+
"key-level comparison within tolerance "
|
|
282
|
+
f"(producer={ev.get('producer', 'unknown')})",
|
|
283
|
+
evidence=f".claude/dw/evidence/reconcile-{table}.json"))
|
|
284
|
+
elif ev.get("result_summary", {}).get("status") == "fail":
|
|
285
|
+
report.items.append(_item("reconcile", "fail",
|
|
286
|
+
"recorded reconciliation did not pass",
|
|
287
|
+
evidence=f".claude/dw/evidence/reconcile-{table}.json"))
|
|
288
|
+
|
|
289
|
+
else:
|
|
290
|
+
report.items.append(_item("reconcile", "unverified", "invalid evidence result state"))
|
|
291
|
+
return report
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def _find_unique(root: Path, patterns: List[str]) -> Optional[Path]:
|
|
295
|
+
for pattern in patterns:
|
|
296
|
+
matches = sorted(root.rglob(pattern))
|
|
297
|
+
matches = [m for m in matches if ".git" not in m.parts and ".claude" not in m.parts]
|
|
298
|
+
if len(matches) == 1:
|
|
299
|
+
return matches[0]
|
|
300
|
+
if matches:
|
|
301
|
+
return None # ambiguous results require an explicit path
|
|
302
|
+
return None
|