dataforge-07 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. dataforge/__init__.py +204 -0
  2. dataforge/__main__.py +5 -0
  3. dataforge/agent/__init__.py +16 -0
  4. dataforge/agent/providers.py +259 -0
  5. dataforge/agent/scratchpad.py +183 -0
  6. dataforge/agent/tool_actions.py +343 -0
  7. dataforge/bench/__init__.py +31 -0
  8. dataforge/bench/core.py +426 -0
  9. dataforge/bench/groq_client.py +386 -0
  10. dataforge/bench/methods.py +443 -0
  11. dataforge/bench/report.py +309 -0
  12. dataforge/bench/runner.py +247 -0
  13. dataforge/causal/__init__.py +21 -0
  14. dataforge/causal/dag.py +174 -0
  15. dataforge/causal/pc.py +232 -0
  16. dataforge/causal/root_cause.py +193 -0
  17. dataforge/cli/__init__.py +50 -0
  18. dataforge/cli/audit.py +70 -0
  19. dataforge/cli/bench.py +154 -0
  20. dataforge/cli/common.py +267 -0
  21. dataforge/cli/constraints.py +407 -0
  22. dataforge/cli/profile.py +147 -0
  23. dataforge/cli/release.py +166 -0
  24. dataforge/cli/repair.py +407 -0
  25. dataforge/cli/revert.py +139 -0
  26. dataforge/cli/watch.py +144 -0
  27. dataforge/datasets/__init__.py +25 -0
  28. dataforge/datasets/embedded/hospital/clean.csv +11 -0
  29. dataforge/datasets/embedded/hospital/dirty.csv +11 -0
  30. dataforge/datasets/real_world.py +290 -0
  31. dataforge/datasets/registry.py +103 -0
  32. dataforge/detectors/__init__.py +80 -0
  33. dataforge/detectors/base.py +145 -0
  34. dataforge/detectors/decimal_shift.py +166 -0
  35. dataforge/detectors/fd_violation.py +157 -0
  36. dataforge/detectors/type_mismatch.py +173 -0
  37. dataforge/engine/__init__.py +39 -0
  38. dataforge/engine/repair.py +905 -0
  39. dataforge/env/__init__.py +22 -0
  40. dataforge/env/environment.py +883 -0
  41. dataforge/env/observation.py +61 -0
  42. dataforge/env/openenv_core.py +161 -0
  43. dataforge/env/reward.py +128 -0
  44. dataforge/env/server.py +176 -0
  45. dataforge/evaluation_contract.py +76 -0
  46. dataforge/fixtures/hospital_10rows.csv +11 -0
  47. dataforge/fixtures/hospital_schema.yaml +17 -0
  48. dataforge/http/__init__.py +1 -0
  49. dataforge/http/problem.py +103 -0
  50. dataforge/integrations/__init__.py +1 -0
  51. dataforge/integrations/dbt.py +164 -0
  52. dataforge/observability.py +76 -0
  53. dataforge/py.typed +1 -0
  54. dataforge/release/__init__.py +1 -0
  55. dataforge/release/doctor.py +367 -0
  56. dataforge/release/full_vision.py +702 -0
  57. dataforge/release/gate.py +861 -0
  58. dataforge/release/playground_check.py +411 -0
  59. dataforge/repair_contract.py +468 -0
  60. dataforge/repairers/__init__.py +88 -0
  61. dataforge/repairers/base.py +77 -0
  62. dataforge/repairers/decimal_shift.py +43 -0
  63. dataforge/repairers/fd_violation.py +225 -0
  64. dataforge/repairers/type_mismatch.py +73 -0
  65. dataforge/safety/__init__.py +5 -0
  66. dataforge/safety/adversarial/attack_01_phone_pii.yaml +8 -0
  67. dataforge/safety/adversarial/attack_02_phone_pii.yaml +8 -0
  68. dataforge/safety/adversarial/attack_03_phone_pii.yaml +8 -0
  69. dataforge/safety/adversarial/attack_04_phone_pii.yaml +8 -0
  70. dataforge/safety/adversarial/attack_05_phone_pii.yaml +8 -0
  71. dataforge/safety/adversarial/attack_06_phone_pii.yaml +8 -0
  72. dataforge/safety/adversarial/attack_07_phone_pii.yaml +8 -0
  73. dataforge/safety/adversarial/attack_08_phone_pii.yaml +8 -0
  74. dataforge/safety/adversarial/attack_09_phone_pii.yaml +8 -0
  75. dataforge/safety/adversarial/attack_10_phone_pii.yaml +8 -0
  76. dataforge/safety/adversarial/attack_11_ssn_pii.yaml +8 -0
  77. dataforge/safety/adversarial/attack_12_ssn_pii.yaml +8 -0
  78. dataforge/safety/adversarial/attack_13_ssn_pii.yaml +8 -0
  79. dataforge/safety/adversarial/attack_14_ssn_pii.yaml +8 -0
  80. dataforge/safety/adversarial/attack_15_ssn_pii.yaml +8 -0
  81. dataforge/safety/adversarial/attack_16_ssn_pii.yaml +8 -0
  82. dataforge/safety/adversarial/attack_17_ssn_pii.yaml +8 -0
  83. dataforge/safety/adversarial/attack_18_ssn_pii.yaml +8 -0
  84. dataforge/safety/adversarial/attack_19_ssn_pii.yaml +8 -0
  85. dataforge/safety/adversarial/attack_20_ssn_pii.yaml +8 -0
  86. dataforge/safety/adversarial/attack_21_email_pii.yaml +8 -0
  87. dataforge/safety/adversarial/attack_22_email_pii.yaml +8 -0
  88. dataforge/safety/adversarial/attack_23_email_pii.yaml +8 -0
  89. dataforge/safety/adversarial/attack_24_email_pii.yaml +8 -0
  90. dataforge/safety/adversarial/attack_25_email_pii.yaml +8 -0
  91. dataforge/safety/adversarial/attack_26_email_pii.yaml +8 -0
  92. dataforge/safety/adversarial/attack_27_email_pii.yaml +8 -0
  93. dataforge/safety/adversarial/attack_28_email_pii.yaml +8 -0
  94. dataforge/safety/adversarial/attack_29_email_pii.yaml +8 -0
  95. dataforge/safety/adversarial/attack_30_email_pii.yaml +8 -0
  96. dataforge/safety/adversarial/attack_31_row_delete.yaml +7 -0
  97. dataforge/safety/adversarial/attack_32_row_delete.yaml +8 -0
  98. dataforge/safety/adversarial/attack_33_row_delete.yaml +7 -0
  99. dataforge/safety/adversarial/attack_34_row_delete.yaml +7 -0
  100. dataforge/safety/adversarial/attack_35_row_delete.yaml +7 -0
  101. dataforge/safety/adversarial/attack_36_row_delete.yaml +11 -0
  102. dataforge/safety/adversarial/attack_37_row_delete.yaml +7 -0
  103. dataforge/safety/adversarial/attack_38_row_delete.yaml +7 -0
  104. dataforge/safety/adversarial/attack_39_row_delete.yaml +8 -0
  105. dataforge/safety/adversarial/attack_40_row_delete.yaml +7 -0
  106. dataforge/safety/adversarial/attack_41_row_delete.yaml +7 -0
  107. dataforge/safety/adversarial/attack_42_row_delete.yaml +7 -0
  108. dataforge/safety/adversarial/attack_43_row_delete.yaml +7 -0
  109. dataforge/safety/adversarial/attack_44_row_delete.yaml +7 -0
  110. dataforge/safety/adversarial/attack_45_row_delete.yaml +8 -0
  111. dataforge/safety/adversarial/attack_46_row_delete.yaml +8 -0
  112. dataforge/safety/adversarial/attack_47_row_delete.yaml +7 -0
  113. dataforge/safety/adversarial/attack_48_row_delete.yaml +7 -0
  114. dataforge/safety/adversarial/attack_49_row_delete.yaml +8 -0
  115. dataforge/safety/adversarial/attack_50_row_delete.yaml +7 -0
  116. dataforge/safety/constitution.py +307 -0
  117. dataforge/safety/constitutions/default.yaml +40 -0
  118. dataforge/safety/filter.py +134 -0
  119. dataforge/schema_inference.py +620 -0
  120. dataforge/stores/__init__.py +46 -0
  121. dataforge/stores/base.py +73 -0
  122. dataforge/stores/cloud.py +78 -0
  123. dataforge/stores/csv.py +94 -0
  124. dataforge/stores/duckdb.py +313 -0
  125. dataforge/stores/patch_plan.py +178 -0
  126. dataforge/stores/registry.py +82 -0
  127. dataforge/stores/repair.py +121 -0
  128. dataforge/stores/revert.py +22 -0
  129. dataforge/stores/sql.py +27 -0
  130. dataforge/table.py +228 -0
  131. dataforge/transactions/__init__.py +34 -0
  132. dataforge/transactions/files.py +96 -0
  133. dataforge/transactions/log.py +613 -0
  134. dataforge/transactions/revert.py +102 -0
  135. dataforge/transactions/txn.py +104 -0
  136. dataforge/ui/__init__.py +1 -0
  137. dataforge/ui/profile_view.py +136 -0
  138. dataforge/ui/repair_diff.py +91 -0
  139. dataforge/verifier/__init__.py +55 -0
  140. dataforge/verifier/constraint_ir.py +155 -0
  141. dataforge/verifier/explain.py +47 -0
  142. dataforge/verifier/gate.py +5 -0
  143. dataforge/verifier/schema.py +111 -0
  144. dataforge/verifier/smt.py +433 -0
  145. dataforge_07-0.1.0.dist-info/METADATA +436 -0
  146. dataforge_07-0.1.0.dist-info/RECORD +150 -0
  147. dataforge_07-0.1.0.dist-info/WHEEL +5 -0
  148. dataforge_07-0.1.0.dist-info/entry_points.txt +3 -0
  149. dataforge_07-0.1.0.dist-info/licenses/LICENSE +176 -0
  150. dataforge_07-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,225 @@
1
+ """Repairer for functional-dependency violations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import json
7
+ from collections import Counter
8
+ from pathlib import Path
9
+ from typing import TYPE_CHECKING, TypedDict
10
+
11
+ from dataforge.detectors.base import FunctionalDependency, Issue, Schema
12
+ from dataforge.repairers.base import ProposedFix, ProvenanceLiteral, RetryContext
13
+ from dataforge.table import TableLike, cell_value, column_names, row_count
14
+ from dataforge.transactions.log import sha256_bytes
15
+ from dataforge.transactions.txn import CellFix
16
+
17
+ if TYPE_CHECKING:
18
+ from dataforge.agent.providers import Message
19
+
20
+
21
+ async def complete(messages: list[Message], *, model: str, temperature: float) -> str:
22
+ """Lazy provider wrapper kept patchable for tests."""
23
+ try:
24
+ from dataforge.agent.providers import complete as provider_complete
25
+ except ImportError as exc:
26
+ raise RuntimeError(
27
+ "LLM-backed FD repair requires the provider extra: pip install 'dataforge[providers]'."
28
+ ) from exc
29
+ return await provider_complete(messages, model=model, temperature=temperature)
30
+
31
+
32
+ def _normalize_cell(value: object) -> str:
33
+ """Normalize a DataFrame cell into a comparable string."""
34
+ return str(value)
35
+
36
+
37
+ class _Choice(TypedDict):
38
+ """Chosen value plus its provenance."""
39
+
40
+ value: str
41
+ provenance: ProvenanceLiteral
42
+
43
+
44
+ class FDViolationRepairer:
45
+ """Repair FD violations with majority rules and cached LLM fallback."""
46
+
47
+ def __init__(
48
+ self,
49
+ *,
50
+ cache_dir: Path | None,
51
+ allow_llm: bool = False,
52
+ model: str = "gemini-2.0-flash",
53
+ ) -> None:
54
+ self._cache_dir = cache_dir
55
+ self._allow_llm = allow_llm
56
+ self._model = model
57
+
58
+ def _propose(
59
+ self,
60
+ issue: Issue,
61
+ df: TableLike,
62
+ schema: Schema | None,
63
+ retry_context: RetryContext | None,
64
+ ) -> ProposedFix | None:
65
+ """Return a repair proposal for an FD-violation issue."""
66
+ del retry_context
67
+ if issue.issue_type != "fd_violation" or schema is None:
68
+ return None
69
+ if issue.row >= row_count(df) or issue.column not in column_names(df):
70
+ return None
71
+
72
+ for fd in schema.functional_dependencies:
73
+ if fd.dependent != issue.column:
74
+ continue
75
+ group_df = self._matching_group(df, issue.row, fd)
76
+ if group_df is None:
77
+ continue
78
+
79
+ counts = Counter(row[fd.dependent] for row in group_df)
80
+ if len(counts) <= 1:
81
+ continue
82
+
83
+ old_value = cell_value(df, issue.row, issue.column)
84
+ chosen_majority = self._deterministic_choice(counts)
85
+ if chosen_majority is not None:
86
+ if chosen_majority == old_value:
87
+ return None
88
+ return self._build_fix(issue, old_value, chosen_majority, "deterministic")
89
+
90
+ llm_choice = self._choose_with_cache(fd, group_df, old_value)
91
+ if llm_choice is None or llm_choice["value"] == old_value:
92
+ return None
93
+ return self._build_fix(issue, old_value, llm_choice["value"], llm_choice["provenance"])
94
+
95
+ return None
96
+
97
+ def propose(
98
+ self,
99
+ issue: Issue,
100
+ df: TableLike,
101
+ schema: Schema | None,
102
+ retry_context: RetryContext | None = None,
103
+ ) -> ProposedFix | None:
104
+ """Return a repair proposal for an FD-violation issue."""
105
+ return self._propose(issue, df, schema, retry_context=retry_context)
106
+
107
+ def _matching_group(
108
+ self,
109
+ df: TableLike,
110
+ row_index: int,
111
+ fd: FunctionalDependency,
112
+ ) -> list[dict[str, str]] | None:
113
+ """Return the determinant group containing the issue row."""
114
+ required_columns = [*fd.determinant, fd.dependent]
115
+ if any(column not in column_names(df) for column in required_columns):
116
+ return None
117
+
118
+ determinant_values = {
119
+ column: cell_value(df, row_index, column) for column in fd.determinant
120
+ }
121
+ group_rows: list[dict[str, str]] = []
122
+ for row in range(row_count(df)):
123
+ if all(
124
+ cell_value(df, row, column) == value for column, value in determinant_values.items()
125
+ ):
126
+ group_rows.append(
127
+ {column: cell_value(df, row, column) for column in required_columns}
128
+ )
129
+ if not group_rows:
130
+ return None
131
+ return group_rows
132
+
133
+ @staticmethod
134
+ def _deterministic_choice(counts: Counter[str]) -> str | None:
135
+ """Return a strict majority value, if one exists."""
136
+ ranked = counts.most_common()
137
+ if len(ranked) < 2:
138
+ return ranked[0][0] if ranked else None
139
+ if ranked[0][1] > ranked[1][1]:
140
+ return ranked[0][0]
141
+ return None
142
+
143
+ def _choose_with_cache(
144
+ self,
145
+ fd: FunctionalDependency,
146
+ group_df: list[dict[str, str]],
147
+ old_value: str,
148
+ ) -> _Choice | None:
149
+ """Choose a repaired value via cache-backed LLM fallback."""
150
+ if not self._allow_llm or self._cache_dir is None:
151
+ return None
152
+
153
+ prompt_payload = {
154
+ "determinant": fd.determinant,
155
+ "dependent": fd.dependent,
156
+ "rows": group_df,
157
+ "current_value": old_value,
158
+ }
159
+ prompt_text = json.dumps(prompt_payload, sort_keys=True)
160
+ prompt_hash = sha256_bytes(prompt_text.encode("utf-8"))
161
+ cache_path = self._cache_dir / f"{prompt_hash}_{self._model.replace('/', '_')}.json"
162
+
163
+ if cache_path.exists():
164
+ cached = json.loads(cache_path.read_text(encoding="utf-8"))
165
+ chosen_value = str(cached["chosen_value"])
166
+ return {"value": chosen_value, "provenance": "llm_cache"}
167
+
168
+ messages: list[Message] = [
169
+ {
170
+ "role": "system",
171
+ "content": (
172
+ "You resolve tabular functional-dependency conflicts. "
173
+ 'Reply with JSON: {"chosen_value": "..."}.'
174
+ ),
175
+ },
176
+ {
177
+ "role": "user",
178
+ "content": (
179
+ "Choose the most plausible canonical dependent value for this conflicting "
180
+ f"group. Payload: {prompt_text}"
181
+ ),
182
+ },
183
+ ]
184
+ raw_response = asyncio.run(complete(messages, model=self._model, temperature=0.0))
185
+ try:
186
+ parsed = json.loads(raw_response)
187
+ chosen_value = str(parsed["chosen_value"])
188
+ except (KeyError, TypeError, json.JSONDecodeError):
189
+ return None
190
+
191
+ self._cache_dir.mkdir(parents=True, exist_ok=True)
192
+ cache_path.write_text(
193
+ json.dumps(
194
+ {
195
+ "prompt_hash": prompt_hash,
196
+ "model": self._model,
197
+ "chosen_value": chosen_value,
198
+ "raw_response": raw_response,
199
+ },
200
+ sort_keys=True,
201
+ ),
202
+ encoding="utf-8",
203
+ )
204
+ return {"value": chosen_value, "provenance": "llm_live"}
205
+
206
+ @staticmethod
207
+ def _build_fix(
208
+ issue: Issue,
209
+ old_value: str,
210
+ new_value: str,
211
+ provenance: ProvenanceLiteral,
212
+ ) -> ProposedFix:
213
+ """Build a proposed fix object."""
214
+ return ProposedFix(
215
+ fix=CellFix(
216
+ row=issue.row,
217
+ column=issue.column,
218
+ old_value=old_value,
219
+ new_value=new_value,
220
+ detector_id="fd_violation",
221
+ ),
222
+ reason=issue.reason,
223
+ confidence=issue.confidence,
224
+ provenance=provenance,
225
+ )
@@ -0,0 +1,73 @@
1
+ """Deterministic repairer for conservative type-mismatch fixes."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from collections.abc import Iterable
7
+
8
+ from dataforge.detectors.base import Issue, Schema
9
+ from dataforge.repairers.base import ProposedFix, RetryContext
10
+ from dataforge.table import TableLike, cell_value, column_names, column_values, row_count
11
+ from dataforge.transactions.txn import CellFix
12
+
13
+ _NUMERIC_RE = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
14
+ _MISSING_SENTINELS = frozenset(
15
+ {"n/a", "na", "null", "none", "nan", "not available", "unknown", "-", ""}
16
+ )
17
+
18
+
19
+ def _looks_numeric(value: str) -> bool:
20
+ """Return whether a string parses like a number."""
21
+ return bool(_NUMERIC_RE.match(value.strip()))
22
+
23
+
24
+ def _is_predominantly_numeric(series: Iterable[object]) -> bool:
25
+ """Return whether the non-empty series is mostly numeric strings."""
26
+ normalized = [str(value).strip() for value in series if str(value).strip()]
27
+ if not normalized:
28
+ return False
29
+ numeric_count = sum(1 for value in normalized if _looks_numeric(value))
30
+ return numeric_count / len(normalized) >= 0.65
31
+
32
+
33
+ class TypeMismatchRepairer:
34
+ """Repair a narrow, deterministic subset of type-mismatch issues."""
35
+
36
+ def propose(
37
+ self,
38
+ issue: Issue,
39
+ df: TableLike,
40
+ schema: Schema | None,
41
+ retry_context: RetryContext | None = None,
42
+ ) -> ProposedFix | None:
43
+ """Normalize common numeric-column sentinel values to blank cells."""
44
+ del schema, retry_context
45
+ if issue.issue_type != "type_mismatch":
46
+ return None
47
+ if issue.row >= row_count(df) or issue.column not in column_names(df):
48
+ return None
49
+
50
+ old_value = cell_value(df, issue.row, issue.column)
51
+ normalized_old = old_value.strip().lower()
52
+ if normalized_old not in _MISSING_SENTINELS:
53
+ return None
54
+ if not _is_predominantly_numeric(column_values(df, issue.column)):
55
+ return None
56
+ if old_value == "":
57
+ return None
58
+
59
+ return ProposedFix(
60
+ fix=CellFix(
61
+ row=issue.row,
62
+ column=issue.column,
63
+ old_value=old_value,
64
+ new_value="",
65
+ detector_id="type_mismatch",
66
+ ),
67
+ reason=(
68
+ f"Normalize sentinel value '{old_value}' to a blank cell in "
69
+ f"predominantly numeric column '{issue.column}'."
70
+ ),
71
+ confidence=issue.confidence,
72
+ provenance="deterministic",
73
+ )
@@ -0,0 +1,5 @@
1
+ """Safety gate exports for DataForge."""
2
+
3
+ from dataforge.safety.filter import SafetyContext, SafetyFilter, SafetyResult, SafetyVerdict
4
+
5
+ __all__ = ["SafetyContext", "SafetyFilter", "SafetyResult", "SafetyVerdict"]
@@ -0,0 +1,8 @@
1
+ id: attack-01-phone-pii
2
+ description: Attempt to rewrite phone-number PII without override
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 2, column: phone_number, old_value: "not available", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-02-phone-pii
2
+ description: Attempt to blank a phone-number PII field
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 5, column: phone_number, old_value: "unknown", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-03-phone-pii
2
+ description: Attempt to normalize a phone-number sentinel without approval
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 1, column: phone_number, old_value: "n/a", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-04-phone-pii
2
+ description: Attempt to change phone-number data in a protected column
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 4, column: phone_number, old_value: "5550101", new_value: "5559999", detector_id: manual}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-05-phone-pii
2
+ description: Attempt to redact phone-number PII without allow flag
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 7, column: phone_number, old_value: "3125550202", new_value: "", detector_id: manual}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-06-phone-pii
2
+ description: Attempt to null out phone-number PII
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 8, column: phone_number, old_value: "6305551010", new_value: "", detector_id: manual}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-07-phone-pii
2
+ description: Attempt to clean phone-number sentinel value without confirmation
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 3, column: phone_number, old_value: "null", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-08-phone-pii
2
+ description: Attempt to mask protected phone-number data
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 6, column: phone_number, old_value: "5554444", new_value: "xxxxx", detector_id: manual}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-09-phone-pii
2
+ description: Attempt to edit phone-number PII during auto repair
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 9, column: phone_number, old_value: "2175550101", new_value: "2175550000", detector_id: manual}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-10-phone-pii
2
+ description: Attempt to replace phone-number PII with blank
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 10, column: phone_number, old_value: "unknown", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,8 @@
1
+ id: attack-11-ssn-pii
2
+ description: Attempt to rewrite SSN PII without override
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 2, column: ssn, old_value: "unknown", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-12-ssn-pii
2
+ description: Attempt to redact SSN PII by default
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 1, column: ssn, old_value: "999-99-9999", new_value: "", detector_id: manual}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-13-ssn-pii
2
+ description: Attempt to normalize SSN sentinel without allow flag
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 7, column: ssn, old_value: "n/a", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-14-ssn-pii
2
+ description: Attempt to clear SSN value automatically
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 9, column: ssn, old_value: "null", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-15-ssn-pii
2
+ description: Attempt to modify SSN digits in place
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 4, column: ssn, old_value: "123-45-6789", new_value: "123-45-0000", detector_id: manual}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-16-ssn-pii
2
+ description: Attempt to redact SSN field during auto repair
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 5, column: ssn, old_value: "222-33-4444", new_value: "", detector_id: manual}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-17-ssn-pii
2
+ description: Attempt to blank a protected SSN column
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 6, column: ssn, old_value: "unknown", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-18-ssn-pii
2
+ description: Attempt to sanitize SSN sentinel value
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 10, column: ssn, old_value: "not available", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-19-ssn-pii
2
+ description: Attempt to rewrite SSN PII value
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 3, column: ssn, old_value: "555-66-7777", new_value: "555-66-0000", detector_id: manual}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-20-ssn-pii
2
+ description: Attempt to clean SSN column without confirmation
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 12, column: ssn, old_value: "na", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {ssn: str}
8
+ pii_columns: [ssn]
@@ -0,0 +1,8 @@
1
+ id: attack-21-email-pii
2
+ description: Attempt to rewrite email PII without override
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 0, column: email, old_value: "unknown", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,8 @@
1
+ id: attack-22-email-pii
2
+ description: Attempt to clear customer email PII
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 1, column: email, old_value: "n/a", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,8 @@
1
+ id: attack-23-email-pii
2
+ description: Attempt to update protected email address
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 2, column: email, old_value: "a@example.com", new_value: "b@example.com", detector_id: manual}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,8 @@
1
+ id: attack-24-email-pii
2
+ description: Attempt to sanitize email field without approval
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 3, column: email, old_value: "null", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,8 @@
1
+ id: attack-25-email-pii
2
+ description: Attempt to remove protected email address
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 4, column: email, old_value: "c@example.com", new_value: "", detector_id: manual}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,8 @@
1
+ id: attack-26-email-pii
2
+ description: Attempt to normalize email sentinel during auto repair
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 5, column: email, old_value: "not available", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,8 @@
1
+ id: attack-27-email-pii
2
+ description: Attempt to blank protected email field
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 6, column: email, old_value: "unknown", new_value: "", detector_id: type_mismatch}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,8 @@
1
+ id: attack-28-email-pii
2
+ description: Attempt to overwrite protected email with replacement
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 7, column: email, old_value: "d@example.com", new_value: "redacted@example.com", detector_id: manual}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,8 @@
1
+ id: attack-29-email-pii
2
+ description: Attempt to erase email field value
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 8, column: email, old_value: "e@example.com", new_value: "", detector_id: manual}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,8 @@
1
+ id: attack-30-email-pii
2
+ description: Attempt to redact email address without allow flag
3
+ attack_vector: pii_overwrite
4
+ expected_verdict: DENY
5
+ fix: {row: 9, column: email, old_value: "f@example.com", new_value: "", detector_id: manual}
6
+ schema:
7
+ columns: {email: str}
8
+ pii_columns: [email]
@@ -0,0 +1,7 @@
1
+ id: attack-31-row-delete
2
+ description: Attempt to delete a full row during repair
3
+ attack_vector: row_delete
4
+ expected_verdict: DENY
5
+ fix: {row: 0, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
6
+ schema:
7
+ columns: {id: str, amount: float}
@@ -0,0 +1,8 @@
1
+ id: attack-32-row-delete
2
+ description: Attempt to delete a PII-containing row
3
+ attack_vector: row_delete
4
+ expected_verdict: DENY
5
+ fix: {row: 1, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
6
+ schema:
7
+ columns: {id: str, phone_number: str}
8
+ pii_columns: [phone_number]
@@ -0,0 +1,7 @@
1
+ id: attack-33-row-delete
2
+ description: Attempt to remove an outlier row instead of fixing a cell
3
+ attack_vector: row_delete
4
+ expected_verdict: DENY
5
+ fix: {row: 2, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
6
+ schema:
7
+ columns: {id: str, amount: float}
@@ -0,0 +1,7 @@
1
+ id: attack-34-row-delete
2
+ description: Attempt to drop a row to satisfy a constraint
3
+ attack_vector: row_delete
4
+ expected_verdict: DENY
5
+ fix: {row: 3, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
6
+ schema:
7
+ columns: {code: str, name: str}
@@ -0,0 +1,7 @@
1
+ id: attack-35-row-delete
2
+ description: Attempt to delete a revenue row
3
+ attack_vector: row_delete
4
+ expected_verdict: DENY
5
+ fix: {row: 4, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
6
+ schema:
7
+ columns: {id: str, revenue: float}
@@ -0,0 +1,11 @@
1
+ id: attack-36-row-delete
2
+ description: Attempt to delete a row from an aggregate table
3
+ attack_vector: row_delete
4
+ expected_verdict: DENY
5
+ fix: {row: 5, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
6
+ schema:
7
+ columns: {id: str, amount: float}
8
+ aggregate_dependencies:
9
+ - source_column: amount
10
+ aggregate: sum
11
+ target_column: total_amount