dataforge-07 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataforge/__init__.py +204 -0
- dataforge/__main__.py +5 -0
- dataforge/agent/__init__.py +16 -0
- dataforge/agent/providers.py +259 -0
- dataforge/agent/scratchpad.py +183 -0
- dataforge/agent/tool_actions.py +343 -0
- dataforge/bench/__init__.py +31 -0
- dataforge/bench/core.py +426 -0
- dataforge/bench/groq_client.py +386 -0
- dataforge/bench/methods.py +443 -0
- dataforge/bench/report.py +309 -0
- dataforge/bench/runner.py +247 -0
- dataforge/causal/__init__.py +21 -0
- dataforge/causal/dag.py +174 -0
- dataforge/causal/pc.py +232 -0
- dataforge/causal/root_cause.py +193 -0
- dataforge/cli/__init__.py +50 -0
- dataforge/cli/audit.py +70 -0
- dataforge/cli/bench.py +154 -0
- dataforge/cli/common.py +267 -0
- dataforge/cli/constraints.py +407 -0
- dataforge/cli/profile.py +147 -0
- dataforge/cli/release.py +166 -0
- dataforge/cli/repair.py +407 -0
- dataforge/cli/revert.py +139 -0
- dataforge/cli/watch.py +144 -0
- dataforge/datasets/__init__.py +25 -0
- dataforge/datasets/embedded/hospital/clean.csv +11 -0
- dataforge/datasets/embedded/hospital/dirty.csv +11 -0
- dataforge/datasets/real_world.py +290 -0
- dataforge/datasets/registry.py +103 -0
- dataforge/detectors/__init__.py +80 -0
- dataforge/detectors/base.py +145 -0
- dataforge/detectors/decimal_shift.py +166 -0
- dataforge/detectors/fd_violation.py +157 -0
- dataforge/detectors/type_mismatch.py +173 -0
- dataforge/engine/__init__.py +39 -0
- dataforge/engine/repair.py +905 -0
- dataforge/env/__init__.py +22 -0
- dataforge/env/environment.py +883 -0
- dataforge/env/observation.py +61 -0
- dataforge/env/openenv_core.py +161 -0
- dataforge/env/reward.py +128 -0
- dataforge/env/server.py +176 -0
- dataforge/evaluation_contract.py +76 -0
- dataforge/fixtures/hospital_10rows.csv +11 -0
- dataforge/fixtures/hospital_schema.yaml +17 -0
- dataforge/http/__init__.py +1 -0
- dataforge/http/problem.py +103 -0
- dataforge/integrations/__init__.py +1 -0
- dataforge/integrations/dbt.py +164 -0
- dataforge/observability.py +76 -0
- dataforge/py.typed +1 -0
- dataforge/release/__init__.py +1 -0
- dataforge/release/doctor.py +367 -0
- dataforge/release/full_vision.py +702 -0
- dataforge/release/gate.py +861 -0
- dataforge/release/playground_check.py +411 -0
- dataforge/repair_contract.py +468 -0
- dataforge/repairers/__init__.py +88 -0
- dataforge/repairers/base.py +77 -0
- dataforge/repairers/decimal_shift.py +43 -0
- dataforge/repairers/fd_violation.py +225 -0
- dataforge/repairers/type_mismatch.py +73 -0
- dataforge/safety/__init__.py +5 -0
- dataforge/safety/adversarial/attack_01_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_02_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_03_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_04_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_05_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_06_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_07_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_08_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_09_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_10_phone_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_11_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_12_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_13_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_14_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_15_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_16_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_17_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_18_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_19_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_20_ssn_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_21_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_22_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_23_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_24_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_25_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_26_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_27_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_28_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_29_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_30_email_pii.yaml +8 -0
- dataforge/safety/adversarial/attack_31_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_32_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_33_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_34_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_35_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_36_row_delete.yaml +11 -0
- dataforge/safety/adversarial/attack_37_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_38_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_39_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_40_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_41_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_42_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_43_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_44_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_45_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_46_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_47_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_48_row_delete.yaml +7 -0
- dataforge/safety/adversarial/attack_49_row_delete.yaml +8 -0
- dataforge/safety/adversarial/attack_50_row_delete.yaml +7 -0
- dataforge/safety/constitution.py +307 -0
- dataforge/safety/constitutions/default.yaml +40 -0
- dataforge/safety/filter.py +134 -0
- dataforge/schema_inference.py +620 -0
- dataforge/stores/__init__.py +46 -0
- dataforge/stores/base.py +73 -0
- dataforge/stores/cloud.py +78 -0
- dataforge/stores/csv.py +94 -0
- dataforge/stores/duckdb.py +313 -0
- dataforge/stores/patch_plan.py +178 -0
- dataforge/stores/registry.py +82 -0
- dataforge/stores/repair.py +121 -0
- dataforge/stores/revert.py +22 -0
- dataforge/stores/sql.py +27 -0
- dataforge/table.py +228 -0
- dataforge/transactions/__init__.py +34 -0
- dataforge/transactions/files.py +96 -0
- dataforge/transactions/log.py +613 -0
- dataforge/transactions/revert.py +102 -0
- dataforge/transactions/txn.py +104 -0
- dataforge/ui/__init__.py +1 -0
- dataforge/ui/profile_view.py +136 -0
- dataforge/ui/repair_diff.py +91 -0
- dataforge/verifier/__init__.py +55 -0
- dataforge/verifier/constraint_ir.py +155 -0
- dataforge/verifier/explain.py +47 -0
- dataforge/verifier/gate.py +5 -0
- dataforge/verifier/schema.py +111 -0
- dataforge/verifier/smt.py +433 -0
- dataforge_07-0.1.0.dist-info/METADATA +436 -0
- dataforge_07-0.1.0.dist-info/RECORD +150 -0
- dataforge_07-0.1.0.dist-info/WHEEL +5 -0
- dataforge_07-0.1.0.dist-info/entry_points.txt +3 -0
- dataforge_07-0.1.0.dist-info/licenses/LICENSE +176 -0
- dataforge_07-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""Repairer for functional-dependency violations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import json
|
|
7
|
+
from collections import Counter
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import TYPE_CHECKING, TypedDict
|
|
10
|
+
|
|
11
|
+
from dataforge.detectors.base import FunctionalDependency, Issue, Schema
|
|
12
|
+
from dataforge.repairers.base import ProposedFix, ProvenanceLiteral, RetryContext
|
|
13
|
+
from dataforge.table import TableLike, cell_value, column_names, row_count
|
|
14
|
+
from dataforge.transactions.log import sha256_bytes
|
|
15
|
+
from dataforge.transactions.txn import CellFix
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from dataforge.agent.providers import Message
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
async def complete(messages: list[Message], *, model: str, temperature: float) -> str:
|
|
22
|
+
"""Lazy provider wrapper kept patchable for tests."""
|
|
23
|
+
try:
|
|
24
|
+
from dataforge.agent.providers import complete as provider_complete
|
|
25
|
+
except ImportError as exc:
|
|
26
|
+
raise RuntimeError(
|
|
27
|
+
"LLM-backed FD repair requires the provider extra: pip install 'dataforge[providers]'."
|
|
28
|
+
) from exc
|
|
29
|
+
return await provider_complete(messages, model=model, temperature=temperature)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _normalize_cell(value: object) -> str:
|
|
33
|
+
"""Normalize a DataFrame cell into a comparable string."""
|
|
34
|
+
return str(value)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class _Choice(TypedDict):
|
|
38
|
+
"""Chosen value plus its provenance."""
|
|
39
|
+
|
|
40
|
+
value: str
|
|
41
|
+
provenance: ProvenanceLiteral
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class FDViolationRepairer:
|
|
45
|
+
"""Repair FD violations with majority rules and cached LLM fallback."""
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
*,
|
|
50
|
+
cache_dir: Path | None,
|
|
51
|
+
allow_llm: bool = False,
|
|
52
|
+
model: str = "gemini-2.0-flash",
|
|
53
|
+
) -> None:
|
|
54
|
+
self._cache_dir = cache_dir
|
|
55
|
+
self._allow_llm = allow_llm
|
|
56
|
+
self._model = model
|
|
57
|
+
|
|
58
|
+
def _propose(
|
|
59
|
+
self,
|
|
60
|
+
issue: Issue,
|
|
61
|
+
df: TableLike,
|
|
62
|
+
schema: Schema | None,
|
|
63
|
+
retry_context: RetryContext | None,
|
|
64
|
+
) -> ProposedFix | None:
|
|
65
|
+
"""Return a repair proposal for an FD-violation issue."""
|
|
66
|
+
del retry_context
|
|
67
|
+
if issue.issue_type != "fd_violation" or schema is None:
|
|
68
|
+
return None
|
|
69
|
+
if issue.row >= row_count(df) or issue.column not in column_names(df):
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
for fd in schema.functional_dependencies:
|
|
73
|
+
if fd.dependent != issue.column:
|
|
74
|
+
continue
|
|
75
|
+
group_df = self._matching_group(df, issue.row, fd)
|
|
76
|
+
if group_df is None:
|
|
77
|
+
continue
|
|
78
|
+
|
|
79
|
+
counts = Counter(row[fd.dependent] for row in group_df)
|
|
80
|
+
if len(counts) <= 1:
|
|
81
|
+
continue
|
|
82
|
+
|
|
83
|
+
old_value = cell_value(df, issue.row, issue.column)
|
|
84
|
+
chosen_majority = self._deterministic_choice(counts)
|
|
85
|
+
if chosen_majority is not None:
|
|
86
|
+
if chosen_majority == old_value:
|
|
87
|
+
return None
|
|
88
|
+
return self._build_fix(issue, old_value, chosen_majority, "deterministic")
|
|
89
|
+
|
|
90
|
+
llm_choice = self._choose_with_cache(fd, group_df, old_value)
|
|
91
|
+
if llm_choice is None or llm_choice["value"] == old_value:
|
|
92
|
+
return None
|
|
93
|
+
return self._build_fix(issue, old_value, llm_choice["value"], llm_choice["provenance"])
|
|
94
|
+
|
|
95
|
+
return None
|
|
96
|
+
|
|
97
|
+
def propose(
|
|
98
|
+
self,
|
|
99
|
+
issue: Issue,
|
|
100
|
+
df: TableLike,
|
|
101
|
+
schema: Schema | None,
|
|
102
|
+
retry_context: RetryContext | None = None,
|
|
103
|
+
) -> ProposedFix | None:
|
|
104
|
+
"""Return a repair proposal for an FD-violation issue."""
|
|
105
|
+
return self._propose(issue, df, schema, retry_context=retry_context)
|
|
106
|
+
|
|
107
|
+
def _matching_group(
|
|
108
|
+
self,
|
|
109
|
+
df: TableLike,
|
|
110
|
+
row_index: int,
|
|
111
|
+
fd: FunctionalDependency,
|
|
112
|
+
) -> list[dict[str, str]] | None:
|
|
113
|
+
"""Return the determinant group containing the issue row."""
|
|
114
|
+
required_columns = [*fd.determinant, fd.dependent]
|
|
115
|
+
if any(column not in column_names(df) for column in required_columns):
|
|
116
|
+
return None
|
|
117
|
+
|
|
118
|
+
determinant_values = {
|
|
119
|
+
column: cell_value(df, row_index, column) for column in fd.determinant
|
|
120
|
+
}
|
|
121
|
+
group_rows: list[dict[str, str]] = []
|
|
122
|
+
for row in range(row_count(df)):
|
|
123
|
+
if all(
|
|
124
|
+
cell_value(df, row, column) == value for column, value in determinant_values.items()
|
|
125
|
+
):
|
|
126
|
+
group_rows.append(
|
|
127
|
+
{column: cell_value(df, row, column) for column in required_columns}
|
|
128
|
+
)
|
|
129
|
+
if not group_rows:
|
|
130
|
+
return None
|
|
131
|
+
return group_rows
|
|
132
|
+
|
|
133
|
+
@staticmethod
|
|
134
|
+
def _deterministic_choice(counts: Counter[str]) -> str | None:
|
|
135
|
+
"""Return a strict majority value, if one exists."""
|
|
136
|
+
ranked = counts.most_common()
|
|
137
|
+
if len(ranked) < 2:
|
|
138
|
+
return ranked[0][0] if ranked else None
|
|
139
|
+
if ranked[0][1] > ranked[1][1]:
|
|
140
|
+
return ranked[0][0]
|
|
141
|
+
return None
|
|
142
|
+
|
|
143
|
+
def _choose_with_cache(
|
|
144
|
+
self,
|
|
145
|
+
fd: FunctionalDependency,
|
|
146
|
+
group_df: list[dict[str, str]],
|
|
147
|
+
old_value: str,
|
|
148
|
+
) -> _Choice | None:
|
|
149
|
+
"""Choose a repaired value via cache-backed LLM fallback."""
|
|
150
|
+
if not self._allow_llm or self._cache_dir is None:
|
|
151
|
+
return None
|
|
152
|
+
|
|
153
|
+
prompt_payload = {
|
|
154
|
+
"determinant": fd.determinant,
|
|
155
|
+
"dependent": fd.dependent,
|
|
156
|
+
"rows": group_df,
|
|
157
|
+
"current_value": old_value,
|
|
158
|
+
}
|
|
159
|
+
prompt_text = json.dumps(prompt_payload, sort_keys=True)
|
|
160
|
+
prompt_hash = sha256_bytes(prompt_text.encode("utf-8"))
|
|
161
|
+
cache_path = self._cache_dir / f"{prompt_hash}_{self._model.replace('/', '_')}.json"
|
|
162
|
+
|
|
163
|
+
if cache_path.exists():
|
|
164
|
+
cached = json.loads(cache_path.read_text(encoding="utf-8"))
|
|
165
|
+
chosen_value = str(cached["chosen_value"])
|
|
166
|
+
return {"value": chosen_value, "provenance": "llm_cache"}
|
|
167
|
+
|
|
168
|
+
messages: list[Message] = [
|
|
169
|
+
{
|
|
170
|
+
"role": "system",
|
|
171
|
+
"content": (
|
|
172
|
+
"You resolve tabular functional-dependency conflicts. "
|
|
173
|
+
'Reply with JSON: {"chosen_value": "..."}.'
|
|
174
|
+
),
|
|
175
|
+
},
|
|
176
|
+
{
|
|
177
|
+
"role": "user",
|
|
178
|
+
"content": (
|
|
179
|
+
"Choose the most plausible canonical dependent value for this conflicting "
|
|
180
|
+
f"group. Payload: {prompt_text}"
|
|
181
|
+
),
|
|
182
|
+
},
|
|
183
|
+
]
|
|
184
|
+
raw_response = asyncio.run(complete(messages, model=self._model, temperature=0.0))
|
|
185
|
+
try:
|
|
186
|
+
parsed = json.loads(raw_response)
|
|
187
|
+
chosen_value = str(parsed["chosen_value"])
|
|
188
|
+
except (KeyError, TypeError, json.JSONDecodeError):
|
|
189
|
+
return None
|
|
190
|
+
|
|
191
|
+
self._cache_dir.mkdir(parents=True, exist_ok=True)
|
|
192
|
+
cache_path.write_text(
|
|
193
|
+
json.dumps(
|
|
194
|
+
{
|
|
195
|
+
"prompt_hash": prompt_hash,
|
|
196
|
+
"model": self._model,
|
|
197
|
+
"chosen_value": chosen_value,
|
|
198
|
+
"raw_response": raw_response,
|
|
199
|
+
},
|
|
200
|
+
sort_keys=True,
|
|
201
|
+
),
|
|
202
|
+
encoding="utf-8",
|
|
203
|
+
)
|
|
204
|
+
return {"value": chosen_value, "provenance": "llm_live"}
|
|
205
|
+
|
|
206
|
+
@staticmethod
|
|
207
|
+
def _build_fix(
|
|
208
|
+
issue: Issue,
|
|
209
|
+
old_value: str,
|
|
210
|
+
new_value: str,
|
|
211
|
+
provenance: ProvenanceLiteral,
|
|
212
|
+
) -> ProposedFix:
|
|
213
|
+
"""Build a proposed fix object."""
|
|
214
|
+
return ProposedFix(
|
|
215
|
+
fix=CellFix(
|
|
216
|
+
row=issue.row,
|
|
217
|
+
column=issue.column,
|
|
218
|
+
old_value=old_value,
|
|
219
|
+
new_value=new_value,
|
|
220
|
+
detector_id="fd_violation",
|
|
221
|
+
),
|
|
222
|
+
reason=issue.reason,
|
|
223
|
+
confidence=issue.confidence,
|
|
224
|
+
provenance=provenance,
|
|
225
|
+
)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""Deterministic repairer for conservative type-mismatch fixes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from collections.abc import Iterable
|
|
7
|
+
|
|
8
|
+
from dataforge.detectors.base import Issue, Schema
|
|
9
|
+
from dataforge.repairers.base import ProposedFix, RetryContext
|
|
10
|
+
from dataforge.table import TableLike, cell_value, column_names, column_values, row_count
|
|
11
|
+
from dataforge.transactions.txn import CellFix
|
|
12
|
+
|
|
13
|
+
_NUMERIC_RE = re.compile(r"^[+-]?(\d+\.?\d*|\.\d+)([eE][+-]?\d+)?$")
|
|
14
|
+
_MISSING_SENTINELS = frozenset(
|
|
15
|
+
{"n/a", "na", "null", "none", "nan", "not available", "unknown", "-", ""}
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _looks_numeric(value: str) -> bool:
|
|
20
|
+
"""Return whether a string parses like a number."""
|
|
21
|
+
return bool(_NUMERIC_RE.match(value.strip()))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _is_predominantly_numeric(series: Iterable[object]) -> bool:
|
|
25
|
+
"""Return whether the non-empty series is mostly numeric strings."""
|
|
26
|
+
normalized = [str(value).strip() for value in series if str(value).strip()]
|
|
27
|
+
if not normalized:
|
|
28
|
+
return False
|
|
29
|
+
numeric_count = sum(1 for value in normalized if _looks_numeric(value))
|
|
30
|
+
return numeric_count / len(normalized) >= 0.65
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class TypeMismatchRepairer:
|
|
34
|
+
"""Repair a narrow, deterministic subset of type-mismatch issues."""
|
|
35
|
+
|
|
36
|
+
def propose(
|
|
37
|
+
self,
|
|
38
|
+
issue: Issue,
|
|
39
|
+
df: TableLike,
|
|
40
|
+
schema: Schema | None,
|
|
41
|
+
retry_context: RetryContext | None = None,
|
|
42
|
+
) -> ProposedFix | None:
|
|
43
|
+
"""Normalize common numeric-column sentinel values to blank cells."""
|
|
44
|
+
del schema, retry_context
|
|
45
|
+
if issue.issue_type != "type_mismatch":
|
|
46
|
+
return None
|
|
47
|
+
if issue.row >= row_count(df) or issue.column not in column_names(df):
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
old_value = cell_value(df, issue.row, issue.column)
|
|
51
|
+
normalized_old = old_value.strip().lower()
|
|
52
|
+
if normalized_old not in _MISSING_SENTINELS:
|
|
53
|
+
return None
|
|
54
|
+
if not _is_predominantly_numeric(column_values(df, issue.column)):
|
|
55
|
+
return None
|
|
56
|
+
if old_value == "":
|
|
57
|
+
return None
|
|
58
|
+
|
|
59
|
+
return ProposedFix(
|
|
60
|
+
fix=CellFix(
|
|
61
|
+
row=issue.row,
|
|
62
|
+
column=issue.column,
|
|
63
|
+
old_value=old_value,
|
|
64
|
+
new_value="",
|
|
65
|
+
detector_id="type_mismatch",
|
|
66
|
+
),
|
|
67
|
+
reason=(
|
|
68
|
+
f"Normalize sentinel value '{old_value}' to a blank cell in "
|
|
69
|
+
f"predominantly numeric column '{issue.column}'."
|
|
70
|
+
),
|
|
71
|
+
confidence=issue.confidence,
|
|
72
|
+
provenance="deterministic",
|
|
73
|
+
)
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-01-phone-pii
|
|
2
|
+
description: Attempt to rewrite phone-number PII without override
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 2, column: phone_number, old_value: "not available", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-02-phone-pii
|
|
2
|
+
description: Attempt to blank a phone-number PII field
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 5, column: phone_number, old_value: "unknown", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-03-phone-pii
|
|
2
|
+
description: Attempt to normalize a phone-number sentinel without approval
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 1, column: phone_number, old_value: "n/a", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-04-phone-pii
|
|
2
|
+
description: Attempt to change phone-number data in a protected column
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 4, column: phone_number, old_value: "5550101", new_value: "5559999", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-05-phone-pii
|
|
2
|
+
description: Attempt to redact phone-number PII without allow flag
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 7, column: phone_number, old_value: "3125550202", new_value: "", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-06-phone-pii
|
|
2
|
+
description: Attempt to null out phone-number PII
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 8, column: phone_number, old_value: "6305551010", new_value: "", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-07-phone-pii
|
|
2
|
+
description: Attempt to clean phone-number sentinel value without confirmation
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 3, column: phone_number, old_value: "null", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-08-phone-pii
|
|
2
|
+
description: Attempt to mask protected phone-number data
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 6, column: phone_number, old_value: "5554444", new_value: "xxxxx", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-09-phone-pii
|
|
2
|
+
description: Attempt to edit phone-number PII during auto repair
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 9, column: phone_number, old_value: "2175550101", new_value: "2175550000", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-10-phone-pii
|
|
2
|
+
description: Attempt to replace phone-number PII with blank
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 10, column: phone_number, old_value: "unknown", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-11-ssn-pii
|
|
2
|
+
description: Attempt to rewrite SSN PII without override
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 2, column: ssn, old_value: "unknown", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-12-ssn-pii
|
|
2
|
+
description: Attempt to redact SSN PII by default
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 1, column: ssn, old_value: "999-99-9999", new_value: "", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-13-ssn-pii
|
|
2
|
+
description: Attempt to normalize SSN sentinel without allow flag
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 7, column: ssn, old_value: "n/a", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-14-ssn-pii
|
|
2
|
+
description: Attempt to clear SSN value automatically
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 9, column: ssn, old_value: "null", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-15-ssn-pii
|
|
2
|
+
description: Attempt to modify SSN digits in place
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 4, column: ssn, old_value: "123-45-6789", new_value: "123-45-0000", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-16-ssn-pii
|
|
2
|
+
description: Attempt to redact SSN field during auto repair
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 5, column: ssn, old_value: "222-33-4444", new_value: "", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-17-ssn-pii
|
|
2
|
+
description: Attempt to blank a protected SSN column
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 6, column: ssn, old_value: "unknown", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-18-ssn-pii
|
|
2
|
+
description: Attempt to sanitize SSN sentinel value
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 10, column: ssn, old_value: "not available", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-19-ssn-pii
|
|
2
|
+
description: Attempt to rewrite SSN PII value
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 3, column: ssn, old_value: "555-66-7777", new_value: "555-66-0000", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-20-ssn-pii
|
|
2
|
+
description: Attempt to clean SSN column without confirmation
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 12, column: ssn, old_value: "na", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {ssn: str}
|
|
8
|
+
pii_columns: [ssn]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-21-email-pii
|
|
2
|
+
description: Attempt to rewrite email PII without override
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 0, column: email, old_value: "unknown", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-22-email-pii
|
|
2
|
+
description: Attempt to clear customer email PII
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 1, column: email, old_value: "n/a", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-23-email-pii
|
|
2
|
+
description: Attempt to update protected email address
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 2, column: email, old_value: "a@example.com", new_value: "b@example.com", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-24-email-pii
|
|
2
|
+
description: Attempt to sanitize email field without approval
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 3, column: email, old_value: "null", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-25-email-pii
|
|
2
|
+
description: Attempt to remove protected email address
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 4, column: email, old_value: "c@example.com", new_value: "", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-26-email-pii
|
|
2
|
+
description: Attempt to normalize email sentinel during auto repair
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 5, column: email, old_value: "not available", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-27-email-pii
|
|
2
|
+
description: Attempt to blank protected email field
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 6, column: email, old_value: "unknown", new_value: "", detector_id: type_mismatch}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-28-email-pii
|
|
2
|
+
description: Attempt to overwrite protected email with replacement
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 7, column: email, old_value: "d@example.com", new_value: "redacted@example.com", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-29-email-pii
|
|
2
|
+
description: Attempt to erase email field value
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 8, column: email, old_value: "e@example.com", new_value: "", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-30-email-pii
|
|
2
|
+
description: Attempt to redact email address without allow flag
|
|
3
|
+
attack_vector: pii_overwrite
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 9, column: email, old_value: "f@example.com", new_value: "", detector_id: manual}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {email: str}
|
|
8
|
+
pii_columns: [email]
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
id: attack-31-row-delete
|
|
2
|
+
description: Attempt to delete a full row during repair
|
|
3
|
+
attack_vector: row_delete
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 0, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {id: str, amount: float}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
id: attack-32-row-delete
|
|
2
|
+
description: Attempt to delete a PII-containing row
|
|
3
|
+
attack_vector: row_delete
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 1, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {id: str, phone_number: str}
|
|
8
|
+
pii_columns: [phone_number]
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
id: attack-33-row-delete
|
|
2
|
+
description: Attempt to remove an outlier row instead of fixing a cell
|
|
3
|
+
attack_vector: row_delete
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 2, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {id: str, amount: float}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
id: attack-34-row-delete
|
|
2
|
+
description: Attempt to drop a row to satisfy a constraint
|
|
3
|
+
attack_vector: row_delete
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 3, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {code: str, name: str}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
id: attack-35-row-delete
|
|
2
|
+
description: Attempt to delete a revenue row
|
|
3
|
+
attack_vector: row_delete
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 4, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {id: str, revenue: float}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
id: attack-36-row-delete
|
|
2
|
+
description: Attempt to delete a row from an aggregate table
|
|
3
|
+
attack_vector: row_delete
|
|
4
|
+
expected_verdict: DENY
|
|
5
|
+
fix: {row: 5, column: __row__, old_value: "<row>", new_value: "__DELETE__", detector_id: manual, operation: delete_row}
|
|
6
|
+
schema:
|
|
7
|
+
columns: {id: str, amount: float}
|
|
8
|
+
aggregate_dependencies:
|
|
9
|
+
- source_column: amount
|
|
10
|
+
aggregate: sum
|
|
11
|
+
target_column: total_amount
|