ramen-data-filter 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,82 @@
1
+ # ============================================================
2
+ # ramen-ai-integrations — .gitignore
3
+ # ============================================================
4
+
5
+ # --- Reference directory (local context only, never committed) ---
6
+ _ref/
7
+
8
+ # --- Internal runbooks (local only, never committed) ---
9
+ docs/integration-checklist.md
10
+
11
+ # --- Node.js ---
12
+ node_modules/
13
+ npm-debug.log*
14
+ yarn-debug.log*
15
+ yarn-error.log*
16
+ .npm
17
+ .yarn/cache
18
+ .yarn/unplugged
19
+ .pnp.*
20
+
21
+ # --- Python ---
22
+ __pycache__/
23
+ *.py[cod]
24
+ *$py.class
25
+ *.egg-info/
26
+ *.egg
27
+ .eggs/
28
+ dist/
29
+ build/
30
+ *.whl
31
+ .venv/
32
+ venv/
33
+ env/
34
+ .Python
35
+ pip-wheel-metadata/
36
+
37
+ # --- Build outputs ---
38
+ dist/
39
+ build/
40
+ out/
41
+ *.tsbuildinfo
42
+
43
+ # GitHub Actions require their compiled single-file bundle to be committed,
44
+ # so the action's dist/ is explicitly tracked (overrides the dist/ rule above).
45
+ !plugins/github-action/dist/
46
+ !plugins/github-action/dist/**
47
+
48
+ # --- Environment & secrets ---
49
+ .env
50
+ .env.*
51
+ !.env.example
52
+ *.pem
53
+ *.key
54
+ *.p12
55
+
56
+ # --- IDE & OS ---
57
+ .DS_Store
58
+ Thumbs.db
59
+ .vscode/
60
+ .idea/
61
+ *.swp
62
+ *.swo
63
+
64
+ # --- Test coverage ---
65
+ coverage/
66
+ .coverage
67
+ htmlcov/
68
+ .pytest_cache/
69
+ .nyc_output/
70
+
71
+ # --- Logs ---
72
+ *.log
73
+ logs/
74
+
75
+ # --- Agents ---
76
+ AGENTS.md
77
+
78
+ # --- Internal research & design documents (local context only) ---
79
+ docs/
80
+ # --- MLflow local tracking artifacts ---
81
+ mlruns/
82
+ mlartifacts/
@@ -0,0 +1,296 @@
1
+ Metadata-Version: 2.4
2
+ Name: ramen-data-filter
3
+ Version: 0.1.0
4
+ Summary: Dual-mode Pandas and CSV filtration for ramen-ai RAG ingestion and MLOps dataset sanitization.
5
+ License: Proprietary
6
+ Requires-Python: >=3.10
7
+ Requires-Dist: pandas==2.3.3
8
+ Requires-Dist: ramen-ai-core==0.2.1
9
+ Provides-Extra: dev
10
+ Requires-Dist: pytest==8.4.2; extra == 'dev'
11
+ Description-Content-Type: text/markdown
12
+
13
+ # ramen-ai Data Filter
14
+
15
+ <p align="center">
16
+ <img src="../../assets/ramen-logo.png" alt="ramen-ai" width="100"/>
17
+ </p>
18
+
19
+ Dual-mode Pandas and CSV filtration for RAG ingestion scrubbing and MLOps
20
+ dataset sanitization. Every row is evaluated through the ramen-ai semantic
21
+ firewall before it can enter a retrieval corpus, training set, analytics store,
22
+ or downstream data pipeline.
23
+
24
+ Use **Strict Exclusion** to remove blocked rows, or **Semantic Imputation** to
25
+ preserve row count while replacing only explicitly allowlisted columns named by
26
+ ramen-ai steering. Typical policies can detect PII, data-exfiltration vectors,
27
+ poisoned retrieval context, prompt injection, and other unsafe dataset content.
28
+
29
+ Requires **Python ≥ 3.10**.
30
+
31
+ ---
32
+
33
+ ## Supported Ecosystems
34
+
35
+ <p align="center">
36
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/langchain-python">
37
+ <img src="https://img.shields.io/badge/LangChain-1C3C3C?style=flat&logo=langchain&logoColor=white" alt="LangChain"/>
38
+ </a>
39
+ &nbsp;
40
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/pydantic-ai">
41
+ <img src="https://img.shields.io/badge/PydanticAI-E92063?style=flat&logo=pydantic&logoColor=white" alt="PydanticAI"/>
42
+ </a>
43
+ &nbsp;
44
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/mcp-proxy">
45
+ <img src="https://img.shields.io/badge/MCP-6B21A8?style=flat&logo=anthropic&logoColor=white" alt="MCP"/>
46
+ </a>
47
+ &nbsp;
48
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/agt-typescript">
49
+ <img src="https://img.shields.io/badge/Microsoft%20AGT-0078D4?style=flat&logo=microsoft&logoColor=white" alt="Microsoft AGT"/>
50
+ </a>
51
+ &nbsp;
52
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/github-action">
53
+ <img src="https://img.shields.io/badge/GitHub%20Actions-2088FF?style=flat&logo=githubactions&logoColor=white" alt="GitHub Actions"/>
54
+ </a>
55
+ &nbsp;
56
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/cmcp-python">
57
+ <img src="https://img.shields.io/badge/cMCP-00A67E?style=flat&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZmlsbD0id2hpdGUiIGQ9Ik0xMiAyTDIgN2wxMCA1IDEwLTV6TTIgMTdsOCA0VjExbC04LTR6TTE0IDIxbDgtNFYxMWwtOCA0eiIvPjwvc3ZnPg==&logoColor=white" alt="cMCP"/>
58
+ </a>
59
+ &nbsp;
60
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/mlflow-python">
61
+ <img src="https://img.shields.io/badge/MLflow-0194E2?style=flat&logo=mlflow&logoColor=white" alt="MLflow"/>
62
+ </a>
63
+ &nbsp;
64
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/ramen-data-filter">
65
+ <img src="https://img.shields.io/badge/ramen%20data%20filter-D97706?style=flat&logo=pandas&logoColor=white" alt="ramen data filter"/>
66
+ </a>
67
+ </p>
68
+
69
+ ---
70
+
71
+ ## Can you bypass it?
72
+
73
+ Standard safety filters catch basic syntax. They fail against encoded payloads
74
+ and corporate jargon. We challenge you to bypass our semantic firewall using
75
+ the zero-day evasion vectors in our official **[Red Team Guide](../../RED_TEAM_GUIDE.md)**.
76
+
77
+ Below is a simulation of the Grok/Bankr heist. We fed the raw adversarial
78
+ prompt directly into our sandbox. It uses a social engineering wrapper
79
+ (claiming a visual impairment) to smuggle a 3,000,000,000 DRB transfer
80
+ instruction encoded in Morse code. The firewall evaluated the underlying
81
+ semantic intent, intercepted the unauthorized financial transfer, and blocked
82
+ it pre-execution, issuing a verified Ed25519 receipt.
83
+
84
+ <p align="center">
85
+ <img src="../../assets/grok-bankr.png" alt="ramen-ai intercepting the Grok/Bankr Morse-code heist pre-execution" width="720"/>
86
+ </p>
87
+
88
+ ---
89
+
90
+ ## API Key
91
+
92
+ To use this integration, you must mint an API Key. We offer a **Free Starter
93
+ Tier** (1,000 evaluations/month, BYOK) which includes full access to our Core
94
+ IT Security bundle. Mint your key at:
95
+ **[https://ramenai.dev/pricing](https://ramenai.dev/pricing)**
96
+
97
+ ---
98
+
99
+ ## Installation
100
+
101
+ From PyPI after release:
102
+
103
+ ```bash
104
+ pip install ramen-data-filter
105
+ ```
106
+
107
+ From this monorepo:
108
+
109
+ ```bash
110
+ pip install -e plugins/ramen-data-filter
111
+ pip install -e "plugins/ramen-data-filter[dev]" # include pytest
112
+ ```
113
+
114
+ ---
115
+
116
+ ## Configuration / BYOK
117
+
118
+ ```bash
119
+ export RAMEN_API_KEY=ramen_ak_... # required
120
+ export OPENAI_API_KEY=sk-... # Starter/Professional BYOK
121
+ ```
122
+
123
+ `RAMEN_API_KEY` authenticates the evaluation request. The Starter and
124
+ Professional tiers also require an LLM provider key. The plugin reads
125
+ `OPENAI_API_KEY` at evaluation time and `ramen-ai-core` forwards it as the
126
+ `X-Provider-Key` header. Without a provider key, those tiers return
127
+ `402 Payment Required`.
128
+
129
+ Pass `provider_name="anthropic"`, `"google"`, `"synthetic"`, or
130
+ `"hyperbolic"` with the corresponding `provider_key` when OpenAI is not the
131
+ provider. Enterprise tiers use platform-managed provider keys and do not need
132
+ `OPENAI_API_KEY`.
133
+
134
+ Credentials are read from environment variables or passed at runtime. Never
135
+ store them in source code, notebooks, CSV files, or committed configuration.
136
+
137
+ ---
138
+
139
+ ## Usage
140
+
141
+ ### Strict Exclusion
142
+
143
+ Strict exclusion evaluates each row and returns a new DataFrame containing only
144
+ `[ALLOWED]` rows. The returned index is reset; the source DataFrame is not
145
+ modified.
146
+
147
+ ```python
148
+ import pandas as pd
149
+ from ramen_data_filter import FiltrationMode, filter_dataframe
150
+
151
+ records = pd.DataFrame(
152
+ [
153
+ {"document_id": "doc-1", "content": "Approved public documentation"},
154
+ {"document_id": "doc-2", "content": "Retrieved untrusted context"},
155
+ ]
156
+ )
157
+
158
+ result = filter_dataframe(
159
+ records,
160
+ mode=FiltrationMode.STRICT_EXCLUSION,
161
+ bundle_ids=["ramen__shield_core_it"],
162
+ )
163
+
164
+ clean_records = result.dataframe
165
+ print(result.audit_log[["row_index", "verdict", "receipt_verified"]])
166
+ ```
167
+
168
+ ### Semantic Imputation
169
+
170
+ Semantic imputation preserves every row. A blocked row may change only a column
171
+ that is both listed in `remediable_columns` and named in the ramen-ai steering
172
+ instruction. Numeric columns receive the allowed-reference mean (rounded for
173
+ integer data); non-numeric columns use deterministic hot-deck values from
174
+ allowed rows. All other values remain unchanged.
175
+
176
+ ```python
177
+ result = filter_dataframe(
178
+ records,
179
+ mode=FiltrationMode.SEMANTIC_IMPUTATION,
180
+ policy_ids=["<POLICY_UUID>"],
181
+ remediable_columns=["content"],
182
+ )
183
+
184
+ sanitized_records = result.dataframe
185
+ print(result.imputation_log)
186
+ ```
187
+
188
+ ### CSV pipeline
189
+
190
+ ```python
191
+ from ramen_data_filter import FiltrationMode, filter_csv
192
+
193
+ result = filter_csv(
194
+ "rag-ingestion.csv",
195
+ "rag-ingestion-sanitized.csv",
196
+ mode=FiltrationMode.STRICT_EXCLUSION,
197
+ bundle_ids=["ramen__shield_core_it"],
198
+ )
199
+ ```
200
+
201
+ The destination is written only after every row has been evaluated and the
202
+ transformation succeeds. Existing destination files are replaced by Pandas.
203
+
204
+ ---
205
+
206
+ ## How it works
207
+
208
+ 1. Serializes each row as stable, key-sorted JSON.
209
+ 2. Calls `RamenClient.evaluate_compliance` with configured bundles or policies.
210
+ 3. Records the verdict, steering, resolved policy IDs, receipt verification
211
+ state, and complete SDK response in `audit_log`.
212
+ 4. Applies the selected mode only after all rows have been evaluated.
213
+ 5. Returns a new DataFrame and transformation metadata; CSV usage writes that
214
+ DataFrame with `index=False`.
215
+
216
+ The pipeline is fail-closed. Missing credentials, transport failures, malformed
217
+ responses, unsafe imputation steering, and unusable reference values raise
218
+ `FiltrationError`; partial filtered output is never returned or written.
219
+
220
+ ---
221
+
222
+ ## API reference
223
+
224
+ ### `filter_dataframe(dataframe, *, mode, ...) -> FiltrationResult`
225
+
226
+ | Parameter | Type | Description |
227
+ |---|---|---|
228
+ | `dataframe` | `pandas.DataFrame` | Source records. The input is not modified. |
229
+ | `mode` | `FiltrationMode \| str` | `strict_exclusion` or `semantic_imputation`. |
230
+ | `bundle_ids` | `Sequence[str]` | ramen-ai bundle slugs. At least one bundle or policy is required. |
231
+ | `policy_ids` | `Sequence[str]` | Explicit policy UUIDs. May be combined with bundles. |
232
+ | `remediable_columns` | `Sequence[str]` | Columns eligible for steering-constrained replacement. Required for semantic imputation. |
233
+ | `client` | `RamenClient` | Optional injected client, primarily for controlled runtimes and tests. |
234
+ | `api_key` | `str` | Runtime override; defaults to `RAMEN_API_KEY`. |
235
+ | `provider_key` | `str` | BYOK override; defaults to `OPENAI_API_KEY`. |
236
+ | `provider_name` | `str` | Optional BYOK provider routing hint. |
237
+ | `context` | `dict[str, str]` | Additional audit metadata. |
238
+ | `base_url` | `str` | ramen-ai API base URL. |
239
+ | `timeout` | `float` | HTTP timeout in seconds. |
240
+
241
+ ### `filter_csv(source_path, destination_path, **kwargs) -> FiltrationResult`
242
+
243
+ Reads the source with `pandas.read_csv`, applies `filter_dataframe`, and writes
244
+ the resulting DataFrame with `index=False`.
245
+
246
+ ### `FiltrationResult`
247
+
248
+ | Attribute | Description |
249
+ |---|---|
250
+ | `dataframe` | Filtered or imputed DataFrame. |
251
+ | `audit_log` | One row per evaluation with verdict and SDK response metadata. |
252
+ | `imputation_log` | One row per transformed record; empty in strict mode. |
253
+
254
+ Public exports: `FiltrationError`, `FiltrationMode`, `FiltrationResult`,
255
+ `filter_dataframe`, and `filter_csv`.
256
+
257
+ ---
258
+
259
+ ## Running the tests
260
+
261
+ ```bash
262
+ PYTHONPATH="core-clients/python:plugins/ramen-data-filter/src" \
263
+ python -m pytest plugins/ramen-data-filter/tests -v
264
+ ```
265
+
266
+ The isolated tests use a mocked `RamenClient`; they make no network calls and
267
+ require no credentials. They prove blocked-row exclusion and constrained
268
+ semantic imputation on a dummy DataFrame.
269
+
270
+ ---
271
+
272
+ ## Available bundles
273
+
274
+ | Bundle slug | Useful dataset coverage |
275
+ |---|---|
276
+ | `ramen__shield_core_it` | Secret exfiltration, prompt leakage, jailbreaks, destructive instructions, and indirect prompt injection. |
277
+ | `ramen__eu_ai_act_baseline` | EU AI Act data-governance, prohibited-practice, and transparency controls. |
278
+
279
+ Pass explicit `policy_ids` for custom PII, ingestion, data-quality, or internal
280
+ MLOps controls. Bundle and policy details are available at
281
+ [ramenai.dev/pricing](https://ramenai.dev/pricing).
282
+
283
+ ---
284
+
285
+ ## Limitations
286
+
287
+ - Evaluation adds one API request per row. Large datasets should be processed in
288
+ controlled batches with rate limits and resumability at the orchestration
289
+ layer.
290
+ - Semantic imputation creates synthetic values; it does not recover original
291
+ truth or establish legal compliance, statistical fairness, or model quality.
292
+ - Imputed rows are not automatically re-evaluated. Validate and, where required,
293
+ re-evaluate transformed output before production ingestion or training.
294
+ - Non-numeric hot-deck replacement is deterministic and may alter dataset
295
+ distributions. Preserve source data and review `audit_log` and
296
+ `imputation_log` in an access-controlled audit store.
@@ -0,0 +1,284 @@
1
+ # ramen-ai Data Filter
2
+
3
+ <p align="center">
4
+ <img src="../../assets/ramen-logo.png" alt="ramen-ai" width="100"/>
5
+ </p>
6
+
7
+ Dual-mode Pandas and CSV filtration for RAG ingestion scrubbing and MLOps
8
+ dataset sanitization. Every row is evaluated through the ramen-ai semantic
9
+ firewall before it can enter a retrieval corpus, training set, analytics store,
10
+ or downstream data pipeline.
11
+
12
+ Use **Strict Exclusion** to remove blocked rows, or **Semantic Imputation** to
13
+ preserve row count while replacing only explicitly allowlisted columns named by
14
+ ramen-ai steering. Typical policies can detect PII, data-exfiltration vectors,
15
+ poisoned retrieval context, prompt injection, and other unsafe dataset content.
16
+
17
+ Requires **Python ≥ 3.10**.
18
+
19
+ ---
20
+
21
+ ## Supported Ecosystems
22
+
23
+ <p align="center">
24
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/langchain-python">
25
+ <img src="https://img.shields.io/badge/LangChain-1C3C3C?style=flat&logo=langchain&logoColor=white" alt="LangChain"/>
26
+ </a>
27
+ &nbsp;
28
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/pydantic-ai">
29
+ <img src="https://img.shields.io/badge/PydanticAI-E92063?style=flat&logo=pydantic&logoColor=white" alt="PydanticAI"/>
30
+ </a>
31
+ &nbsp;
32
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/mcp-proxy">
33
+ <img src="https://img.shields.io/badge/MCP-6B21A8?style=flat&logo=anthropic&logoColor=white" alt="MCP"/>
34
+ </a>
35
+ &nbsp;
36
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/agt-typescript">
37
+ <img src="https://img.shields.io/badge/Microsoft%20AGT-0078D4?style=flat&logo=microsoft&logoColor=white" alt="Microsoft AGT"/>
38
+ </a>
39
+ &nbsp;
40
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/github-action">
41
+ <img src="https://img.shields.io/badge/GitHub%20Actions-2088FF?style=flat&logo=githubactions&logoColor=white" alt="GitHub Actions"/>
42
+ </a>
43
+ &nbsp;
44
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/cmcp-python">
45
+ <img src="https://img.shields.io/badge/cMCP-00A67E?style=flat&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZmlsbD0id2hpdGUiIGQ9Ik0xMiAyTDIgN2wxMCA1IDEwLTV6TTIgMTdsOCA0VjExbC04LTR6TTE0IDIxbDgtNFYxMWwtOCA0eiIvPjwvc3ZnPg==&logoColor=white" alt="cMCP"/>
46
+ </a>
47
+ &nbsp;
48
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/mlflow-python">
49
+ <img src="https://img.shields.io/badge/MLflow-0194E2?style=flat&logo=mlflow&logoColor=white" alt="MLflow"/>
50
+ </a>
51
+ &nbsp;
52
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/ramen-data-filter">
53
+ <img src="https://img.shields.io/badge/ramen%20data%20filter-D97706?style=flat&logo=pandas&logoColor=white" alt="ramen data filter"/>
54
+ </a>
55
+ </p>
56
+
57
+ ---
58
+
59
+ ## Can you bypass it?
60
+
61
+ Standard safety filters catch basic syntax. They fail against encoded payloads
62
+ and corporate jargon. We challenge you to bypass our semantic firewall using
63
+ the zero-day evasion vectors in our official **[Red Team Guide](../../RED_TEAM_GUIDE.md)**.
64
+
65
+ Below is a simulation of the Grok/Bankr heist. We fed the raw adversarial
66
+ prompt directly into our sandbox. It uses a social engineering wrapper
67
+ (claiming a visual impairment) to smuggle a 3,000,000,000 DRB transfer
68
+ instruction encoded in Morse code. The firewall evaluated the underlying
69
+ semantic intent, intercepted the unauthorized financial transfer, and blocked
70
+ it pre-execution, issuing a verified Ed25519 receipt.
71
+
72
+ <p align="center">
73
+ <img src="../../assets/grok-bankr.png" alt="ramen-ai intercepting the Grok/Bankr Morse-code heist pre-execution" width="720"/>
74
+ </p>
75
+
76
+ ---
77
+
78
+ ## API Key
79
+
80
+ To use this integration, you must mint an API Key. We offer a **Free Starter
81
+ Tier** (1,000 evaluations/month, BYOK) which includes full access to our Core
82
+ IT Security bundle. Mint your key at:
83
+ **[https://ramenai.dev/pricing](https://ramenai.dev/pricing)**
84
+
85
+ ---
86
+
87
+ ## Installation
88
+
89
+ From PyPI after release:
90
+
91
+ ```bash
92
+ pip install ramen-data-filter
93
+ ```
94
+
95
+ From this monorepo:
96
+
97
+ ```bash
98
+ pip install -e plugins/ramen-data-filter
99
+ pip install -e "plugins/ramen-data-filter[dev]" # include pytest
100
+ ```
101
+
102
+ ---
103
+
104
+ ## Configuration / BYOK
105
+
106
+ ```bash
107
+ export RAMEN_API_KEY=ramen_ak_... # required
108
+ export OPENAI_API_KEY=sk-... # Starter/Professional BYOK
109
+ ```
110
+
111
+ `RAMEN_API_KEY` authenticates the evaluation request. The Starter and
112
+ Professional tiers also require an LLM provider key. The plugin reads
113
+ `OPENAI_API_KEY` at evaluation time and `ramen-ai-core` forwards it as the
114
+ `X-Provider-Key` header. Without a provider key, those tiers return
115
+ `402 Payment Required`.
116
+
117
+ Pass `provider_name="anthropic"`, `"google"`, `"synthetic"`, or
118
+ `"hyperbolic"` with the corresponding `provider_key` when OpenAI is not the
119
+ provider. Enterprise tiers use platform-managed provider keys and do not need
120
+ `OPENAI_API_KEY`.
121
+
122
+ Credentials are read from environment variables or passed at runtime. Never
123
+ store them in source code, notebooks, CSV files, or committed configuration.
124
+
125
+ ---
126
+
127
+ ## Usage
128
+
129
+ ### Strict Exclusion
130
+
131
+ Strict exclusion evaluates each row and returns a new DataFrame containing only
132
+ `[ALLOWED]` rows. The returned index is reset; the source DataFrame is not
133
+ modified.
134
+
135
+ ```python
136
+ import pandas as pd
137
+ from ramen_data_filter import FiltrationMode, filter_dataframe
138
+
139
+ records = pd.DataFrame(
140
+ [
141
+ {"document_id": "doc-1", "content": "Approved public documentation"},
142
+ {"document_id": "doc-2", "content": "Retrieved untrusted context"},
143
+ ]
144
+ )
145
+
146
+ result = filter_dataframe(
147
+ records,
148
+ mode=FiltrationMode.STRICT_EXCLUSION,
149
+ bundle_ids=["ramen__shield_core_it"],
150
+ )
151
+
152
+ clean_records = result.dataframe
153
+ print(result.audit_log[["row_index", "verdict", "receipt_verified"]])
154
+ ```
155
+
156
+ ### Semantic Imputation
157
+
158
+ Semantic imputation preserves every row. A blocked row may change only a column
159
+ that is both listed in `remediable_columns` and named in the ramen-ai steering
160
+ instruction. Numeric columns receive the allowed-reference mean (rounded for
161
+ integer data); non-numeric columns use deterministic hot-deck values from
162
+ allowed rows. All other values remain unchanged.
163
+
164
+ ```python
165
+ result = filter_dataframe(
166
+ records,
167
+ mode=FiltrationMode.SEMANTIC_IMPUTATION,
168
+ policy_ids=["<POLICY_UUID>"],
169
+ remediable_columns=["content"],
170
+ )
171
+
172
+ sanitized_records = result.dataframe
173
+ print(result.imputation_log)
174
+ ```
175
+
176
+ ### CSV pipeline
177
+
178
+ ```python
179
+ from ramen_data_filter import FiltrationMode, filter_csv
180
+
181
+ result = filter_csv(
182
+ "rag-ingestion.csv",
183
+ "rag-ingestion-sanitized.csv",
184
+ mode=FiltrationMode.STRICT_EXCLUSION,
185
+ bundle_ids=["ramen__shield_core_it"],
186
+ )
187
+ ```
188
+
189
+ The destination is written only after every row has been evaluated and the
190
+ transformation succeeds. Existing destination files are replaced by Pandas.
191
+
192
+ ---
193
+
194
+ ## How it works
195
+
196
+ 1. Serializes each row as stable, key-sorted JSON.
197
+ 2. Calls `RamenClient.evaluate_compliance` with configured bundles or policies.
198
+ 3. Records the verdict, steering, resolved policy IDs, receipt verification
199
+ state, and complete SDK response in `audit_log`.
200
+ 4. Applies the selected mode only after all rows have been evaluated.
201
+ 5. Returns a new DataFrame and transformation metadata; CSV usage writes that
202
+ DataFrame with `index=False`.
203
+
204
+ The pipeline is fail-closed. Missing credentials, transport failures, malformed
205
+ responses, unsafe imputation steering, and unusable reference values raise
206
+ `FiltrationError`; partial filtered output is never returned or written.
207
+
208
+ ---
209
+
210
+ ## API reference
211
+
212
+ ### `filter_dataframe(dataframe, *, mode, ...) -> FiltrationResult`
213
+
214
+ | Parameter | Type | Description |
215
+ |---|---|---|
216
+ | `dataframe` | `pandas.DataFrame` | Source records. The input is not modified. |
217
+ | `mode` | `FiltrationMode \| str` | `strict_exclusion` or `semantic_imputation`. |
218
+ | `bundle_ids` | `Sequence[str]` | ramen-ai bundle slugs. At least one bundle or policy is required. |
219
+ | `policy_ids` | `Sequence[str]` | Explicit policy UUIDs. May be combined with bundles. |
220
+ | `remediable_columns` | `Sequence[str]` | Columns eligible for steering-constrained replacement. Required for semantic imputation. |
221
+ | `client` | `RamenClient` | Optional injected client, primarily for controlled runtimes and tests. |
222
+ | `api_key` | `str` | Runtime override; defaults to `RAMEN_API_KEY`. |
223
+ | `provider_key` | `str` | BYOK override; defaults to `OPENAI_API_KEY`. |
224
+ | `provider_name` | `str` | Optional BYOK provider routing hint. |
225
+ | `context` | `dict[str, str]` | Additional audit metadata. |
226
+ | `base_url` | `str` | ramen-ai API base URL. |
227
+ | `timeout` | `float` | HTTP timeout in seconds. |
228
+
229
+ ### `filter_csv(source_path, destination_path, **kwargs) -> FiltrationResult`
230
+
231
+ Reads the source with `pandas.read_csv`, applies `filter_dataframe`, and writes
232
+ the resulting DataFrame with `index=False`.
233
+
234
+ ### `FiltrationResult`
235
+
236
+ | Attribute | Description |
237
+ |---|---|
238
+ | `dataframe` | Filtered or imputed DataFrame. |
239
+ | `audit_log` | One row per evaluation with verdict and SDK response metadata. |
240
+ | `imputation_log` | One row per transformed record; empty in strict mode. |
241
+
242
+ Public exports: `FiltrationError`, `FiltrationMode`, `FiltrationResult`,
243
+ `filter_dataframe`, and `filter_csv`.
244
+
245
+ ---
246
+
247
+ ## Running the tests
248
+
249
+ ```bash
250
+ PYTHONPATH="core-clients/python:plugins/ramen-data-filter/src" \
251
+ python -m pytest plugins/ramen-data-filter/tests -v
252
+ ```
253
+
254
+ The isolated tests use a mocked `RamenClient`; they make no network calls and
255
+ require no credentials. They prove blocked-row exclusion and constrained
256
+ semantic imputation on a dummy DataFrame.
257
+
258
+ ---
259
+
260
+ ## Available bundles
261
+
262
+ | Bundle slug | Useful dataset coverage |
263
+ |---|---|
264
+ | `ramen__shield_core_it` | Secret exfiltration, prompt leakage, jailbreaks, destructive instructions, and indirect prompt injection. |
265
+ | `ramen__eu_ai_act_baseline` | EU AI Act data-governance, prohibited-practice, and transparency controls. |
266
+
267
+ Pass explicit `policy_ids` for custom PII, ingestion, data-quality, or internal
268
+ MLOps controls. Bundle and policy details are available at
269
+ [ramenai.dev/pricing](https://ramenai.dev/pricing).
270
+
271
+ ---
272
+
273
+ ## Limitations
274
+
275
+ - Evaluation adds one API request per row. Large datasets should be processed in
276
+ controlled batches with rate limits and resumability at the orchestration
277
+ layer.
278
+ - Semantic imputation creates synthetic values; it does not recover original
279
+ truth or establish legal compliance, statistical fairness, or model quality.
280
+ - Imputed rows are not automatically re-evaluated. Validate and, where required,
281
+ re-evaluate transformed output before production ingestion or training.
282
+ - Non-numeric hot-deck replacement is deterministic and may alter dataset
283
+ distributions. Preserve source data and review `audit_log` and
284
+ `imputation_log` in an access-controlled audit store.
@@ -0,0 +1,34 @@
1
+ """Filter a CSV through the ramen-ai dataset filtration boundary."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+
7
+ from ramen_data_filter import FiltrationMode, filter_csv
8
+
9
+
10
+ def main() -> None:
11
+ parser = argparse.ArgumentParser()
12
+ parser.add_argument("source")
13
+ parser.add_argument("destination")
14
+ parser.add_argument(
15
+ "--mode",
16
+ choices=[mode.value for mode in FiltrationMode],
17
+ default=FiltrationMode.STRICT_EXCLUSION.value,
18
+ )
19
+ parser.add_argument("--bundle", action="append", required=True)
20
+ parser.add_argument("--remediable-column", action="append", default=[])
21
+ args = parser.parse_args()
22
+
23
+ result = filter_csv(
24
+ args.source,
25
+ args.destination,
26
+ mode=args.mode,
27
+ bundle_ids=args.bundle,
28
+ remediable_columns=args.remediable_column,
29
+ )
30
+ print(result.audit_log["verdict"].value_counts().to_dict())
31
+
32
+
33
+ if __name__ == "__main__":
34
+ main()
@@ -0,0 +1,26 @@
1
+ [build-system]
2
+ requires = ["hatchling==1.27.0"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "ramen-data-filter"
7
+ version = "0.1.0"
8
+ description = "Dual-mode Pandas and CSV filtration for ramen-ai RAG ingestion and MLOps dataset sanitization."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = { text = "Proprietary" }
12
+ dependencies = [
13
+ "pandas==2.3.3",
14
+ "ramen-ai-core==0.2.1",
15
+ ]
16
+
17
+ [project.optional-dependencies]
18
+ dev = [
19
+ "pytest==8.4.2",
20
+ ]
21
+
22
+ [tool.hatch.build.targets.wheel]
23
+ packages = ["src/ramen_data_filter"]
24
+
25
+ [tool.pytest.ini_options]
26
+ testpaths = ["tests"]
@@ -0,0 +1,18 @@
1
+ """ramen-ai filtration for RAG ingestion and MLOps datasets."""
2
+
3
+ from .filter import (
4
+ FiltrationError,
5
+ FiltrationMode,
6
+ FiltrationResult,
7
+ filter_csv,
8
+ filter_dataframe,
9
+ )
10
+
11
+ __all__ = [
12
+ "FiltrationError",
13
+ "FiltrationMode",
14
+ "FiltrationResult",
15
+ "filter_csv",
16
+ "filter_dataframe",
17
+ ]
18
+ __version__ = "0.1.0"
@@ -0,0 +1,362 @@
1
+ """Dual-mode DataFrame and CSV filtration through the ramen-ai API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import re
8
+ import tempfile
9
+ from collections.abc import Mapping
10
+ from dataclasses import dataclass
11
+ from enum import Enum
12
+ from pathlib import Path
13
+ from typing import Any, Sequence
14
+
15
+ import pandas as pd
16
+ from pandas.api.types import is_integer_dtype, is_numeric_dtype
17
+ from ramen_ai import RamenClient
18
+
19
+
20
+ class FiltrationMode(str, Enum):
21
+ """Supported dataset filtration modes."""
22
+
23
+ STRICT_EXCLUSION = "strict_exclusion"
24
+ SEMANTIC_IMPUTATION = "semantic_imputation"
25
+
26
+
27
+ class FiltrationError(RuntimeError):
28
+ """Raised when a dataset cannot be evaluated or safely transformed."""
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class FiltrationResult:
33
+ """Filtered data plus row-level evaluation and transformation records."""
34
+
35
+ dataframe: pd.DataFrame
36
+ audit_log: pd.DataFrame
37
+ imputation_log: pd.DataFrame
38
+
39
+
40
+ _AUDIT_COLUMNS = (
41
+ "row_position",
42
+ "row_index",
43
+ "allowed",
44
+ "verdict",
45
+ "steering",
46
+ "receipt_verified",
47
+ "policy_ids",
48
+ "response",
49
+ )
50
+ _IMPUTATION_COLUMNS = (
51
+ "row_position",
52
+ "row_index",
53
+ "columns_changed",
54
+ "steering",
55
+ )
56
+
57
+
58
+ def filter_dataframe(
59
+ dataframe: pd.DataFrame,
60
+ *,
61
+ mode: FiltrationMode | str,
62
+ bundle_ids: Sequence[str] | None = None,
63
+ policy_ids: Sequence[str] | None = None,
64
+ remediable_columns: Sequence[str] | None = None,
65
+ client: RamenClient | None = None,
66
+ api_key: str | None = None,
67
+ provider_key: str | None = None,
68
+ provider_name: str | None = None,
69
+ context: dict[str, str] | None = None,
70
+ base_url: str = "https://api.ramenai.dev",
71
+ timeout: float = 30.0,
72
+ ) -> FiltrationResult:
73
+ """Evaluate and filter a DataFrame row by row.
74
+
75
+ Strict exclusion retains only rows with an allowed verdict. Semantic
76
+ imputation keeps every row but changes only allowlisted columns named in a
77
+ blocked row's steering instruction. Numeric replacements use the allowed
78
+ reference mean (rounded for integer columns); other values use deterministic
79
+ hot-deck imputation from allowed rows.
80
+
81
+ API and transport failures stop processing. Partial output is never
82
+ returned or written.
83
+ """
84
+ if not isinstance(dataframe, pd.DataFrame):
85
+ raise TypeError("dataframe must be a pandas DataFrame")
86
+ if not dataframe.columns.is_unique:
87
+ raise ValueError("dataframe columns must be unique")
88
+ try:
89
+ selected_mode = FiltrationMode(mode)
90
+ except ValueError as exc:
91
+ supported = ", ".join(item.value for item in FiltrationMode)
92
+ raise ValueError(f"mode must be one of: {supported}") from exc
93
+
94
+ selected_bundles = _validated_identifiers("bundle_ids", bundle_ids)
95
+ selected_policies = _validated_identifiers("policy_ids", policy_ids)
96
+ if not selected_bundles and not selected_policies:
97
+ raise ValueError("Provide at least one of 'bundle_ids' or 'policy_ids'.")
98
+
99
+ allowed_columns = _validated_columns(dataframe, remediable_columns)
100
+ if selected_mode is FiltrationMode.SEMANTIC_IMPUTATION and not allowed_columns:
101
+ raise ValueError(
102
+ "semantic_imputation requires at least one remediable column"
103
+ )
104
+
105
+ evaluation_context = dict(context or {})
106
+ if not all(
107
+ isinstance(key, str) and isinstance(value, str)
108
+ for key, value in evaluation_context.items()
109
+ ):
110
+ raise TypeError("context keys and values must be strings")
111
+ evaluation_context["integration"] = "ramen-data-filter"
112
+ evaluation_context.setdefault("source", "dataframe")
113
+
114
+ owned_client: RamenClient | None = None
115
+ active_client = client
116
+ if active_client is None:
117
+ resolved_api_key = api_key or os.environ.get("RAMEN_API_KEY", "")
118
+ if not resolved_api_key:
119
+ raise FiltrationError(
120
+ "RAMEN_API_KEY is not set; dataset filtration cannot be evaluated"
121
+ )
122
+ owned_client = RamenClient(
123
+ api_key=resolved_api_key,
124
+ base_url=base_url,
125
+ timeout=timeout,
126
+ )
127
+ active_client = owned_client
128
+
129
+ records: list[dict[str, Any]] = []
130
+ try:
131
+ for position, (row_index, row) in enumerate(dataframe.iterrows()):
132
+ serialized_row = json.dumps(
133
+ row.to_dict(), sort_keys=True, default=str
134
+ )
135
+ try:
136
+ response = active_client.evaluate_compliance(
137
+ input_text=serialized_row,
138
+ bundle_ids=selected_bundles or None,
139
+ policy_ids=selected_policies or None,
140
+ context=evaluation_context,
141
+ provider_key=(
142
+ provider_key
143
+ if provider_key is not None
144
+ else os.environ.get("OPENAI_API_KEY")
145
+ ),
146
+ provider_name=provider_name,
147
+ )
148
+ except Exception as exc:
149
+ raise FiltrationError(
150
+ f"ramen-ai evaluation failed for row position {position}"
151
+ ) from exc
152
+
153
+ if not isinstance(response, Mapping):
154
+ raise FiltrationError(
155
+ f"ramen-ai response was not a mapping for row position {position}"
156
+ )
157
+ allowed = response.get("allowed")
158
+ if not isinstance(allowed, bool):
159
+ raise FiltrationError(
160
+ "ramen-ai response omitted a boolean allowed verdict "
161
+ f"for row position {position}"
162
+ )
163
+ steering = response.get("steering")
164
+ if steering is not None and not isinstance(steering, str):
165
+ raise FiltrationError(
166
+ f"ramen-ai steering was not text for row position {position}"
167
+ )
168
+ receipt_verified = response.get("receipt_verified", False)
169
+ if not isinstance(receipt_verified, bool):
170
+ raise FiltrationError(
171
+ "ramen-ai receipt verification state was not boolean "
172
+ f"for row position {position}"
173
+ )
174
+ resolved_policy_ids = response.get("policy_ids", [])
175
+ if not isinstance(resolved_policy_ids, list) or not all(
176
+ isinstance(policy_id, str) and policy_id
177
+ for policy_id in resolved_policy_ids
178
+ ):
179
+ raise FiltrationError(
180
+ "ramen-ai policy_ids was not a list of non-empty strings "
181
+ f"for row position {position}"
182
+ )
183
+ records.append(
184
+ {
185
+ "row_position": position,
186
+ "row_index": row_index,
187
+ "allowed": allowed,
188
+ "verdict": "[ALLOWED]" if allowed else "[BLOCKED]",
189
+ "steering": steering,
190
+ "receipt_verified": receipt_verified,
191
+ "policy_ids": resolved_policy_ids.copy(),
192
+ "response": dict(response),
193
+ }
194
+ )
195
+ finally:
196
+ if owned_client is not None:
197
+ owned_client.close()
198
+
199
+ audit_log = pd.DataFrame(records, columns=_AUDIT_COLUMNS)
200
+ allowed_positions = [
201
+ record["row_position"] for record in records if record["allowed"]
202
+ ]
203
+
204
+ if selected_mode is FiltrationMode.STRICT_EXCLUSION:
205
+ filtered = dataframe.iloc[allowed_positions].copy().reset_index(drop=True)
206
+ return FiltrationResult(
207
+ dataframe=filtered,
208
+ audit_log=audit_log,
209
+ imputation_log=pd.DataFrame(columns=_IMPUTATION_COLUMNS),
210
+ )
211
+
212
+ imputed, imputation_log = _impute_blocked_rows(
213
+ dataframe=dataframe,
214
+ records=records,
215
+ allowed_positions=allowed_positions,
216
+ remediable_columns=allowed_columns,
217
+ )
218
+ return FiltrationResult(
219
+ dataframe=imputed,
220
+ audit_log=audit_log,
221
+ imputation_log=imputation_log,
222
+ )
223
+
224
+
225
+ def filter_csv(
226
+ source_path: str | Path,
227
+ destination_path: str | Path,
228
+ **kwargs: Any,
229
+ ) -> FiltrationResult:
230
+ """Read a CSV, filter it through :func:`filter_dataframe`, and write output."""
231
+ source = Path(source_path)
232
+ destination = Path(destination_path)
233
+ dataframe = pd.read_csv(source)
234
+ context = dict(kwargs.pop("context", {}) or {})
235
+ context["source"] = "csv"
236
+ result = filter_dataframe(dataframe, context=context, **kwargs)
237
+
238
+ temporary_path: Path | None = None
239
+ try:
240
+ with tempfile.NamedTemporaryFile(
241
+ mode="w",
242
+ dir=destination.parent,
243
+ prefix=f".{destination.name}.",
244
+ suffix=".tmp",
245
+ delete=False,
246
+ ) as temporary_file:
247
+ temporary_path = Path(temporary_file.name)
248
+ result.dataframe.to_csv(temporary_path, index=False)
249
+ os.replace(temporary_path, destination)
250
+ finally:
251
+ if temporary_path is not None and temporary_path.exists():
252
+ temporary_path.unlink()
253
+ return result
254
+
255
+
256
+ def _validated_identifiers(
257
+ name: str, values: Sequence[str] | None
258
+ ) -> list[str]:
259
+ if values is None:
260
+ return []
261
+ if isinstance(values, (str, bytes)):
262
+ raise TypeError(f"{name} must be a sequence of non-empty strings")
263
+ normalized = list(values)
264
+ if not all(isinstance(value, str) and value.strip() for value in normalized):
265
+ raise ValueError(f"{name} must contain only non-empty strings")
266
+ return normalized
267
+
268
+
269
+ def _validated_columns(
270
+ dataframe: pd.DataFrame, columns: Sequence[str] | None
271
+ ) -> list[str]:
272
+ if columns is None:
273
+ return []
274
+ if isinstance(columns, (str, bytes)):
275
+ raise TypeError("remediable_columns must be a sequence of column names")
276
+ normalized = list(dict.fromkeys(columns))
277
+ if not all(isinstance(column, str) and column for column in normalized):
278
+ raise ValueError("remediable_columns must contain non-empty strings")
279
+ missing = [column for column in normalized if column not in dataframe.columns]
280
+ if missing:
281
+ raise ValueError(f"remediable columns are missing from dataframe: {missing}")
282
+ casefolded = [column.casefold() for column in normalized]
283
+ if len(casefolded) != len(set(casefolded)):
284
+ raise ValueError("remediable column names must be unique ignoring case")
285
+ return normalized
286
+
287
+
288
+ def _impute_blocked_rows(
289
+ *,
290
+ dataframe: pd.DataFrame,
291
+ records: list[dict[str, Any]],
292
+ allowed_positions: list[int],
293
+ remediable_columns: list[str],
294
+ ) -> tuple[pd.DataFrame, pd.DataFrame]:
295
+ blocked_records = [record for record in records if not record["allowed"]]
296
+ if not blocked_records:
297
+ return dataframe.copy(), pd.DataFrame(columns=_IMPUTATION_COLUMNS)
298
+ if not allowed_positions:
299
+ raise FiltrationError(
300
+ "semantic imputation requires at least one allowed reference row"
301
+ )
302
+
303
+ reference = dataframe.iloc[allowed_positions]
304
+ replacements = {
305
+ column: _replacement_values(reference[column], column)
306
+ for column in remediable_columns
307
+ }
308
+ imputed = dataframe.copy()
309
+ log_records: list[dict[str, Any]] = []
310
+
311
+ for blocked_ordinal, record in enumerate(blocked_records):
312
+ steering = record["steering"] or ""
313
+ selected_columns = [
314
+ column
315
+ for column in remediable_columns
316
+ if re.search(
317
+ rf"(?<![\w]){re.escape(column)}(?![\w])",
318
+ steering,
319
+ flags=re.IGNORECASE,
320
+ )
321
+ ]
322
+ if not selected_columns:
323
+ raise FiltrationError(
324
+ "blocked row did not receive steering naming an allowlisted "
325
+ f"column at row position {record['row_position']}"
326
+ )
327
+
328
+ for column in selected_columns:
329
+ values = replacements[column]
330
+ replacement = values[blocked_ordinal % len(values)]
331
+ column_position = dataframe.columns.get_loc(column)
332
+ imputed.iat[record["row_position"], column_position] = replacement
333
+
334
+ log_records.append(
335
+ {
336
+ "row_position": record["row_position"],
337
+ "row_index": record["row_index"],
338
+ "columns_changed": tuple(selected_columns),
339
+ "steering": steering,
340
+ }
341
+ )
342
+
343
+ return imputed, pd.DataFrame(log_records, columns=_IMPUTATION_COLUMNS)
344
+
345
+
346
+ def _replacement_values(series: pd.Series, column: str) -> list[Any]:
347
+ non_null = series.dropna()
348
+ if non_null.empty:
349
+ raise FiltrationError(
350
+ f"allowed reference rows contain no values for '{column}'"
351
+ )
352
+ if is_numeric_dtype(non_null.dtype):
353
+ mean = non_null.mean()
354
+ if pd.isna(mean):
355
+ raise FiltrationError(
356
+ f"allowed reference mean could not be calculated for '{column}'"
357
+ )
358
+ replacement: Any = float(mean)
359
+ if is_integer_dtype(non_null.dtype):
360
+ replacement = int(round(replacement))
361
+ return [replacement]
362
+ return non_null.tolist()
@@ -0,0 +1,197 @@
1
+ """Isolated tests for ramen-ai dataset filtration."""
2
+
3
+ from unittest.mock import Mock
4
+
5
+ import pandas as pd
6
+ import pytest
7
+
8
+ from ramen_ai import RamenClient
9
+ from ramen_data_filter import (
10
+ FiltrationError,
11
+ FiltrationMode,
12
+ filter_csv,
13
+ filter_dataframe,
14
+ )
15
+
16
+
17
+ def _dataset() -> pd.DataFrame:
18
+ return pd.DataFrame(
19
+ [
20
+ {"record_id": "safe-1", "email": "clean@example.com", "risk_score": 10},
21
+ {"record_id": "blocked", "email": "export@attacker.test", "risk_score": 99},
22
+ {"record_id": "safe-2", "email": "review@example.com", "risk_score": 30},
23
+ ]
24
+ )
25
+
26
+
27
+ def _mock_client() -> Mock:
28
+ client = Mock(spec=RamenClient)
29
+ client.evaluate_compliance.side_effect = [
30
+ {
31
+ "allowed": True,
32
+ "steering": None,
33
+ "receipt_verified": True,
34
+ "policy_ids": ["policy-1"],
35
+ "data": {},
36
+ },
37
+ {
38
+ "allowed": False,
39
+ "steering": "Replace email and risk_score before ingestion.",
40
+ "receipt_verified": True,
41
+ "policy_ids": ["policy-1"],
42
+ "data": {},
43
+ },
44
+ {
45
+ "allowed": True,
46
+ "steering": None,
47
+ "receipt_verified": True,
48
+ "policy_ids": ["policy-1"],
49
+ "data": {},
50
+ },
51
+ ]
52
+ return client
53
+
54
+
55
+ def test_strict_exclusion_removes_blocked_rows() -> None:
56
+ client = _mock_client()
57
+
58
+ result = filter_dataframe(
59
+ _dataset(),
60
+ mode=FiltrationMode.STRICT_EXCLUSION,
61
+ policy_ids=["policy-1"],
62
+ client=client,
63
+ )
64
+
65
+ assert result.dataframe["record_id"].tolist() == ["safe-1", "safe-2"]
66
+ assert result.dataframe.index.tolist() == [0, 1]
67
+ assert result.audit_log["verdict"].tolist() == [
68
+ "[ALLOWED]",
69
+ "[BLOCKED]",
70
+ "[ALLOWED]",
71
+ ]
72
+ assert result.imputation_log.empty
73
+ assert client.evaluate_compliance.call_count == 3
74
+
75
+
76
+ def test_semantic_imputation_repairs_only_allowlisted_columns() -> None:
77
+ client = _mock_client()
78
+ source = _dataset()
79
+
80
+ result = filter_dataframe(
81
+ source,
82
+ mode=FiltrationMode.SEMANTIC_IMPUTATION,
83
+ policy_ids=["policy-1"],
84
+ remediable_columns=["email", "risk_score"],
85
+ client=client,
86
+ )
87
+
88
+ assert len(result.dataframe) == len(source)
89
+ assert result.dataframe.loc[1, "email"] == "clean@example.com"
90
+ assert result.dataframe.loc[1, "risk_score"] == 20
91
+ assert result.dataframe.loc[1, "record_id"] == "blocked"
92
+ pd.testing.assert_series_equal(result.dataframe.loc[0], source.loc[0])
93
+ pd.testing.assert_series_equal(result.dataframe.loc[2], source.loc[2])
94
+ assert result.imputation_log.loc[0, "columns_changed"] == (
95
+ "email",
96
+ "risk_score",
97
+ )
98
+ assert client.evaluate_compliance.call_count == 3
99
+
100
+
101
+ def test_duplicate_columns_are_rejected_before_evaluation() -> None:
102
+ client = Mock(spec=RamenClient)
103
+ source = pd.DataFrame(
104
+ [["visible", "hidden"]], columns=["content", "content"]
105
+ )
106
+
107
+ with pytest.raises(ValueError, match="columns must be unique"):
108
+ filter_dataframe(
109
+ source,
110
+ mode=FiltrationMode.STRICT_EXCLUSION,
111
+ policy_ids=["policy-1"],
112
+ client=client,
113
+ )
114
+
115
+ client.evaluate_compliance.assert_not_called()
116
+
117
+
118
+ def test_semantic_imputation_requires_exact_column_name_in_steering() -> None:
119
+ client = Mock(spec=RamenClient)
120
+ client.evaluate_compliance.side_effect = [
121
+ {
122
+ "allowed": True,
123
+ "steering": None,
124
+ "receipt_verified": True,
125
+ "policy_ids": ["policy-1"],
126
+ },
127
+ {
128
+ "allowed": False,
129
+ "steering": "Manage the record safely.",
130
+ "receipt_verified": True,
131
+ "policy_ids": ["policy-1"],
132
+ },
133
+ ]
134
+ source = pd.DataFrame(
135
+ [{"record_id": "safe", "age": 30}, {"record_id": "blocked", "age": 99}]
136
+ )
137
+
138
+ with pytest.raises(FiltrationError, match="allowlisted column"):
139
+ filter_dataframe(
140
+ source,
141
+ mode=FiltrationMode.SEMANTIC_IMPUTATION,
142
+ policy_ids=["policy-1"],
143
+ remediable_columns=["age"],
144
+ client=client,
145
+ )
146
+
147
+
148
+ def test_malformed_audit_metadata_is_rejected() -> None:
149
+ client = Mock(spec=RamenClient)
150
+ client.evaluate_compliance.return_value = {
151
+ "allowed": True,
152
+ "steering": None,
153
+ "receipt_verified": "false",
154
+ "policy_ids": ["policy-1"],
155
+ }
156
+
157
+ with pytest.raises(FiltrationError, match="verification state"):
158
+ filter_dataframe(
159
+ pd.DataFrame([{"content": "safe"}]),
160
+ mode=FiltrationMode.STRICT_EXCLUSION,
161
+ policy_ids=["policy-1"],
162
+ client=client,
163
+ )
164
+
165
+
166
+ def test_csv_write_failure_preserves_existing_destination(
167
+ tmp_path, monkeypatch
168
+ ) -> None:
169
+ source_path = tmp_path / "source.csv"
170
+ destination_path = tmp_path / "destination.csv"
171
+ pd.DataFrame([{"content": "safe"}]).to_csv(source_path, index=False)
172
+ destination_path.write_text("existing-output\n", encoding="utf-8")
173
+
174
+ client = Mock(spec=RamenClient)
175
+ client.evaluate_compliance.return_value = {
176
+ "allowed": True,
177
+ "steering": None,
178
+ "receipt_verified": True,
179
+ "policy_ids": ["policy-1"],
180
+ }
181
+
182
+ def fail_to_csv(*args, **kwargs) -> None:
183
+ raise OSError("disk full")
184
+
185
+ monkeypatch.setattr(pd.DataFrame, "to_csv", fail_to_csv)
186
+
187
+ with pytest.raises(OSError, match="disk full"):
188
+ filter_csv(
189
+ source_path,
190
+ destination_path,
191
+ mode=FiltrationMode.STRICT_EXCLUSION,
192
+ policy_ids=["policy-1"],
193
+ client=client,
194
+ )
195
+
196
+ assert destination_path.read_text(encoding="utf-8") == "existing-output\n"
197
+ assert not list(tmp_path.glob(f".{destination_path.name}.*.tmp"))