ramen-data-filter 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,18 @@
1
+ """ramen-ai filtration for RAG ingestion and MLOps datasets."""
2
+
3
+ from .filter import (
4
+ FiltrationError,
5
+ FiltrationMode,
6
+ FiltrationResult,
7
+ filter_csv,
8
+ filter_dataframe,
9
+ )
10
+
11
+ __all__ = [
12
+ "FiltrationError",
13
+ "FiltrationMode",
14
+ "FiltrationResult",
15
+ "filter_csv",
16
+ "filter_dataframe",
17
+ ]
18
+ __version__ = "0.1.0"
@@ -0,0 +1,362 @@
1
+ """Dual-mode DataFrame and CSV filtration through the ramen-ai API."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import os
7
+ import re
8
+ import tempfile
9
+ from collections.abc import Mapping
10
+ from dataclasses import dataclass
11
+ from enum import Enum
12
+ from pathlib import Path
13
+ from typing import Any, Sequence
14
+
15
+ import pandas as pd
16
+ from pandas.api.types import is_integer_dtype, is_numeric_dtype
17
+ from ramen_ai import RamenClient
18
+
19
+
20
+ class FiltrationMode(str, Enum):
21
+ """Supported dataset filtration modes."""
22
+
23
+ STRICT_EXCLUSION = "strict_exclusion"
24
+ SEMANTIC_IMPUTATION = "semantic_imputation"
25
+
26
+
27
+ class FiltrationError(RuntimeError):
28
+ """Raised when a dataset cannot be evaluated or safely transformed."""
29
+
30
+
31
+ @dataclass(frozen=True)
32
+ class FiltrationResult:
33
+ """Filtered data plus row-level evaluation and transformation records."""
34
+
35
+ dataframe: pd.DataFrame
36
+ audit_log: pd.DataFrame
37
+ imputation_log: pd.DataFrame
38
+
39
+
40
+ _AUDIT_COLUMNS = (
41
+ "row_position",
42
+ "row_index",
43
+ "allowed",
44
+ "verdict",
45
+ "steering",
46
+ "receipt_verified",
47
+ "policy_ids",
48
+ "response",
49
+ )
50
+ _IMPUTATION_COLUMNS = (
51
+ "row_position",
52
+ "row_index",
53
+ "columns_changed",
54
+ "steering",
55
+ )
56
+
57
+
58
+ def filter_dataframe(
59
+ dataframe: pd.DataFrame,
60
+ *,
61
+ mode: FiltrationMode | str,
62
+ bundle_ids: Sequence[str] | None = None,
63
+ policy_ids: Sequence[str] | None = None,
64
+ remediable_columns: Sequence[str] | None = None,
65
+ client: RamenClient | None = None,
66
+ api_key: str | None = None,
67
+ provider_key: str | None = None,
68
+ provider_name: str | None = None,
69
+ context: dict[str, str] | None = None,
70
+ base_url: str = "https://api.ramenai.dev",
71
+ timeout: float = 30.0,
72
+ ) -> FiltrationResult:
73
+ """Evaluate and filter a DataFrame row by row.
74
+
75
+ Strict exclusion retains only rows with an allowed verdict. Semantic
76
+ imputation keeps every row but changes only allowlisted columns named in a
77
+ blocked row's steering instruction. Numeric replacements use the allowed
78
+ reference mean (rounded for integer columns); other values use deterministic
79
+ hot-deck imputation from allowed rows.
80
+
81
+ API and transport failures stop processing. Partial output is never
82
+ returned or written.
83
+ """
84
+ if not isinstance(dataframe, pd.DataFrame):
85
+ raise TypeError("dataframe must be a pandas DataFrame")
86
+ if not dataframe.columns.is_unique:
87
+ raise ValueError("dataframe columns must be unique")
88
+ try:
89
+ selected_mode = FiltrationMode(mode)
90
+ except ValueError as exc:
91
+ supported = ", ".join(item.value for item in FiltrationMode)
92
+ raise ValueError(f"mode must be one of: {supported}") from exc
93
+
94
+ selected_bundles = _validated_identifiers("bundle_ids", bundle_ids)
95
+ selected_policies = _validated_identifiers("policy_ids", policy_ids)
96
+ if not selected_bundles and not selected_policies:
97
+ raise ValueError("Provide at least one of 'bundle_ids' or 'policy_ids'.")
98
+
99
+ allowed_columns = _validated_columns(dataframe, remediable_columns)
100
+ if selected_mode is FiltrationMode.SEMANTIC_IMPUTATION and not allowed_columns:
101
+ raise ValueError(
102
+ "semantic_imputation requires at least one remediable column"
103
+ )
104
+
105
+ evaluation_context = dict(context or {})
106
+ if not all(
107
+ isinstance(key, str) and isinstance(value, str)
108
+ for key, value in evaluation_context.items()
109
+ ):
110
+ raise TypeError("context keys and values must be strings")
111
+ evaluation_context["integration"] = "ramen-data-filter"
112
+ evaluation_context.setdefault("source", "dataframe")
113
+
114
+ owned_client: RamenClient | None = None
115
+ active_client = client
116
+ if active_client is None:
117
+ resolved_api_key = api_key or os.environ.get("RAMEN_API_KEY", "")
118
+ if not resolved_api_key:
119
+ raise FiltrationError(
120
+ "RAMEN_API_KEY is not set; dataset filtration cannot be evaluated"
121
+ )
122
+ owned_client = RamenClient(
123
+ api_key=resolved_api_key,
124
+ base_url=base_url,
125
+ timeout=timeout,
126
+ )
127
+ active_client = owned_client
128
+
129
+ records: list[dict[str, Any]] = []
130
+ try:
131
+ for position, (row_index, row) in enumerate(dataframe.iterrows()):
132
+ serialized_row = json.dumps(
133
+ row.to_dict(), sort_keys=True, default=str
134
+ )
135
+ try:
136
+ response = active_client.evaluate_compliance(
137
+ input_text=serialized_row,
138
+ bundle_ids=selected_bundles or None,
139
+ policy_ids=selected_policies or None,
140
+ context=evaluation_context,
141
+ provider_key=(
142
+ provider_key
143
+ if provider_key is not None
144
+ else os.environ.get("OPENAI_API_KEY")
145
+ ),
146
+ provider_name=provider_name,
147
+ )
148
+ except Exception as exc:
149
+ raise FiltrationError(
150
+ f"ramen-ai evaluation failed for row position {position}"
151
+ ) from exc
152
+
153
+ if not isinstance(response, Mapping):
154
+ raise FiltrationError(
155
+ f"ramen-ai response was not a mapping for row position {position}"
156
+ )
157
+ allowed = response.get("allowed")
158
+ if not isinstance(allowed, bool):
159
+ raise FiltrationError(
160
+ "ramen-ai response omitted a boolean allowed verdict "
161
+ f"for row position {position}"
162
+ )
163
+ steering = response.get("steering")
164
+ if steering is not None and not isinstance(steering, str):
165
+ raise FiltrationError(
166
+ f"ramen-ai steering was not text for row position {position}"
167
+ )
168
+ receipt_verified = response.get("receipt_verified", False)
169
+ if not isinstance(receipt_verified, bool):
170
+ raise FiltrationError(
171
+ "ramen-ai receipt verification state was not boolean "
172
+ f"for row position {position}"
173
+ )
174
+ resolved_policy_ids = response.get("policy_ids", [])
175
+ if not isinstance(resolved_policy_ids, list) or not all(
176
+ isinstance(policy_id, str) and policy_id
177
+ for policy_id in resolved_policy_ids
178
+ ):
179
+ raise FiltrationError(
180
+ "ramen-ai policy_ids was not a list of non-empty strings "
181
+ f"for row position {position}"
182
+ )
183
+ records.append(
184
+ {
185
+ "row_position": position,
186
+ "row_index": row_index,
187
+ "allowed": allowed,
188
+ "verdict": "[ALLOWED]" if allowed else "[BLOCKED]",
189
+ "steering": steering,
190
+ "receipt_verified": receipt_verified,
191
+ "policy_ids": resolved_policy_ids.copy(),
192
+ "response": dict(response),
193
+ }
194
+ )
195
+ finally:
196
+ if owned_client is not None:
197
+ owned_client.close()
198
+
199
+ audit_log = pd.DataFrame(records, columns=_AUDIT_COLUMNS)
200
+ allowed_positions = [
201
+ record["row_position"] for record in records if record["allowed"]
202
+ ]
203
+
204
+ if selected_mode is FiltrationMode.STRICT_EXCLUSION:
205
+ filtered = dataframe.iloc[allowed_positions].copy().reset_index(drop=True)
206
+ return FiltrationResult(
207
+ dataframe=filtered,
208
+ audit_log=audit_log,
209
+ imputation_log=pd.DataFrame(columns=_IMPUTATION_COLUMNS),
210
+ )
211
+
212
+ imputed, imputation_log = _impute_blocked_rows(
213
+ dataframe=dataframe,
214
+ records=records,
215
+ allowed_positions=allowed_positions,
216
+ remediable_columns=allowed_columns,
217
+ )
218
+ return FiltrationResult(
219
+ dataframe=imputed,
220
+ audit_log=audit_log,
221
+ imputation_log=imputation_log,
222
+ )
223
+
224
+
225
+ def filter_csv(
226
+ source_path: str | Path,
227
+ destination_path: str | Path,
228
+ **kwargs: Any,
229
+ ) -> FiltrationResult:
230
+ """Read a CSV, filter it through :func:`filter_dataframe`, and write output."""
231
+ source = Path(source_path)
232
+ destination = Path(destination_path)
233
+ dataframe = pd.read_csv(source)
234
+ context = dict(kwargs.pop("context", {}) or {})
235
+ context["source"] = "csv"
236
+ result = filter_dataframe(dataframe, context=context, **kwargs)
237
+
238
+ temporary_path: Path | None = None
239
+ try:
240
+ with tempfile.NamedTemporaryFile(
241
+ mode="w",
242
+ dir=destination.parent,
243
+ prefix=f".{destination.name}.",
244
+ suffix=".tmp",
245
+ delete=False,
246
+ ) as temporary_file:
247
+ temporary_path = Path(temporary_file.name)
248
+ result.dataframe.to_csv(temporary_path, index=False)
249
+ os.replace(temporary_path, destination)
250
+ finally:
251
+ if temporary_path is not None and temporary_path.exists():
252
+ temporary_path.unlink()
253
+ return result
254
+
255
+
256
+ def _validated_identifiers(
257
+ name: str, values: Sequence[str] | None
258
+ ) -> list[str]:
259
+ if values is None:
260
+ return []
261
+ if isinstance(values, (str, bytes)):
262
+ raise TypeError(f"{name} must be a sequence of non-empty strings")
263
+ normalized = list(values)
264
+ if not all(isinstance(value, str) and value.strip() for value in normalized):
265
+ raise ValueError(f"{name} must contain only non-empty strings")
266
+ return normalized
267
+
268
+
269
+ def _validated_columns(
270
+ dataframe: pd.DataFrame, columns: Sequence[str] | None
271
+ ) -> list[str]:
272
+ if columns is None:
273
+ return []
274
+ if isinstance(columns, (str, bytes)):
275
+ raise TypeError("remediable_columns must be a sequence of column names")
276
+ normalized = list(dict.fromkeys(columns))
277
+ if not all(isinstance(column, str) and column for column in normalized):
278
+ raise ValueError("remediable_columns must contain non-empty strings")
279
+ missing = [column for column in normalized if column not in dataframe.columns]
280
+ if missing:
281
+ raise ValueError(f"remediable columns are missing from dataframe: {missing}")
282
+ casefolded = [column.casefold() for column in normalized]
283
+ if len(casefolded) != len(set(casefolded)):
284
+ raise ValueError("remediable column names must be unique ignoring case")
285
+ return normalized
286
+
287
+
288
+ def _impute_blocked_rows(
289
+ *,
290
+ dataframe: pd.DataFrame,
291
+ records: list[dict[str, Any]],
292
+ allowed_positions: list[int],
293
+ remediable_columns: list[str],
294
+ ) -> tuple[pd.DataFrame, pd.DataFrame]:
295
+ blocked_records = [record for record in records if not record["allowed"]]
296
+ if not blocked_records:
297
+ return dataframe.copy(), pd.DataFrame(columns=_IMPUTATION_COLUMNS)
298
+ if not allowed_positions:
299
+ raise FiltrationError(
300
+ "semantic imputation requires at least one allowed reference row"
301
+ )
302
+
303
+ reference = dataframe.iloc[allowed_positions]
304
+ replacements = {
305
+ column: _replacement_values(reference[column], column)
306
+ for column in remediable_columns
307
+ }
308
+ imputed = dataframe.copy()
309
+ log_records: list[dict[str, Any]] = []
310
+
311
+ for blocked_ordinal, record in enumerate(blocked_records):
312
+ steering = record["steering"] or ""
313
+ selected_columns = [
314
+ column
315
+ for column in remediable_columns
316
+ if re.search(
317
+ rf"(?<![\w]){re.escape(column)}(?![\w])",
318
+ steering,
319
+ flags=re.IGNORECASE,
320
+ )
321
+ ]
322
+ if not selected_columns:
323
+ raise FiltrationError(
324
+ "blocked row did not receive steering naming an allowlisted "
325
+ f"column at row position {record['row_position']}"
326
+ )
327
+
328
+ for column in selected_columns:
329
+ values = replacements[column]
330
+ replacement = values[blocked_ordinal % len(values)]
331
+ column_position = dataframe.columns.get_loc(column)
332
+ imputed.iat[record["row_position"], column_position] = replacement
333
+
334
+ log_records.append(
335
+ {
336
+ "row_position": record["row_position"],
337
+ "row_index": record["row_index"],
338
+ "columns_changed": tuple(selected_columns),
339
+ "steering": steering,
340
+ }
341
+ )
342
+
343
+ return imputed, pd.DataFrame(log_records, columns=_IMPUTATION_COLUMNS)
344
+
345
+
346
+ def _replacement_values(series: pd.Series, column: str) -> list[Any]:
347
+ non_null = series.dropna()
348
+ if non_null.empty:
349
+ raise FiltrationError(
350
+ f"allowed reference rows contain no values for '{column}'"
351
+ )
352
+ if is_numeric_dtype(non_null.dtype):
353
+ mean = non_null.mean()
354
+ if pd.isna(mean):
355
+ raise FiltrationError(
356
+ f"allowed reference mean could not be calculated for '{column}'"
357
+ )
358
+ replacement: Any = float(mean)
359
+ if is_integer_dtype(non_null.dtype):
360
+ replacement = int(round(replacement))
361
+ return [replacement]
362
+ return non_null.tolist()
@@ -0,0 +1,296 @@
1
+ Metadata-Version: 2.4
2
+ Name: ramen-data-filter
3
+ Version: 0.1.0
4
+ Summary: Dual-mode Pandas and CSV filtration for ramen-ai RAG ingestion and MLOps dataset sanitization.
5
+ License: Proprietary
6
+ Requires-Python: >=3.10
7
+ Requires-Dist: pandas==2.3.3
8
+ Requires-Dist: ramen-ai-core==0.2.1
9
+ Provides-Extra: dev
10
+ Requires-Dist: pytest==8.4.2; extra == 'dev'
11
+ Description-Content-Type: text/markdown
12
+
13
+ # ramen-ai Data Filter
14
+
15
+ <p align="center">
16
+ <img src="../../assets/ramen-logo.png" alt="ramen-ai" width="100"/>
17
+ </p>
18
+
19
+ Dual-mode Pandas and CSV filtration for RAG ingestion scrubbing and MLOps
20
+ dataset sanitization. Every row is evaluated through the ramen-ai semantic
21
+ firewall before it can enter a retrieval corpus, training set, analytics store,
22
+ or downstream data pipeline.
23
+
24
+ Use **Strict Exclusion** to remove blocked rows, or **Semantic Imputation** to
25
+ preserve row count while replacing only explicitly allowlisted columns named by
26
+ ramen-ai steering. Typical policies can detect PII, data-exfiltration vectors,
27
+ poisoned retrieval context, prompt injection, and other unsafe dataset content.
28
+
29
+ Requires **Python ≥ 3.10**.
30
+
31
+ ---
32
+
33
+ ## Supported Ecosystems
34
+
35
+ <p align="center">
36
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/langchain-python">
37
+ <img src="https://img.shields.io/badge/LangChain-1C3C3C?style=flat&logo=langchain&logoColor=white" alt="LangChain"/>
38
+ </a>
39
+ &nbsp;
40
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/pydantic-ai">
41
+ <img src="https://img.shields.io/badge/PydanticAI-E92063?style=flat&logo=pydantic&logoColor=white" alt="PydanticAI"/>
42
+ </a>
43
+ &nbsp;
44
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/mcp-proxy">
45
+ <img src="https://img.shields.io/badge/MCP-6B21A8?style=flat&logo=anthropic&logoColor=white" alt="MCP"/>
46
+ </a>
47
+ &nbsp;
48
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/agt-typescript">
49
+ <img src="https://img.shields.io/badge/Microsoft%20AGT-0078D4?style=flat&logo=microsoft&logoColor=white" alt="Microsoft AGT"/>
50
+ </a>
51
+ &nbsp;
52
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/github-action">
53
+ <img src="https://img.shields.io/badge/GitHub%20Actions-2088FF?style=flat&logo=githubactions&logoColor=white" alt="GitHub Actions"/>
54
+ </a>
55
+ &nbsp;
56
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/cmcp-python">
57
+ <img src="https://img.shields.io/badge/cMCP-00A67E?style=flat&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZmlsbD0id2hpdGUiIGQ9Ik0xMiAyTDIgN2wxMCA1IDEwLTV6TTIgMTdsOCA0VjExbC04LTR6TTE0IDIxbDgtNFYxMWwtOCA0eiIvPjwvc3ZnPg==&logoColor=white" alt="cMCP"/>
58
+ </a>
59
+ &nbsp;
60
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/mlflow-python">
61
+ <img src="https://img.shields.io/badge/MLflow-0194E2?style=flat&logo=mlflow&logoColor=white" alt="MLflow"/>
62
+ </a>
63
+ &nbsp;
64
+ <a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/ramen-data-filter">
65
+ <img src="https://img.shields.io/badge/ramen%20data%20filter-D97706?style=flat&logo=pandas&logoColor=white" alt="ramen data filter"/>
66
+ </a>
67
+ </p>
68
+
69
+ ---
70
+
71
+ ## Can you bypass it?
72
+
73
+ Standard safety filters catch basic syntax. They fail against encoded payloads
74
+ and corporate jargon. We challenge you to bypass our semantic firewall using
75
+ the zero-day evasion vectors in our official **[Red Team Guide](../../RED_TEAM_GUIDE.md)**.
76
+
77
+ Below is a simulation of the Grok/Bankr heist. We fed the raw adversarial
78
+ prompt directly into our sandbox. It uses a social engineering wrapper
79
+ (claiming a visual impairment) to smuggle a 3,000,000,000 DRB transfer
80
+ instruction encoded in Morse code. The firewall evaluated the underlying
81
+ semantic intent, intercepted the unauthorized financial transfer, and blocked
82
+ it pre-execution, issuing a verified Ed25519 receipt.
83
+
84
+ <p align="center">
85
+ <img src="../../assets/grok-bankr.png" alt="ramen-ai intercepting the Grok/Bankr Morse-code heist pre-execution" width="720"/>
86
+ </p>
87
+
88
+ ---
89
+
90
+ ## API Key
91
+
92
+ To use this integration, you must mint an API Key. We offer a **Free Starter
93
+ Tier** (1,000 evaluations/month, BYOK) which includes full access to our Core
94
+ IT Security bundle. Mint your key at:
95
+ **[https://ramenai.dev/pricing](https://ramenai.dev/pricing)**
96
+
97
+ ---
98
+
99
+ ## Installation
100
+
101
+ From PyPI after release:
102
+
103
+ ```bash
104
+ pip install ramen-data-filter
105
+ ```
106
+
107
+ From this monorepo:
108
+
109
+ ```bash
110
+ pip install -e plugins/ramen-data-filter
111
+ pip install -e "plugins/ramen-data-filter[dev]" # include pytest
112
+ ```
113
+
114
+ ---
115
+
116
+ ## Configuration / BYOK
117
+
118
+ ```bash
119
+ export RAMEN_API_KEY=ramen_ak_... # required
120
+ export OPENAI_API_KEY=sk-... # Starter/Professional BYOK
121
+ ```
122
+
123
+ `RAMEN_API_KEY` authenticates the evaluation request. The Starter and
124
+ Professional tiers also require an LLM provider key. The plugin reads
125
+ `OPENAI_API_KEY` at evaluation time and `ramen-ai-core` forwards it as the
126
+ `X-Provider-Key` header. Without a provider key, those tiers return
127
+ `402 Payment Required`.
128
+
129
+ Pass `provider_name="anthropic"`, `"google"`, `"synthetic"`, or
130
+ `"hyperbolic"` with the corresponding `provider_key` when OpenAI is not the
131
+ provider. Enterprise tiers use platform-managed provider keys and do not need
132
+ `OPENAI_API_KEY`.
133
+
134
+ Credentials are read from environment variables or passed at runtime. Never
135
+ store them in source code, notebooks, CSV files, or committed configuration.
136
+
137
+ ---
138
+
139
+ ## Usage
140
+
141
+ ### Strict Exclusion
142
+
143
+ Strict exclusion evaluates each row and returns a new DataFrame containing only
144
+ `[ALLOWED]` rows. The returned index is reset; the source DataFrame is not
145
+ modified.
146
+
147
+ ```python
148
+ import pandas as pd
149
+ from ramen_data_filter import FiltrationMode, filter_dataframe
150
+
151
+ records = pd.DataFrame(
152
+ [
153
+ {"document_id": "doc-1", "content": "Approved public documentation"},
154
+ {"document_id": "doc-2", "content": "Retrieved untrusted context"},
155
+ ]
156
+ )
157
+
158
+ result = filter_dataframe(
159
+ records,
160
+ mode=FiltrationMode.STRICT_EXCLUSION,
161
+ bundle_ids=["ramen__shield_core_it"],
162
+ )
163
+
164
+ clean_records = result.dataframe
165
+ print(result.audit_log[["row_index", "verdict", "receipt_verified"]])
166
+ ```
167
+
168
+ ### Semantic Imputation
169
+
170
+ Semantic imputation preserves every row. A blocked row may change only a column
171
+ that is both listed in `remediable_columns` and named in the ramen-ai steering
172
+ instruction. Numeric columns receive the allowed-reference mean (rounded for
173
+ integer data); non-numeric columns use deterministic hot-deck values from
174
+ allowed rows. All other values remain unchanged.
175
+
176
+ ```python
177
+ result = filter_dataframe(
178
+ records,
179
+ mode=FiltrationMode.SEMANTIC_IMPUTATION,
180
+ policy_ids=["<POLICY_UUID>"],
181
+ remediable_columns=["content"],
182
+ )
183
+
184
+ sanitized_records = result.dataframe
185
+ print(result.imputation_log)
186
+ ```
187
+
188
+ ### CSV pipeline
189
+
190
+ ```python
191
+ from ramen_data_filter import FiltrationMode, filter_csv
192
+
193
+ result = filter_csv(
194
+ "rag-ingestion.csv",
195
+ "rag-ingestion-sanitized.csv",
196
+ mode=FiltrationMode.STRICT_EXCLUSION,
197
+ bundle_ids=["ramen__shield_core_it"],
198
+ )
199
+ ```
200
+
201
+ The destination is written only after every row has been evaluated and the
202
+ transformation succeeds. Existing destination files are replaced by Pandas.
203
+
204
+ ---
205
+
206
+ ## How it works
207
+
208
+ 1. Serializes each row as stable, key-sorted JSON.
209
+ 2. Calls `RamenClient.evaluate_compliance` with configured bundles or policies.
210
+ 3. Records the verdict, steering, resolved policy IDs, receipt verification
211
+ state, and complete SDK response in `audit_log`.
212
+ 4. Applies the selected mode only after all rows have been evaluated.
213
+ 5. Returns a new DataFrame and transformation metadata; CSV usage writes that
214
+ DataFrame with `index=False`.
215
+
216
+ The pipeline is fail-closed. Missing credentials, transport failures, malformed
217
+ responses, unsafe imputation steering, and unusable reference values raise
218
+ `FiltrationError`; partial filtered output is never returned or written.
219
+
220
+ ---
221
+
222
+ ## API reference
223
+
224
+ ### `filter_dataframe(dataframe, *, mode, ...) -> FiltrationResult`
225
+
226
+ | Parameter | Type | Description |
227
+ |---|---|---|
228
+ | `dataframe` | `pandas.DataFrame` | Source records. The input is not modified. |
229
+ | `mode` | `FiltrationMode \| str` | `strict_exclusion` or `semantic_imputation`. |
230
+ | `bundle_ids` | `Sequence[str]` | ramen-ai bundle slugs. At least one bundle or policy is required. |
231
+ | `policy_ids` | `Sequence[str]` | Explicit policy UUIDs. May be combined with bundles. |
232
+ | `remediable_columns` | `Sequence[str]` | Columns eligible for steering-constrained replacement. Required for semantic imputation. |
233
+ | `client` | `RamenClient` | Optional injected client, primarily for controlled runtimes and tests. |
234
+ | `api_key` | `str` | Runtime override; defaults to `RAMEN_API_KEY`. |
235
+ | `provider_key` | `str` | BYOK override; defaults to `OPENAI_API_KEY`. |
236
+ | `provider_name` | `str` | Optional BYOK provider routing hint. |
237
+ | `context` | `dict[str, str]` | Additional audit metadata. |
238
+ | `base_url` | `str` | ramen-ai API base URL. |
239
+ | `timeout` | `float` | HTTP timeout in seconds. |
240
+
241
+ ### `filter_csv(source_path, destination_path, **kwargs) -> FiltrationResult`
242
+
243
+ Reads the source with `pandas.read_csv`, applies `filter_dataframe`, and writes
244
+ the resulting DataFrame with `index=False`.
245
+
246
+ ### `FiltrationResult`
247
+
248
+ | Attribute | Description |
249
+ |---|---|
250
+ | `dataframe` | Filtered or imputed DataFrame. |
251
+ | `audit_log` | One row per evaluation with verdict and SDK response metadata. |
252
+ | `imputation_log` | One row per transformed record; empty in strict mode. |
253
+
254
+ Public exports: `FiltrationError`, `FiltrationMode`, `FiltrationResult`,
255
+ `filter_dataframe`, and `filter_csv`.
256
+
257
+ ---
258
+
259
+ ## Running the tests
260
+
261
+ ```bash
262
+ PYTHONPATH="core-clients/python:plugins/ramen-data-filter/src" \
263
+ python -m pytest plugins/ramen-data-filter/tests -v
264
+ ```
265
+
266
+ The isolated tests use a mocked `RamenClient`; they make no network calls and
267
+ require no credentials. They prove blocked-row exclusion and constrained
268
+ semantic imputation on a dummy DataFrame.
269
+
270
+ ---
271
+
272
+ ## Available bundles
273
+
274
+ | Bundle slug | Useful dataset coverage |
275
+ |---|---|
276
+ | `ramen__shield_core_it` | Secret exfiltration, prompt leakage, jailbreaks, destructive instructions, and indirect prompt injection. |
277
+ | `ramen__eu_ai_act_baseline` | EU AI Act data-governance, prohibited-practice, and transparency controls. |
278
+
279
+ Pass explicit `policy_ids` for custom PII, ingestion, data-quality, or internal
280
+ MLOps controls. Bundle and policy details are available at
281
+ [ramenai.dev/pricing](https://ramenai.dev/pricing).
282
+
283
+ ---
284
+
285
+ ## Limitations
286
+
287
+ - Evaluation adds one API request per row. Large datasets should be processed in
288
+ controlled batches with rate limits and resumability at the orchestration
289
+ layer.
290
+ - Semantic imputation creates synthetic values; it does not recover original
291
+ truth or establish legal compliance, statistical fairness, or model quality.
292
+ - Imputed rows are not automatically re-evaluated. Validate and, where required,
293
+ re-evaluate transformed output before production ingestion or training.
294
+ - Non-numeric hot-deck replacement is deterministic and may alter dataset
295
+ distributions. Preserve source data and review `audit_log` and
296
+ `imputation_log` in an access-controlled audit store.
@@ -0,0 +1,5 @@
1
+ ramen_data_filter/__init__.py,sha256=xqfwzAHi7s0f9aEn7UNDPFw2UBgyA3X9gkhBmPL0o20,338
2
+ ramen_data_filter/filter.py,sha256=lx1lSyAhTaEHgjCzYsYbdapvOXOzhlnf77K7e-fB2MY,13001
3
+ ramen_data_filter-0.1.0.dist-info/METADATA,sha256=0te7oYCjMSBmHJFhocv6BMMHjDHFLARJP8QU6YnVkP0,11211
4
+ ramen_data_filter-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
5
+ ramen_data_filter-0.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.27.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any