ramen-data-filter 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""ramen-ai filtration for RAG ingestion and MLOps datasets."""
|
|
2
|
+
|
|
3
|
+
from .filter import (
|
|
4
|
+
FiltrationError,
|
|
5
|
+
FiltrationMode,
|
|
6
|
+
FiltrationResult,
|
|
7
|
+
filter_csv,
|
|
8
|
+
filter_dataframe,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"FiltrationError",
|
|
13
|
+
"FiltrationMode",
|
|
14
|
+
"FiltrationResult",
|
|
15
|
+
"filter_csv",
|
|
16
|
+
"filter_dataframe",
|
|
17
|
+
]
|
|
18
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,362 @@
|
|
|
1
|
+
"""Dual-mode DataFrame and CSV filtration through the ramen-ai API."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import re
|
|
8
|
+
import tempfile
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from enum import Enum
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Sequence
|
|
14
|
+
|
|
15
|
+
import pandas as pd
|
|
16
|
+
from pandas.api.types import is_integer_dtype, is_numeric_dtype
|
|
17
|
+
from ramen_ai import RamenClient
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class FiltrationMode(str, Enum):
|
|
21
|
+
"""Supported dataset filtration modes."""
|
|
22
|
+
|
|
23
|
+
STRICT_EXCLUSION = "strict_exclusion"
|
|
24
|
+
SEMANTIC_IMPUTATION = "semantic_imputation"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class FiltrationError(RuntimeError):
|
|
28
|
+
"""Raised when a dataset cannot be evaluated or safely transformed."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(frozen=True)
|
|
32
|
+
class FiltrationResult:
|
|
33
|
+
"""Filtered data plus row-level evaluation and transformation records."""
|
|
34
|
+
|
|
35
|
+
dataframe: pd.DataFrame
|
|
36
|
+
audit_log: pd.DataFrame
|
|
37
|
+
imputation_log: pd.DataFrame
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
_AUDIT_COLUMNS = (
|
|
41
|
+
"row_position",
|
|
42
|
+
"row_index",
|
|
43
|
+
"allowed",
|
|
44
|
+
"verdict",
|
|
45
|
+
"steering",
|
|
46
|
+
"receipt_verified",
|
|
47
|
+
"policy_ids",
|
|
48
|
+
"response",
|
|
49
|
+
)
|
|
50
|
+
_IMPUTATION_COLUMNS = (
|
|
51
|
+
"row_position",
|
|
52
|
+
"row_index",
|
|
53
|
+
"columns_changed",
|
|
54
|
+
"steering",
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def filter_dataframe(
|
|
59
|
+
dataframe: pd.DataFrame,
|
|
60
|
+
*,
|
|
61
|
+
mode: FiltrationMode | str,
|
|
62
|
+
bundle_ids: Sequence[str] | None = None,
|
|
63
|
+
policy_ids: Sequence[str] | None = None,
|
|
64
|
+
remediable_columns: Sequence[str] | None = None,
|
|
65
|
+
client: RamenClient | None = None,
|
|
66
|
+
api_key: str | None = None,
|
|
67
|
+
provider_key: str | None = None,
|
|
68
|
+
provider_name: str | None = None,
|
|
69
|
+
context: dict[str, str] | None = None,
|
|
70
|
+
base_url: str = "https://api.ramenai.dev",
|
|
71
|
+
timeout: float = 30.0,
|
|
72
|
+
) -> FiltrationResult:
|
|
73
|
+
"""Evaluate and filter a DataFrame row by row.
|
|
74
|
+
|
|
75
|
+
Strict exclusion retains only rows with an allowed verdict. Semantic
|
|
76
|
+
imputation keeps every row but changes only allowlisted columns named in a
|
|
77
|
+
blocked row's steering instruction. Numeric replacements use the allowed
|
|
78
|
+
reference mean (rounded for integer columns); other values use deterministic
|
|
79
|
+
hot-deck imputation from allowed rows.
|
|
80
|
+
|
|
81
|
+
API and transport failures stop processing. Partial output is never
|
|
82
|
+
returned or written.
|
|
83
|
+
"""
|
|
84
|
+
if not isinstance(dataframe, pd.DataFrame):
|
|
85
|
+
raise TypeError("dataframe must be a pandas DataFrame")
|
|
86
|
+
if not dataframe.columns.is_unique:
|
|
87
|
+
raise ValueError("dataframe columns must be unique")
|
|
88
|
+
try:
|
|
89
|
+
selected_mode = FiltrationMode(mode)
|
|
90
|
+
except ValueError as exc:
|
|
91
|
+
supported = ", ".join(item.value for item in FiltrationMode)
|
|
92
|
+
raise ValueError(f"mode must be one of: {supported}") from exc
|
|
93
|
+
|
|
94
|
+
selected_bundles = _validated_identifiers("bundle_ids", bundle_ids)
|
|
95
|
+
selected_policies = _validated_identifiers("policy_ids", policy_ids)
|
|
96
|
+
if not selected_bundles and not selected_policies:
|
|
97
|
+
raise ValueError("Provide at least one of 'bundle_ids' or 'policy_ids'.")
|
|
98
|
+
|
|
99
|
+
allowed_columns = _validated_columns(dataframe, remediable_columns)
|
|
100
|
+
if selected_mode is FiltrationMode.SEMANTIC_IMPUTATION and not allowed_columns:
|
|
101
|
+
raise ValueError(
|
|
102
|
+
"semantic_imputation requires at least one remediable column"
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
evaluation_context = dict(context or {})
|
|
106
|
+
if not all(
|
|
107
|
+
isinstance(key, str) and isinstance(value, str)
|
|
108
|
+
for key, value in evaluation_context.items()
|
|
109
|
+
):
|
|
110
|
+
raise TypeError("context keys and values must be strings")
|
|
111
|
+
evaluation_context["integration"] = "ramen-data-filter"
|
|
112
|
+
evaluation_context.setdefault("source", "dataframe")
|
|
113
|
+
|
|
114
|
+
owned_client: RamenClient | None = None
|
|
115
|
+
active_client = client
|
|
116
|
+
if active_client is None:
|
|
117
|
+
resolved_api_key = api_key or os.environ.get("RAMEN_API_KEY", "")
|
|
118
|
+
if not resolved_api_key:
|
|
119
|
+
raise FiltrationError(
|
|
120
|
+
"RAMEN_API_KEY is not set; dataset filtration cannot be evaluated"
|
|
121
|
+
)
|
|
122
|
+
owned_client = RamenClient(
|
|
123
|
+
api_key=resolved_api_key,
|
|
124
|
+
base_url=base_url,
|
|
125
|
+
timeout=timeout,
|
|
126
|
+
)
|
|
127
|
+
active_client = owned_client
|
|
128
|
+
|
|
129
|
+
records: list[dict[str, Any]] = []
|
|
130
|
+
try:
|
|
131
|
+
for position, (row_index, row) in enumerate(dataframe.iterrows()):
|
|
132
|
+
serialized_row = json.dumps(
|
|
133
|
+
row.to_dict(), sort_keys=True, default=str
|
|
134
|
+
)
|
|
135
|
+
try:
|
|
136
|
+
response = active_client.evaluate_compliance(
|
|
137
|
+
input_text=serialized_row,
|
|
138
|
+
bundle_ids=selected_bundles or None,
|
|
139
|
+
policy_ids=selected_policies or None,
|
|
140
|
+
context=evaluation_context,
|
|
141
|
+
provider_key=(
|
|
142
|
+
provider_key
|
|
143
|
+
if provider_key is not None
|
|
144
|
+
else os.environ.get("OPENAI_API_KEY")
|
|
145
|
+
),
|
|
146
|
+
provider_name=provider_name,
|
|
147
|
+
)
|
|
148
|
+
except Exception as exc:
|
|
149
|
+
raise FiltrationError(
|
|
150
|
+
f"ramen-ai evaluation failed for row position {position}"
|
|
151
|
+
) from exc
|
|
152
|
+
|
|
153
|
+
if not isinstance(response, Mapping):
|
|
154
|
+
raise FiltrationError(
|
|
155
|
+
f"ramen-ai response was not a mapping for row position {position}"
|
|
156
|
+
)
|
|
157
|
+
allowed = response.get("allowed")
|
|
158
|
+
if not isinstance(allowed, bool):
|
|
159
|
+
raise FiltrationError(
|
|
160
|
+
"ramen-ai response omitted a boolean allowed verdict "
|
|
161
|
+
f"for row position {position}"
|
|
162
|
+
)
|
|
163
|
+
steering = response.get("steering")
|
|
164
|
+
if steering is not None and not isinstance(steering, str):
|
|
165
|
+
raise FiltrationError(
|
|
166
|
+
f"ramen-ai steering was not text for row position {position}"
|
|
167
|
+
)
|
|
168
|
+
receipt_verified = response.get("receipt_verified", False)
|
|
169
|
+
if not isinstance(receipt_verified, bool):
|
|
170
|
+
raise FiltrationError(
|
|
171
|
+
"ramen-ai receipt verification state was not boolean "
|
|
172
|
+
f"for row position {position}"
|
|
173
|
+
)
|
|
174
|
+
resolved_policy_ids = response.get("policy_ids", [])
|
|
175
|
+
if not isinstance(resolved_policy_ids, list) or not all(
|
|
176
|
+
isinstance(policy_id, str) and policy_id
|
|
177
|
+
for policy_id in resolved_policy_ids
|
|
178
|
+
):
|
|
179
|
+
raise FiltrationError(
|
|
180
|
+
"ramen-ai policy_ids was not a list of non-empty strings "
|
|
181
|
+
f"for row position {position}"
|
|
182
|
+
)
|
|
183
|
+
records.append(
|
|
184
|
+
{
|
|
185
|
+
"row_position": position,
|
|
186
|
+
"row_index": row_index,
|
|
187
|
+
"allowed": allowed,
|
|
188
|
+
"verdict": "[ALLOWED]" if allowed else "[BLOCKED]",
|
|
189
|
+
"steering": steering,
|
|
190
|
+
"receipt_verified": receipt_verified,
|
|
191
|
+
"policy_ids": resolved_policy_ids.copy(),
|
|
192
|
+
"response": dict(response),
|
|
193
|
+
}
|
|
194
|
+
)
|
|
195
|
+
finally:
|
|
196
|
+
if owned_client is not None:
|
|
197
|
+
owned_client.close()
|
|
198
|
+
|
|
199
|
+
audit_log = pd.DataFrame(records, columns=_AUDIT_COLUMNS)
|
|
200
|
+
allowed_positions = [
|
|
201
|
+
record["row_position"] for record in records if record["allowed"]
|
|
202
|
+
]
|
|
203
|
+
|
|
204
|
+
if selected_mode is FiltrationMode.STRICT_EXCLUSION:
|
|
205
|
+
filtered = dataframe.iloc[allowed_positions].copy().reset_index(drop=True)
|
|
206
|
+
return FiltrationResult(
|
|
207
|
+
dataframe=filtered,
|
|
208
|
+
audit_log=audit_log,
|
|
209
|
+
imputation_log=pd.DataFrame(columns=_IMPUTATION_COLUMNS),
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
imputed, imputation_log = _impute_blocked_rows(
|
|
213
|
+
dataframe=dataframe,
|
|
214
|
+
records=records,
|
|
215
|
+
allowed_positions=allowed_positions,
|
|
216
|
+
remediable_columns=allowed_columns,
|
|
217
|
+
)
|
|
218
|
+
return FiltrationResult(
|
|
219
|
+
dataframe=imputed,
|
|
220
|
+
audit_log=audit_log,
|
|
221
|
+
imputation_log=imputation_log,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def filter_csv(
|
|
226
|
+
source_path: str | Path,
|
|
227
|
+
destination_path: str | Path,
|
|
228
|
+
**kwargs: Any,
|
|
229
|
+
) -> FiltrationResult:
|
|
230
|
+
"""Read a CSV, filter it through :func:`filter_dataframe`, and write output."""
|
|
231
|
+
source = Path(source_path)
|
|
232
|
+
destination = Path(destination_path)
|
|
233
|
+
dataframe = pd.read_csv(source)
|
|
234
|
+
context = dict(kwargs.pop("context", {}) or {})
|
|
235
|
+
context["source"] = "csv"
|
|
236
|
+
result = filter_dataframe(dataframe, context=context, **kwargs)
|
|
237
|
+
|
|
238
|
+
temporary_path: Path | None = None
|
|
239
|
+
try:
|
|
240
|
+
with tempfile.NamedTemporaryFile(
|
|
241
|
+
mode="w",
|
|
242
|
+
dir=destination.parent,
|
|
243
|
+
prefix=f".{destination.name}.",
|
|
244
|
+
suffix=".tmp",
|
|
245
|
+
delete=False,
|
|
246
|
+
) as temporary_file:
|
|
247
|
+
temporary_path = Path(temporary_file.name)
|
|
248
|
+
result.dataframe.to_csv(temporary_path, index=False)
|
|
249
|
+
os.replace(temporary_path, destination)
|
|
250
|
+
finally:
|
|
251
|
+
if temporary_path is not None and temporary_path.exists():
|
|
252
|
+
temporary_path.unlink()
|
|
253
|
+
return result
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _validated_identifiers(
|
|
257
|
+
name: str, values: Sequence[str] | None
|
|
258
|
+
) -> list[str]:
|
|
259
|
+
if values is None:
|
|
260
|
+
return []
|
|
261
|
+
if isinstance(values, (str, bytes)):
|
|
262
|
+
raise TypeError(f"{name} must be a sequence of non-empty strings")
|
|
263
|
+
normalized = list(values)
|
|
264
|
+
if not all(isinstance(value, str) and value.strip() for value in normalized):
|
|
265
|
+
raise ValueError(f"{name} must contain only non-empty strings")
|
|
266
|
+
return normalized
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _validated_columns(
|
|
270
|
+
dataframe: pd.DataFrame, columns: Sequence[str] | None
|
|
271
|
+
) -> list[str]:
|
|
272
|
+
if columns is None:
|
|
273
|
+
return []
|
|
274
|
+
if isinstance(columns, (str, bytes)):
|
|
275
|
+
raise TypeError("remediable_columns must be a sequence of column names")
|
|
276
|
+
normalized = list(dict.fromkeys(columns))
|
|
277
|
+
if not all(isinstance(column, str) and column for column in normalized):
|
|
278
|
+
raise ValueError("remediable_columns must contain non-empty strings")
|
|
279
|
+
missing = [column for column in normalized if column not in dataframe.columns]
|
|
280
|
+
if missing:
|
|
281
|
+
raise ValueError(f"remediable columns are missing from dataframe: {missing}")
|
|
282
|
+
casefolded = [column.casefold() for column in normalized]
|
|
283
|
+
if len(casefolded) != len(set(casefolded)):
|
|
284
|
+
raise ValueError("remediable column names must be unique ignoring case")
|
|
285
|
+
return normalized
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _impute_blocked_rows(
|
|
289
|
+
*,
|
|
290
|
+
dataframe: pd.DataFrame,
|
|
291
|
+
records: list[dict[str, Any]],
|
|
292
|
+
allowed_positions: list[int],
|
|
293
|
+
remediable_columns: list[str],
|
|
294
|
+
) -> tuple[pd.DataFrame, pd.DataFrame]:
|
|
295
|
+
blocked_records = [record for record in records if not record["allowed"]]
|
|
296
|
+
if not blocked_records:
|
|
297
|
+
return dataframe.copy(), pd.DataFrame(columns=_IMPUTATION_COLUMNS)
|
|
298
|
+
if not allowed_positions:
|
|
299
|
+
raise FiltrationError(
|
|
300
|
+
"semantic imputation requires at least one allowed reference row"
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
reference = dataframe.iloc[allowed_positions]
|
|
304
|
+
replacements = {
|
|
305
|
+
column: _replacement_values(reference[column], column)
|
|
306
|
+
for column in remediable_columns
|
|
307
|
+
}
|
|
308
|
+
imputed = dataframe.copy()
|
|
309
|
+
log_records: list[dict[str, Any]] = []
|
|
310
|
+
|
|
311
|
+
for blocked_ordinal, record in enumerate(blocked_records):
|
|
312
|
+
steering = record["steering"] or ""
|
|
313
|
+
selected_columns = [
|
|
314
|
+
column
|
|
315
|
+
for column in remediable_columns
|
|
316
|
+
if re.search(
|
|
317
|
+
rf"(?<![\w]){re.escape(column)}(?![\w])",
|
|
318
|
+
steering,
|
|
319
|
+
flags=re.IGNORECASE,
|
|
320
|
+
)
|
|
321
|
+
]
|
|
322
|
+
if not selected_columns:
|
|
323
|
+
raise FiltrationError(
|
|
324
|
+
"blocked row did not receive steering naming an allowlisted "
|
|
325
|
+
f"column at row position {record['row_position']}"
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
for column in selected_columns:
|
|
329
|
+
values = replacements[column]
|
|
330
|
+
replacement = values[blocked_ordinal % len(values)]
|
|
331
|
+
column_position = dataframe.columns.get_loc(column)
|
|
332
|
+
imputed.iat[record["row_position"], column_position] = replacement
|
|
333
|
+
|
|
334
|
+
log_records.append(
|
|
335
|
+
{
|
|
336
|
+
"row_position": record["row_position"],
|
|
337
|
+
"row_index": record["row_index"],
|
|
338
|
+
"columns_changed": tuple(selected_columns),
|
|
339
|
+
"steering": steering,
|
|
340
|
+
}
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
return imputed, pd.DataFrame(log_records, columns=_IMPUTATION_COLUMNS)
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _replacement_values(series: pd.Series, column: str) -> list[Any]:
|
|
347
|
+
non_null = series.dropna()
|
|
348
|
+
if non_null.empty:
|
|
349
|
+
raise FiltrationError(
|
|
350
|
+
f"allowed reference rows contain no values for '{column}'"
|
|
351
|
+
)
|
|
352
|
+
if is_numeric_dtype(non_null.dtype):
|
|
353
|
+
mean = non_null.mean()
|
|
354
|
+
if pd.isna(mean):
|
|
355
|
+
raise FiltrationError(
|
|
356
|
+
f"allowed reference mean could not be calculated for '{column}'"
|
|
357
|
+
)
|
|
358
|
+
replacement: Any = float(mean)
|
|
359
|
+
if is_integer_dtype(non_null.dtype):
|
|
360
|
+
replacement = int(round(replacement))
|
|
361
|
+
return [replacement]
|
|
362
|
+
return non_null.tolist()
|
|
@@ -0,0 +1,296 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ramen-data-filter
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Dual-mode Pandas and CSV filtration for ramen-ai RAG ingestion and MLOps dataset sanitization.
|
|
5
|
+
License: Proprietary
|
|
6
|
+
Requires-Python: >=3.10
|
|
7
|
+
Requires-Dist: pandas==2.3.3
|
|
8
|
+
Requires-Dist: ramen-ai-core==0.2.1
|
|
9
|
+
Provides-Extra: dev
|
|
10
|
+
Requires-Dist: pytest==8.4.2; extra == 'dev'
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# ramen-ai Data Filter
|
|
14
|
+
|
|
15
|
+
<p align="center">
|
|
16
|
+
<img src="../../assets/ramen-logo.png" alt="ramen-ai" width="100"/>
|
|
17
|
+
</p>
|
|
18
|
+
|
|
19
|
+
Dual-mode Pandas and CSV filtration for RAG ingestion scrubbing and MLOps
|
|
20
|
+
dataset sanitization. Every row is evaluated through the ramen-ai semantic
|
|
21
|
+
firewall before it can enter a retrieval corpus, training set, analytics store,
|
|
22
|
+
or downstream data pipeline.
|
|
23
|
+
|
|
24
|
+
Use **Strict Exclusion** to remove blocked rows, or **Semantic Imputation** to
|
|
25
|
+
preserve row count while replacing only explicitly allowlisted columns named by
|
|
26
|
+
ramen-ai steering. Typical policies can detect PII, data-exfiltration vectors,
|
|
27
|
+
poisoned retrieval context, prompt injection, and other unsafe dataset content.
|
|
28
|
+
|
|
29
|
+
Requires **Python ≥ 3.10**.
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## Supported Ecosystems
|
|
34
|
+
|
|
35
|
+
<p align="center">
|
|
36
|
+
<a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/langchain-python">
|
|
37
|
+
<img src="https://img.shields.io/badge/LangChain-1C3C3C?style=flat&logo=langchain&logoColor=white" alt="LangChain"/>
|
|
38
|
+
</a>
|
|
39
|
+
|
|
40
|
+
<a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/pydantic-ai">
|
|
41
|
+
<img src="https://img.shields.io/badge/PydanticAI-E92063?style=flat&logo=pydantic&logoColor=white" alt="PydanticAI"/>
|
|
42
|
+
</a>
|
|
43
|
+
|
|
44
|
+
<a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/mcp-proxy">
|
|
45
|
+
<img src="https://img.shields.io/badge/MCP-6B21A8?style=flat&logo=anthropic&logoColor=white" alt="MCP"/>
|
|
46
|
+
</a>
|
|
47
|
+
|
|
48
|
+
<a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/agt-typescript">
|
|
49
|
+
<img src="https://img.shields.io/badge/Microsoft%20AGT-0078D4?style=flat&logo=microsoft&logoColor=white" alt="Microsoft AGT"/>
|
|
50
|
+
</a>
|
|
51
|
+
|
|
52
|
+
<a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/github-action">
|
|
53
|
+
<img src="https://img.shields.io/badge/GitHub%20Actions-2088FF?style=flat&logo=githubactions&logoColor=white" alt="GitHub Actions"/>
|
|
54
|
+
</a>
|
|
55
|
+
|
|
56
|
+
<a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/cmcp-python">
|
|
57
|
+
<img src="https://img.shields.io/badge/cMCP-00A67E?style=flat&logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCI+PHBhdGggZmlsbD0id2hpdGUiIGQ9Ik0xMiAyTDIgN2wxMCA1IDEwLTV6TTIgMTdsOCA0VjExbC04LTR6TTE0IDIxbDgtNFYxMWwtOCA0eiIvPjwvc3ZnPg==&logoColor=white" alt="cMCP"/>
|
|
58
|
+
</a>
|
|
59
|
+
|
|
60
|
+
<a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/mlflow-python">
|
|
61
|
+
<img src="https://img.shields.io/badge/MLflow-0194E2?style=flat&logo=mlflow&logoColor=white" alt="MLflow"/>
|
|
62
|
+
</a>
|
|
63
|
+
|
|
64
|
+
<a href="https://github.com/ramen-ai-dev/ramen-ai-integrations/tree/master/plugins/ramen-data-filter">
|
|
65
|
+
<img src="https://img.shields.io/badge/ramen%20data%20filter-D97706?style=flat&logo=pandas&logoColor=white" alt="ramen data filter"/>
|
|
66
|
+
</a>
|
|
67
|
+
</p>
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## Can you bypass it?
|
|
72
|
+
|
|
73
|
+
Standard safety filters catch basic syntax. They fail against encoded payloads
|
|
74
|
+
and corporate jargon. We challenge you to bypass our semantic firewall using
|
|
75
|
+
the zero-day evasion vectors in our official **[Red Team Guide](../../RED_TEAM_GUIDE.md)**.
|
|
76
|
+
|
|
77
|
+
Below is a simulation of the Grok/Bankr heist. We fed the raw adversarial
|
|
78
|
+
prompt directly into our sandbox. It uses a social engineering wrapper
|
|
79
|
+
(claiming a visual impairment) to smuggle a 3,000,000,000 DRB transfer
|
|
80
|
+
instruction encoded in Morse code. The firewall evaluated the underlying
|
|
81
|
+
semantic intent, intercepted the unauthorized financial transfer, and blocked
|
|
82
|
+
it pre-execution, issuing a verified Ed25519 receipt.
|
|
83
|
+
|
|
84
|
+
<p align="center">
|
|
85
|
+
<img src="../../assets/grok-bankr.png" alt="ramen-ai intercepting the Grok/Bankr Morse-code heist pre-execution" width="720"/>
|
|
86
|
+
</p>
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## API Key
|
|
91
|
+
|
|
92
|
+
To use this integration, you must mint an API Key. We offer a **Free Starter
|
|
93
|
+
Tier** (1,000 evaluations/month, BYOK) which includes full access to our Core
|
|
94
|
+
IT Security bundle. Mint your key at:
|
|
95
|
+
**[https://ramenai.dev/pricing](https://ramenai.dev/pricing)**
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## Installation
|
|
100
|
+
|
|
101
|
+
From PyPI after release:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
pip install ramen-data-filter
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
From this monorepo:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
pip install -e plugins/ramen-data-filter
|
|
111
|
+
pip install -e "plugins/ramen-data-filter[dev]" # include pytest
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Configuration / BYOK
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
export RAMEN_API_KEY=ramen_ak_... # required
|
|
120
|
+
export OPENAI_API_KEY=sk-... # Starter/Professional BYOK
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
`RAMEN_API_KEY` authenticates the evaluation request. The Starter and
|
|
124
|
+
Professional tiers also require an LLM provider key. The plugin reads
|
|
125
|
+
`OPENAI_API_KEY` at evaluation time and `ramen-ai-core` forwards it as the
|
|
126
|
+
`X-Provider-Key` header. Without a provider key, those tiers return
|
|
127
|
+
`402 Payment Required`.
|
|
128
|
+
|
|
129
|
+
Pass `provider_name="anthropic"`, `"google"`, `"synthetic"`, or
|
|
130
|
+
`"hyperbolic"` with the corresponding `provider_key` when OpenAI is not the
|
|
131
|
+
provider. Enterprise tiers use platform-managed provider keys and do not need
|
|
132
|
+
`OPENAI_API_KEY`.
|
|
133
|
+
|
|
134
|
+
Credentials are read from environment variables or passed at runtime. Never
|
|
135
|
+
store them in source code, notebooks, CSV files, or committed configuration.
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
## Usage
|
|
140
|
+
|
|
141
|
+
### Strict Exclusion
|
|
142
|
+
|
|
143
|
+
Strict exclusion evaluates each row and returns a new DataFrame containing only
|
|
144
|
+
`[ALLOWED]` rows. The returned index is reset; the source DataFrame is not
|
|
145
|
+
modified.
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
import pandas as pd
|
|
149
|
+
from ramen_data_filter import FiltrationMode, filter_dataframe
|
|
150
|
+
|
|
151
|
+
records = pd.DataFrame(
|
|
152
|
+
[
|
|
153
|
+
{"document_id": "doc-1", "content": "Approved public documentation"},
|
|
154
|
+
{"document_id": "doc-2", "content": "Retrieved untrusted context"},
|
|
155
|
+
]
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
result = filter_dataframe(
|
|
159
|
+
records,
|
|
160
|
+
mode=FiltrationMode.STRICT_EXCLUSION,
|
|
161
|
+
bundle_ids=["ramen__shield_core_it"],
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
clean_records = result.dataframe
|
|
165
|
+
print(result.audit_log[["row_index", "verdict", "receipt_verified"]])
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
### Semantic Imputation
|
|
169
|
+
|
|
170
|
+
Semantic imputation preserves every row. A blocked row may change only a column
|
|
171
|
+
that is both listed in `remediable_columns` and named in the ramen-ai steering
|
|
172
|
+
instruction. Numeric columns receive the allowed-reference mean (rounded for
|
|
173
|
+
integer data); non-numeric columns use deterministic hot-deck values from
|
|
174
|
+
allowed rows. All other values remain unchanged.
|
|
175
|
+
|
|
176
|
+
```python
|
|
177
|
+
result = filter_dataframe(
|
|
178
|
+
records,
|
|
179
|
+
mode=FiltrationMode.SEMANTIC_IMPUTATION,
|
|
180
|
+
policy_ids=["<POLICY_UUID>"],
|
|
181
|
+
remediable_columns=["content"],
|
|
182
|
+
)
|
|
183
|
+
|
|
184
|
+
sanitized_records = result.dataframe
|
|
185
|
+
print(result.imputation_log)
|
|
186
|
+
```
|
|
187
|
+
|
|
188
|
+
### CSV pipeline
|
|
189
|
+
|
|
190
|
+
```python
|
|
191
|
+
from ramen_data_filter import FiltrationMode, filter_csv
|
|
192
|
+
|
|
193
|
+
result = filter_csv(
|
|
194
|
+
"rag-ingestion.csv",
|
|
195
|
+
"rag-ingestion-sanitized.csv",
|
|
196
|
+
mode=FiltrationMode.STRICT_EXCLUSION,
|
|
197
|
+
bundle_ids=["ramen__shield_core_it"],
|
|
198
|
+
)
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
The destination is written only after every row has been evaluated and the
|
|
202
|
+
transformation succeeds. Existing destination files are replaced by Pandas.
|
|
203
|
+
|
|
204
|
+
---
|
|
205
|
+
|
|
206
|
+
## How it works
|
|
207
|
+
|
|
208
|
+
1. Serializes each row as stable, key-sorted JSON.
|
|
209
|
+
2. Calls `RamenClient.evaluate_compliance` with configured bundles or policies.
|
|
210
|
+
3. Records the verdict, steering, resolved policy IDs, receipt verification
|
|
211
|
+
state, and complete SDK response in `audit_log`.
|
|
212
|
+
4. Applies the selected mode only after all rows have been evaluated.
|
|
213
|
+
5. Returns a new DataFrame and transformation metadata; CSV usage writes that
|
|
214
|
+
DataFrame with `index=False`.
|
|
215
|
+
|
|
216
|
+
The pipeline is fail-closed. Missing credentials, transport failures, malformed
|
|
217
|
+
responses, unsafe imputation steering, and unusable reference values raise
|
|
218
|
+
`FiltrationError`; partial filtered output is never returned or written.
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## API reference
|
|
223
|
+
|
|
224
|
+
### `filter_dataframe(dataframe, *, mode, ...) -> FiltrationResult`
|
|
225
|
+
|
|
226
|
+
| Parameter | Type | Description |
|
|
227
|
+
|---|---|---|
|
|
228
|
+
| `dataframe` | `pandas.DataFrame` | Source records. The input is not modified. |
|
|
229
|
+
| `mode` | `FiltrationMode \| str` | `strict_exclusion` or `semantic_imputation`. |
|
|
230
|
+
| `bundle_ids` | `Sequence[str]` | ramen-ai bundle slugs. At least one bundle or policy is required. |
|
|
231
|
+
| `policy_ids` | `Sequence[str]` | Explicit policy UUIDs. May be combined with bundles. |
|
|
232
|
+
| `remediable_columns` | `Sequence[str]` | Columns eligible for steering-constrained replacement. Required for semantic imputation. |
|
|
233
|
+
| `client` | `RamenClient` | Optional injected client, primarily for controlled runtimes and tests. |
|
|
234
|
+
| `api_key` | `str` | Runtime override; defaults to `RAMEN_API_KEY`. |
|
|
235
|
+
| `provider_key` | `str` | BYOK override; defaults to `OPENAI_API_KEY`. |
|
|
236
|
+
| `provider_name` | `str` | Optional BYOK provider routing hint. |
|
|
237
|
+
| `context` | `dict[str, str]` | Additional audit metadata. |
|
|
238
|
+
| `base_url` | `str` | ramen-ai API base URL. |
|
|
239
|
+
| `timeout` | `float` | HTTP timeout in seconds. |
|
|
240
|
+
|
|
241
|
+
### `filter_csv(source_path, destination_path, **kwargs) -> FiltrationResult`
|
|
242
|
+
|
|
243
|
+
Reads the source with `pandas.read_csv`, applies `filter_dataframe`, and writes
|
|
244
|
+
the resulting DataFrame with `index=False`.
|
|
245
|
+
|
|
246
|
+
### `FiltrationResult`
|
|
247
|
+
|
|
248
|
+
| Attribute | Description |
|
|
249
|
+
|---|---|
|
|
250
|
+
| `dataframe` | Filtered or imputed DataFrame. |
|
|
251
|
+
| `audit_log` | One row per evaluation with verdict and SDK response metadata. |
|
|
252
|
+
| `imputation_log` | One row per transformed record; empty in strict mode. |
|
|
253
|
+
|
|
254
|
+
Public exports: `FiltrationError`, `FiltrationMode`, `FiltrationResult`,
|
|
255
|
+
`filter_dataframe`, and `filter_csv`.
|
|
256
|
+
|
|
257
|
+
---
|
|
258
|
+
|
|
259
|
+
## Running the tests
|
|
260
|
+
|
|
261
|
+
```bash
|
|
262
|
+
PYTHONPATH="core-clients/python:plugins/ramen-data-filter/src" \
|
|
263
|
+
python -m pytest plugins/ramen-data-filter/tests -v
|
|
264
|
+
```
|
|
265
|
+
|
|
266
|
+
The isolated tests use a mocked `RamenClient`; they make no network calls and
|
|
267
|
+
require no credentials. They prove blocked-row exclusion and constrained
|
|
268
|
+
semantic imputation on a dummy DataFrame.
|
|
269
|
+
|
|
270
|
+
---
|
|
271
|
+
|
|
272
|
+
## Available bundles
|
|
273
|
+
|
|
274
|
+
| Bundle slug | Useful dataset coverage |
|
|
275
|
+
|---|---|
|
|
276
|
+
| `ramen__shield_core_it` | Secret exfiltration, prompt leakage, jailbreaks, destructive instructions, and indirect prompt injection. |
|
|
277
|
+
| `ramen__eu_ai_act_baseline` | EU AI Act data-governance, prohibited-practice, and transparency controls. |
|
|
278
|
+
|
|
279
|
+
Pass explicit `policy_ids` for custom PII, ingestion, data-quality, or internal
|
|
280
|
+
MLOps controls. Bundle and policy details are available at
|
|
281
|
+
[ramenai.dev/pricing](https://ramenai.dev/pricing).
|
|
282
|
+
|
|
283
|
+
---
|
|
284
|
+
|
|
285
|
+
## Limitations
|
|
286
|
+
|
|
287
|
+
- Evaluation adds one API request per row. Large datasets should be processed in
|
|
288
|
+
controlled batches with rate limits and resumability at the orchestration
|
|
289
|
+
layer.
|
|
290
|
+
- Semantic imputation creates synthetic values; it does not recover original
|
|
291
|
+
truth or establish legal compliance, statistical fairness, or model quality.
|
|
292
|
+
- Imputed rows are not automatically re-evaluated. Validate and, where required,
|
|
293
|
+
re-evaluate transformed output before production ingestion or training.
|
|
294
|
+
- Non-numeric hot-deck replacement is deterministic and may alter dataset
|
|
295
|
+
distributions. Preserve source data and review `audit_log` and
|
|
296
|
+
`imputation_log` in an access-controlled audit store.
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
ramen_data_filter/__init__.py,sha256=xqfwzAHi7s0f9aEn7UNDPFw2UBgyA3X9gkhBmPL0o20,338
|
|
2
|
+
ramen_data_filter/filter.py,sha256=lx1lSyAhTaEHgjCzYsYbdapvOXOzhlnf77K7e-fB2MY,13001
|
|
3
|
+
ramen_data_filter-0.1.0.dist-info/METADATA,sha256=0te7oYCjMSBmHJFhocv6BMMHjDHFLARJP8QU6YnVkP0,11211
|
|
4
|
+
ramen_data_filter-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
5
|
+
ramen_data_filter-0.1.0.dist-info/RECORD,,
|