llama-index-postprocessor-obscate 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llama_index/postprocessor/obscate/__init__.py +7 -0
- llama_index/postprocessor/obscate/_client.py +306 -0
- llama_index/postprocessor/obscate/base.py +242 -0
- llama_index_postprocessor_obscate-0.1.0.dist-info/METADATA +177 -0
- llama_index_postprocessor_obscate-0.1.0.dist-info/RECORD +7 -0
- llama_index_postprocessor_obscate-0.1.0.dist-info/WHEEL +4 -0
- llama_index_postprocessor_obscate-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Obscate reversible PII redaction for LlamaIndex."""
|
|
2
|
+
from ._client import ObscateClient, ObscateError, RestoreContext
|
|
3
|
+
from .base import ObscateNodeRedactor, ObscatePostprocessor, ObscateQueryEngine
|
|
4
|
+
|
|
5
|
+
__version__ = "0.1.0"
|
|
6
|
+
__all__ = ["ObscatePostprocessor", "ObscateQueryEngine", "ObscateNodeRedactor", "ObscateClient",
|
|
7
|
+
"ObscateError", "RestoreContext"]
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
"""Minimal client for the Obscate HTTP API, shared by the Obscate integrations.
|
|
2
|
+
|
|
3
|
+
This file is vendored byte-for-byte into each package (obscate-mcp,
|
|
4
|
+
langchain-obscate, llama-index-postprocessor-obscate) so every package installs
|
|
5
|
+
on its own. Edit ``shared/_client.py`` and run ``python scripts/sync_client.py``;
|
|
6
|
+
a test fails if the copies drift.
|
|
7
|
+
|
|
8
|
+
Privacy contract (tested in every package):
|
|
9
|
+
* Nothing here logs, prints or writes to disk the text you send, the values
|
|
10
|
+
detected, or the restore map.
|
|
11
|
+
* ``ObscateError`` carries the HTTP status and the server's ``detail`` only.
|
|
12
|
+
The API never echoes request bodies in errors, so neither do we.
|
|
13
|
+
* ``RestoreContext`` keeps the token map in memory only and hides it from
|
|
14
|
+
``repr()`` so it does not end up in tracebacks or debug logs by accident.
|
|
15
|
+
|
|
16
|
+
Key versions: every redact response carries ``key_version`` (the version of
|
|
17
|
+
your account's key that made the tokens). ``RestoreContext`` keeps it and
|
|
18
|
+
every restore sends it back, so a redact -> LLM -> restore round trip keeps
|
|
19
|
+
working across a server-side key rotation with no code on your side. If you
|
|
20
|
+
store fpe tokens to restore later without a context, store ``key_version``
|
|
21
|
+
with them too (see https://obscate.com/docs/strategies).
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import os
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from typing import Any, Iterable
|
|
28
|
+
|
|
29
|
+
import httpx
|
|
30
|
+
|
|
31
|
+
DEFAULT_BASE_URL = "https://api.obscate.com"
|
|
32
|
+
STRATEGIES = ("fpe", "tokenize", "synthetic", "mask", "hash", "partial")
|
|
33
|
+
REVERSIBLE = ("fpe", "tokenize", "synthetic")
|
|
34
|
+
SIGNUP_URL = "https://obscate.com"
|
|
35
|
+
|
|
36
|
+
__all__ = ["ObscateClient", "ObscateError", "RestoreContext", "DEFAULT_BASE_URL",
|
|
37
|
+
"STRATEGIES", "REVERSIBLE"]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class ObscateError(RuntimeError):
|
|
41
|
+
"""An Obscate API call failed. Holds the status code and the server's
|
|
42
|
+
``detail`` message — never any of the text you sent."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, message: str, status: int | None = None, request_id: str | None = None):
|
|
45
|
+
super().__init__(message)
|
|
46
|
+
self.status = status
|
|
47
|
+
self.request_id = request_id
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class RestoreContext:
|
|
52
|
+
"""What is needed to undo one redaction. In-memory only; never logged.
|
|
53
|
+
|
|
54
|
+
For ``fpe`` the context holds no secrets at all (restore needs only your
|
|
55
|
+
API key). For ``tokenize``/``synthetic`` it holds the token map, which
|
|
56
|
+
contains the original values — keep it in memory and let it go when done.
|
|
57
|
+
"""
|
|
58
|
+
strategy: str
|
|
59
|
+
token_map: dict[str, str] = field(default_factory=dict, repr=False)
|
|
60
|
+
type_changes: list[dict] = field(default_factory=list, repr=False)
|
|
61
|
+
keys_redacted: bool = False
|
|
62
|
+
labels: list[str] | None = None
|
|
63
|
+
min_confidence: float = 0.0
|
|
64
|
+
hints: bool = True
|
|
65
|
+
field_map: dict[str, str | None] | None = field(default=None, repr=False)
|
|
66
|
+
key_version: int | None = None # from the redact response; None = server's current
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def reversible(self) -> bool:
|
|
70
|
+
return self.strategy in REVERSIBLE
|
|
71
|
+
|
|
72
|
+
def merge(self, other: "RestoreContext") -> "RestoreContext":
|
|
73
|
+
"""Combine two contexts from the same strategy (token maps are unioned)."""
|
|
74
|
+
if other.strategy != self.strategy:
|
|
75
|
+
raise ValueError("cannot merge restore contexts from different strategies")
|
|
76
|
+
if None not in (self.key_version, other.key_version) and self.key_version != other.key_version:
|
|
77
|
+
raise ValueError("cannot merge restore contexts made under different key versions")
|
|
78
|
+
merged = dict(self.token_map)
|
|
79
|
+
merged.update(other.token_map)
|
|
80
|
+
kv = self.key_version if self.key_version is not None else other.key_version
|
|
81
|
+
return RestoreContext(self.strategy, merged, [], False, self.labels,
|
|
82
|
+
self.min_confidence, self.hints, self.field_map, kv)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _check_strategy(strategy: str) -> None:
|
|
86
|
+
if strategy not in STRATEGIES:
|
|
87
|
+
raise ValueError(f"strategy must be one of {list(STRATEGIES)}")
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class ObscateClient:
|
|
91
|
+
"""Thin synchronous + asynchronous wrapper over the Obscate REST API.
|
|
92
|
+
|
|
93
|
+
``api_key`` defaults to ``$OBSCATE_API_KEY``; ``base_url`` defaults to
|
|
94
|
+
``$OBSCATE_BASE_URL`` or https://api.obscate.com (point it at a
|
|
95
|
+
self-hosted instance if you run one).
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
def __init__(self, api_key: str | None = None, base_url: str | None = None,
|
|
99
|
+
timeout: float = 30.0, user_agent: str = "obscate-python-integration",
|
|
100
|
+
transport: httpx.BaseTransport | None = None,
|
|
101
|
+
async_transport: httpx.AsyncBaseTransport | None = None):
|
|
102
|
+
self._api_key = api_key if api_key is not None else os.environ.get("OBSCATE_API_KEY", "")
|
|
103
|
+
self.base_url = (base_url or os.environ.get("OBSCATE_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
|
|
104
|
+
self.timeout = timeout
|
|
105
|
+
self._headers = {"user-agent": user_agent, "accept": "application/json"}
|
|
106
|
+
self._transport = transport
|
|
107
|
+
self._async_transport = async_transport
|
|
108
|
+
self._http: httpx.Client | None = None
|
|
109
|
+
self._ahttp: httpx.AsyncClient | None = None
|
|
110
|
+
|
|
111
|
+
def __repr__(self) -> str: # never show the key
|
|
112
|
+
return f"ObscateClient(base_url={self.base_url!r}, api_key={'set' if self._api_key else 'missing'})"
|
|
113
|
+
|
|
114
|
+
# -- plumbing ---------------------------------------------------------------
|
|
115
|
+
def _auth_headers(self) -> dict[str, str]:
|
|
116
|
+
if not self._api_key:
|
|
117
|
+
raise ObscateError(
|
|
118
|
+
"OBSCATE_API_KEY is not set. Get a free key (1,000 requests/month) at "
|
|
119
|
+
f"{SIGNUP_URL} and export OBSCATE_API_KEY, or pass api_key=...")
|
|
120
|
+
return {**self._headers, "x-api-key": self._api_key}
|
|
121
|
+
|
|
122
|
+
@staticmethod
|
|
123
|
+
def _handle(resp: httpx.Response) -> Any:
|
|
124
|
+
if resp.status_code < 400:
|
|
125
|
+
return resp.json()
|
|
126
|
+
try:
|
|
127
|
+
detail = resp.json().get("detail")
|
|
128
|
+
except Exception:
|
|
129
|
+
detail = None
|
|
130
|
+
if not isinstance(detail, str):
|
|
131
|
+
# 422 validation errors are a list of {loc,msg,type}; keep field paths + messages only.
|
|
132
|
+
if isinstance(detail, list):
|
|
133
|
+
detail = "; ".join(f"{'.'.join(str(p) for p in d.get('loc', []))}: {d.get('msg')}"
|
|
134
|
+
for d in detail if isinstance(d, dict))
|
|
135
|
+
else:
|
|
136
|
+
detail = resp.reason_phrase
|
|
137
|
+
hint = {401: " (check OBSCATE_API_KEY)",
|
|
138
|
+
402: " (monthly quota reached; see https://obscate.com/pricing)",
|
|
139
|
+
429: f" (retry after {resp.headers.get('retry-after', '?')}s)",
|
|
140
|
+
503: " (detection failed closed: nothing was redacted)"}.get(resp.status_code, "")
|
|
141
|
+
raise ObscateError(f"Obscate API error {resp.status_code}: {detail}{hint}",
|
|
142
|
+
status=resp.status_code, request_id=resp.headers.get("x-request-id"))
|
|
143
|
+
|
|
144
|
+
def _client(self) -> httpx.Client:
|
|
145
|
+
if self._http is None:
|
|
146
|
+
self._http = httpx.Client(base_url=self.base_url, timeout=self.timeout,
|
|
147
|
+
transport=self._transport)
|
|
148
|
+
return self._http
|
|
149
|
+
|
|
150
|
+
def _aclient(self) -> httpx.AsyncClient:
|
|
151
|
+
if self._ahttp is None:
|
|
152
|
+
self._ahttp = httpx.AsyncClient(base_url=self.base_url, timeout=self.timeout,
|
|
153
|
+
transport=self._async_transport)
|
|
154
|
+
return self._ahttp
|
|
155
|
+
|
|
156
|
+
def _post(self, path: str, body: dict) -> Any:
|
|
157
|
+
try:
|
|
158
|
+
resp = self._client().post(path, json=body, headers=self._auth_headers())
|
|
159
|
+
except httpx.HTTPError as e: # network errors carry the URL, not the body
|
|
160
|
+
raise ObscateError(f"could not reach Obscate at {self.base_url}: {type(e).__name__}") from None
|
|
161
|
+
return self._handle(resp)
|
|
162
|
+
|
|
163
|
+
async def _apost(self, path: str, body: dict) -> Any:
|
|
164
|
+
try:
|
|
165
|
+
resp = await self._aclient().post(path, json=body, headers=self._auth_headers())
|
|
166
|
+
except httpx.HTTPError as e:
|
|
167
|
+
raise ObscateError(f"could not reach Obscate at {self.base_url}: {type(e).__name__}") from None
|
|
168
|
+
return self._handle(resp)
|
|
169
|
+
|
|
170
|
+
def close(self) -> None:
|
|
171
|
+
if self._http is not None:
|
|
172
|
+
self._http.close()
|
|
173
|
+
self._http = None
|
|
174
|
+
|
|
175
|
+
async def aclose(self) -> None:
|
|
176
|
+
if self._ahttp is not None:
|
|
177
|
+
await self._ahttp.aclose()
|
|
178
|
+
self._ahttp = None
|
|
179
|
+
|
|
180
|
+
# -- request bodies (shared by sync + async) ----------------------------------
|
|
181
|
+
@staticmethod
|
|
182
|
+
def _redact_body(text, strategy, labels, min_confidence):
|
|
183
|
+
_check_strategy(strategy)
|
|
184
|
+
return {"text": text, "strategy": strategy, "labels": labels, "min_confidence": min_confidence}
|
|
185
|
+
|
|
186
|
+
@staticmethod
|
|
187
|
+
def _redact_json_body(payload, strategy, labels, min_confidence, hints, field_map, keys):
|
|
188
|
+
_check_strategy(strategy)
|
|
189
|
+
if keys not in ("report", "strict", "redact"):
|
|
190
|
+
raise ValueError("keys must be 'report', 'strict' or 'redact'")
|
|
191
|
+
return {"json": payload, "strategy": strategy, "labels": labels,
|
|
192
|
+
"min_confidence": min_confidence, "hints": hints, "field_map": field_map, "keys": keys}
|
|
193
|
+
|
|
194
|
+
@staticmethod
|
|
195
|
+
def _restore_route(text: str, ctx: RestoreContext) -> tuple[str, dict]:
|
|
196
|
+
if ctx.strategy == "fpe":
|
|
197
|
+
body = {"text": text, "labels": ctx.labels, "min_confidence": ctx.min_confidence}
|
|
198
|
+
if ctx.key_version is not None:
|
|
199
|
+
body["key_version"] = ctx.key_version
|
|
200
|
+
return "/v1/restore-fpe", body
|
|
201
|
+
if ctx.strategy in ("tokenize", "synthetic"):
|
|
202
|
+
return "/v1/restore", {"text": text, "token_map": ctx.token_map}
|
|
203
|
+
raise ValueError(f"strategy {ctx.strategy!r} is one-way and cannot be restored; "
|
|
204
|
+
f"use one of {list(REVERSIBLE)}")
|
|
205
|
+
|
|
206
|
+
@staticmethod
|
|
207
|
+
def _restore_json_body(payload: Any, ctx: RestoreContext) -> dict:
|
|
208
|
+
if not ctx.reversible:
|
|
209
|
+
raise ValueError(f"strategy {ctx.strategy!r} is one-way and cannot be restored")
|
|
210
|
+
body = {"json": payload, "token_map": ctx.token_map, "fpe": ctx.strategy == "fpe",
|
|
211
|
+
"type_changes": ctx.type_changes or None, "keys_redacted": ctx.keys_redacted,
|
|
212
|
+
"labels": ctx.labels, "min_confidence": ctx.min_confidence,
|
|
213
|
+
"hints": ctx.hints, "field_map": ctx.field_map}
|
|
214
|
+
if ctx.key_version is not None:
|
|
215
|
+
body["key_version"] = ctx.key_version
|
|
216
|
+
return body
|
|
217
|
+
|
|
218
|
+
@staticmethod
|
|
219
|
+
def _ctx(r: dict, strategy, labels, min_confidence, hints=True, field_map=None) -> RestoreContext:
|
|
220
|
+
kv = r.get("key_version")
|
|
221
|
+
return RestoreContext(strategy, dict(r.get("token_map") or {}), list(r.get("type_changes") or []),
|
|
222
|
+
bool(r.get("keys_redacted")), labels, min_confidence, hints, field_map,
|
|
223
|
+
kv if isinstance(kv, int) else None)
|
|
224
|
+
|
|
225
|
+
# -- sync API ---------------------------------------------------------------
|
|
226
|
+
def redact(self, text: str, strategy: str = "fpe", labels: list[str] | None = None,
|
|
227
|
+
min_confidence: float = 0.0) -> tuple[dict, RestoreContext]:
|
|
228
|
+
"""POST /v1/redact. Returns the raw response and a context for restore()."""
|
|
229
|
+
r = self._post("/v1/redact", self._redact_body(text, strategy, labels, min_confidence))
|
|
230
|
+
return r, self._ctx(r, strategy, labels, min_confidence)
|
|
231
|
+
|
|
232
|
+
def restore(self, text: str, ctx: RestoreContext) -> str:
|
|
233
|
+
"""POST /v1/restore (token map) or /v1/restore-fpe (map-free). Free."""
|
|
234
|
+
path, body = self._restore_route(text, ctx)
|
|
235
|
+
return self._post(path, body)["text"]
|
|
236
|
+
|
|
237
|
+
def detect(self, text: str, labels: list[str] | None = None, min_confidence: float = 0.0) -> dict:
|
|
238
|
+
"""POST /v1/detect. Metered like redact."""
|
|
239
|
+
return self._post("/v1/detect", {"text": text, "labels": labels, "min_confidence": min_confidence})
|
|
240
|
+
|
|
241
|
+
def redact_json(self, payload: Any, strategy: str = "fpe", labels: list[str] | None = None,
|
|
242
|
+
min_confidence: float = 0.0, hints: bool = True,
|
|
243
|
+
field_map: dict[str, str | None] | None = None,
|
|
244
|
+
keys: str = "report") -> tuple[dict, RestoreContext]:
|
|
245
|
+
"""POST /v1/redact-json. One request (one unit up to 100 KB) for the whole payload."""
|
|
246
|
+
r = self._post("/v1/redact-json", self._redact_json_body(payload, strategy, labels, min_confidence,
|
|
247
|
+
hints, field_map, keys))
|
|
248
|
+
return r, self._ctx(r, strategy, labels, min_confidence, hints, field_map)
|
|
249
|
+
|
|
250
|
+
def restore_json(self, payload: Any, ctx: RestoreContext) -> Any:
|
|
251
|
+
"""POST /v1/restore-json. Free."""
|
|
252
|
+
return self._post("/v1/restore-json", self._restore_json_body(payload, ctx))["json"]
|
|
253
|
+
|
|
254
|
+
def redact_texts(self, texts: Iterable[str], strategy: str = "fpe", labels: list[str] | None = None,
|
|
255
|
+
min_confidence: float = 0.0) -> tuple[list[str], RestoreContext]:
|
|
256
|
+
"""Redact several strings in ONE request with consistent tokens across them.
|
|
257
|
+
|
|
258
|
+
Uses /v1/redact-json on a JSON array, so ``[PERSON_1]`` means the same
|
|
259
|
+
person in every string and the whole batch costs one unit (≤ 100 KB).
|
|
260
|
+
"""
|
|
261
|
+
texts = list(texts)
|
|
262
|
+
if not texts:
|
|
263
|
+
return [], RestoreContext(strategy, labels=labels, min_confidence=min_confidence)
|
|
264
|
+
r, ctx = self.redact_json(texts, strategy, labels, min_confidence, hints=False)
|
|
265
|
+
return list(r["redacted"]), ctx
|
|
266
|
+
|
|
267
|
+
def entities(self) -> dict:
|
|
268
|
+
"""GET /v1/entities (public, no key needed)."""
|
|
269
|
+
try:
|
|
270
|
+
resp = self._client().get("/v1/entities", headers=self._headers)
|
|
271
|
+
except httpx.HTTPError as e:
|
|
272
|
+
raise ObscateError(f"could not reach Obscate at {self.base_url}: {type(e).__name__}") from None
|
|
273
|
+
return self._handle(resp)
|
|
274
|
+
|
|
275
|
+
# -- async API --------------------------------------------------------------
|
|
276
|
+
async def aredact(self, text: str, strategy: str = "fpe", labels: list[str] | None = None,
|
|
277
|
+
min_confidence: float = 0.0) -> tuple[dict, RestoreContext]:
|
|
278
|
+
r = await self._apost("/v1/redact", self._redact_body(text, strategy, labels, min_confidence))
|
|
279
|
+
return r, self._ctx(r, strategy, labels, min_confidence)
|
|
280
|
+
|
|
281
|
+
async def arestore(self, text: str, ctx: RestoreContext) -> str:
|
|
282
|
+
path, body = self._restore_route(text, ctx)
|
|
283
|
+
return (await self._apost(path, body))["text"]
|
|
284
|
+
|
|
285
|
+
async def adetect(self, text: str, labels: list[str] | None = None, min_confidence: float = 0.0) -> dict:
|
|
286
|
+
return await self._apost("/v1/detect", {"text": text, "labels": labels, "min_confidence": min_confidence})
|
|
287
|
+
|
|
288
|
+
async def aredact_json(self, payload: Any, strategy: str = "fpe", labels: list[str] | None = None,
|
|
289
|
+
min_confidence: float = 0.0, hints: bool = True,
|
|
290
|
+
field_map: dict[str, str | None] | None = None,
|
|
291
|
+
keys: str = "report") -> tuple[dict, RestoreContext]:
|
|
292
|
+
r = await self._apost("/v1/redact-json", self._redact_json_body(payload, strategy, labels,
|
|
293
|
+
min_confidence, hints, field_map, keys))
|
|
294
|
+
return r, self._ctx(r, strategy, labels, min_confidence, hints, field_map)
|
|
295
|
+
|
|
296
|
+
async def arestore_json(self, payload: Any, ctx: RestoreContext) -> Any:
|
|
297
|
+
return (await self._apost("/v1/restore-json", self._restore_json_body(payload, ctx)))["json"]
|
|
298
|
+
|
|
299
|
+
async def aredact_texts(self, texts: Iterable[str], strategy: str = "fpe",
|
|
300
|
+
labels: list[str] | None = None,
|
|
301
|
+
min_confidence: float = 0.0) -> tuple[list[str], RestoreContext]:
|
|
302
|
+
texts = list(texts)
|
|
303
|
+
if not texts:
|
|
304
|
+
return [], RestoreContext(strategy, labels=labels, min_confidence=min_confidence)
|
|
305
|
+
r, ctx = await self.aredact_json(texts, strategy, labels, min_confidence, hints=False)
|
|
306
|
+
return list(r["redacted"]), ctx
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
"""Obscate for LlamaIndex: redact what the LLM sees, restore what it says.
|
|
2
|
+
|
|
3
|
+
Two building blocks:
|
|
4
|
+
|
|
5
|
+
* ``ObscateQueryEngine``: the one-step option. It wraps a retriever (or an
|
|
6
|
+
existing ``index.as_query_engine()``), redacts the query and the retrieved
|
|
7
|
+
nodes in one Obscate request, runs the response synthesizer on the redacted
|
|
8
|
+
data, and restores the answer. ``source_nodes`` stay the original nodes, so
|
|
9
|
+
your citations are real.
|
|
10
|
+
* ``ObscatePostprocessor``: a node postprocessor that redacts retrieved node
|
|
11
|
+
text and metadata, for pipelines you assemble yourself. Restore the final
|
|
12
|
+
answer with ``postprocessor.restore(text)``.
|
|
13
|
+
|
|
14
|
+
Scope: this protects what the LLM *provider* receives. Embeddings are
|
|
15
|
+
computed when you index and query. If your embedding model is a remote API,
|
|
16
|
+
it still sees raw text. Use a local embedding model, or redact documents
|
|
17
|
+
before indexing.
|
|
18
|
+
"""
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import threading
|
|
22
|
+
from typing import Any, List, Optional
|
|
23
|
+
|
|
24
|
+
from llama_index.core.base.base_retriever import BaseRetriever
|
|
25
|
+
from llama_index.core.base.response.schema import Response, StreamingResponse
|
|
26
|
+
from llama_index.core.postprocessor.types import BaseNodePostprocessor
|
|
27
|
+
from llama_index.core.query_engine.custom import CustomQueryEngine
|
|
28
|
+
from llama_index.core.response_synthesizers import BaseSynthesizer, get_response_synthesizer
|
|
29
|
+
from llama_index.core.schema import MetadataMode, NodeWithScore, QueryBundle
|
|
30
|
+
from pydantic import Field, PrivateAttr
|
|
31
|
+
|
|
32
|
+
from ._client import STRATEGIES, ObscateClient, RestoreContext
|
|
33
|
+
|
|
34
|
+
__all__ = ["ObscatePostprocessor", "ObscateQueryEngine", "ObscateNodeRedactor"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _has_text(obj: Any) -> bool:
|
|
38
|
+
if isinstance(obj, str):
|
|
39
|
+
return bool(obj.strip())
|
|
40
|
+
if isinstance(obj, dict):
|
|
41
|
+
return any(_has_text(v) for v in obj.values())
|
|
42
|
+
if isinstance(obj, list):
|
|
43
|
+
return any(_has_text(v) for v in obj)
|
|
44
|
+
return False
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ObscateNodeRedactor:
|
|
48
|
+
"""Shared logic: one Obscate request per query for the query + all nodes."""
|
|
49
|
+
|
|
50
|
+
def __init__(self, strategy: str = "fpe", labels: Optional[list[str]] = None,
|
|
51
|
+
min_confidence: float = 0.0, redact_metadata: bool = True,
|
|
52
|
+
api_key: Optional[str] = None, base_url: Optional[str] = None,
|
|
53
|
+
client: Optional[ObscateClient] = None):
|
|
54
|
+
if strategy not in STRATEGIES:
|
|
55
|
+
raise ValueError(f"strategy must be one of {list(STRATEGIES)}")
|
|
56
|
+
self.strategy, self.labels, self.min_confidence = strategy, labels, min_confidence
|
|
57
|
+
self.redact_metadata = redact_metadata
|
|
58
|
+
self.client = client or ObscateClient(api_key=api_key, base_url=base_url,
|
|
59
|
+
user_agent="llama-index-postprocessor-obscate/0.1.0")
|
|
60
|
+
|
|
61
|
+
def __repr__(self) -> str:
|
|
62
|
+
return f"ObscateNodeRedactor(strategy={self.strategy!r}, client={self.client!r})"
|
|
63
|
+
|
|
64
|
+
def _entry(self, n: NodeWithScore) -> dict:
|
|
65
|
+
e: dict[str, Any] = {"t": n.node.get_content(metadata_mode=MetadataMode.NONE)}
|
|
66
|
+
if self.redact_metadata:
|
|
67
|
+
hidden = set(n.node.excluded_llm_metadata_keys or [])
|
|
68
|
+
e["m"] = {k: v for k, v in (n.node.metadata or {}).items()
|
|
69
|
+
if isinstance(v, str) and k not in hidden}
|
|
70
|
+
return e
|
|
71
|
+
|
|
72
|
+
@staticmethod
|
|
73
|
+
def _copy(n: NodeWithScore, e: dict) -> NodeWithScore:
|
|
74
|
+
# Copy: the retrieved node objects are shared with your index/docstore.
|
|
75
|
+
node = n.node.model_copy(deep=True)
|
|
76
|
+
node.set_content(e["t"])
|
|
77
|
+
if "m" in e:
|
|
78
|
+
node.metadata = {**node.metadata, **e["m"]}
|
|
79
|
+
return NodeWithScore(node=node, score=n.score)
|
|
80
|
+
|
|
81
|
+
def _args(self) -> dict:
|
|
82
|
+
return {"strategy": self.strategy, "labels": self.labels, "min_confidence": self.min_confidence}
|
|
83
|
+
|
|
84
|
+
def _empty(self) -> RestoreContext:
|
|
85
|
+
return RestoreContext(self.strategy, labels=self.labels, min_confidence=self.min_confidence)
|
|
86
|
+
|
|
87
|
+
def redact(self, nodes: List[NodeWithScore], query: Optional[str] = None
|
|
88
|
+
) -> tuple[List[NodeWithScore], Optional[str], RestoreContext]:
|
|
89
|
+
payload = {"q": query or "", "n": [self._entry(n) for n in nodes]}
|
|
90
|
+
if not _has_text(payload):
|
|
91
|
+
return list(nodes), query, self._empty()
|
|
92
|
+
r, ctx = self.client.redact_json(payload, **self._args())
|
|
93
|
+
red = r["redacted"]
|
|
94
|
+
return [self._copy(n, e) for n, e in zip(nodes, red["n"])], (red["q"] if query is not None else None), ctx
|
|
95
|
+
|
|
96
|
+
async def aredact(self, nodes: List[NodeWithScore], query: Optional[str] = None
|
|
97
|
+
) -> tuple[List[NodeWithScore], Optional[str], RestoreContext]:
|
|
98
|
+
payload = {"q": query or "", "n": [self._entry(n) for n in nodes]}
|
|
99
|
+
if not _has_text(payload):
|
|
100
|
+
return list(nodes), query, self._empty()
|
|
101
|
+
r, ctx = await self.client.aredact_json(payload, **self._args())
|
|
102
|
+
red = r["redacted"]
|
|
103
|
+
return [self._copy(n, e) for n, e in zip(nodes, red["n"])], (red["q"] if query is not None else None), ctx
|
|
104
|
+
|
|
105
|
+
def restore(self, text: str, ctx: RestoreContext) -> str:
|
|
106
|
+
if not ctx.reversible or not _has_text(text):
|
|
107
|
+
return text
|
|
108
|
+
return self.client.restore(text, ctx)
|
|
109
|
+
|
|
110
|
+
async def arestore(self, text: str, ctx: RestoreContext) -> str:
|
|
111
|
+
if not ctx.reversible or not _has_text(text):
|
|
112
|
+
return text
|
|
113
|
+
return await self.client.arestore(text, ctx)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class ObscatePostprocessor(BaseNodePostprocessor):
|
|
117
|
+
"""Node postprocessor that redacts node text (and string metadata) before
|
|
118
|
+
synthesis. With the default ``fpe`` strategy, ``restore(answer)`` needs no
|
|
119
|
+
state. With ``tokenize``/``synthetic``, the token map of the most recent
|
|
120
|
+
call is kept in memory per thread, so restore in the same thread before
|
|
121
|
+
the next query.
|
|
122
|
+
|
|
123
|
+
qe = index.as_query_engine(node_postprocessors=[obscate := ObscatePostprocessor()])
|
|
124
|
+
answer = obscate.restore(str(qe.query("...")))
|
|
125
|
+
|
|
126
|
+
Note: a postprocessor cannot change the query text the LLM sees. Use
|
|
127
|
+
``ObscateQueryEngine`` to redact the question as well.
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
strategy: str = Field(default="fpe", description="fpe | tokenize | synthetic | mask | hash | partial")
|
|
131
|
+
labels: Optional[List[str]] = Field(default=None)
|
|
132
|
+
min_confidence: float = Field(default=0.0)
|
|
133
|
+
redact_metadata: bool = Field(default=True)
|
|
134
|
+
_redactor: ObscateNodeRedactor = PrivateAttr()
|
|
135
|
+
_local: threading.local = PrivateAttr(default_factory=threading.local)
|
|
136
|
+
|
|
137
|
+
def __init__(self, client: Optional[ObscateClient] = None, api_key: Optional[str] = None,
|
|
138
|
+
base_url: Optional[str] = None, **kwargs: Any):
|
|
139
|
+
super().__init__(**kwargs)
|
|
140
|
+
self._redactor = ObscateNodeRedactor(self.strategy, self.labels, self.min_confidence,
|
|
141
|
+
self.redact_metadata, api_key, base_url, client)
|
|
142
|
+
|
|
143
|
+
@classmethod
|
|
144
|
+
def class_name(cls) -> str:
|
|
145
|
+
return "ObscatePostprocessor"
|
|
146
|
+
|
|
147
|
+
def _postprocess_nodes(self, nodes: List[NodeWithScore],
|
|
148
|
+
query_bundle: Optional[QueryBundle] = None) -> List[NodeWithScore]:
|
|
149
|
+
red, _, ctx = self._redactor.redact(nodes)
|
|
150
|
+
self._local.ctx = ctx
|
|
151
|
+
return red
|
|
152
|
+
|
|
153
|
+
def _ctx(self) -> RestoreContext:
|
|
154
|
+
ctx = getattr(self._local, "ctx", None)
|
|
155
|
+
if ctx is None:
|
|
156
|
+
if self.strategy != "fpe":
|
|
157
|
+
raise RuntimeError("no redaction to restore yet in this thread")
|
|
158
|
+
ctx = self._redactor._empty()
|
|
159
|
+
return ctx
|
|
160
|
+
|
|
161
|
+
def restore(self, text: Any) -> str:
|
|
162
|
+
"""Restore an answer (str or Response) produced from redacted nodes."""
|
|
163
|
+
return self._redactor.restore(str(text), self._ctx())
|
|
164
|
+
|
|
165
|
+
async def arestore(self, text: Any) -> str:
|
|
166
|
+
return await self._redactor.arestore(str(text), self._ctx())
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
class ObscateQueryEngine(CustomQueryEngine):
|
|
170
|
+
"""Query engine that keeps PII away from the LLM and restores the answer.
|
|
171
|
+
|
|
172
|
+
safe = ObscateQueryEngine.wrap(index.as_query_engine())
|
|
173
|
+
print(safe.query("What did Ravi Kumar ask about?"))
|
|
174
|
+
|
|
175
|
+
Retrieval runs on the original query (locally, against your index). Only
|
|
176
|
+
the synthesis LLM sees redacted data. Streaming synthesizers are read to
|
|
177
|
+
completion before restoring, because a token can be split across chunks.
|
|
178
|
+
"""
|
|
179
|
+
|
|
180
|
+
retriever: BaseRetriever
|
|
181
|
+
response_synthesizer: BaseSynthesizer
|
|
182
|
+
node_postprocessors: List[BaseNodePostprocessor] = Field(default_factory=list)
|
|
183
|
+
strategy: str = "fpe"
|
|
184
|
+
labels: Optional[List[str]] = None
|
|
185
|
+
min_confidence: float = 0.0
|
|
186
|
+
redact_metadata: bool = True
|
|
187
|
+
_redactor: ObscateNodeRedactor = PrivateAttr()
|
|
188
|
+
|
|
189
|
+
def __init__(self, client: Optional[ObscateClient] = None, api_key: Optional[str] = None,
|
|
190
|
+
base_url: Optional[str] = None, **kwargs: Any):
|
|
191
|
+
super().__init__(**kwargs)
|
|
192
|
+
self._redactor = ObscateNodeRedactor(self.strategy, self.labels, self.min_confidence,
|
|
193
|
+
self.redact_metadata, api_key, base_url, client)
|
|
194
|
+
|
|
195
|
+
@classmethod
|
|
196
|
+
def from_args(cls, retriever: BaseRetriever, llm: Any = None, **kwargs: Any) -> "ObscateQueryEngine":
|
|
197
|
+
synth = kwargs.pop("response_synthesizer", None) or get_response_synthesizer(llm=llm)
|
|
198
|
+
return cls(retriever=retriever, response_synthesizer=synth, **kwargs)
|
|
199
|
+
|
|
200
|
+
@classmethod
|
|
201
|
+
def wrap(cls, query_engine: Any, **kwargs: Any) -> "ObscateQueryEngine":
|
|
202
|
+
"""Wrap a ``RetrieverQueryEngine`` (what ``index.as_query_engine()`` returns)."""
|
|
203
|
+
try:
|
|
204
|
+
return cls(retriever=query_engine.retriever,
|
|
205
|
+
response_synthesizer=query_engine._response_synthesizer,
|
|
206
|
+
node_postprocessors=list(getattr(query_engine, "_node_postprocessors", [])),
|
|
207
|
+
**kwargs)
|
|
208
|
+
except AttributeError:
|
|
209
|
+
raise TypeError("wrap() expects a RetrieverQueryEngine; use from_args(retriever, llm) instead")
|
|
210
|
+
|
|
211
|
+
def _retrieve(self, query: str) -> List[NodeWithScore]:
|
|
212
|
+
nodes = self.retriever.retrieve(query)
|
|
213
|
+
for p in self.node_postprocessors:
|
|
214
|
+
nodes = p.postprocess_nodes(nodes, query_bundle=QueryBundle(query))
|
|
215
|
+
return nodes
|
|
216
|
+
|
|
217
|
+
@staticmethod
|
|
218
|
+
def _text(resp: Any) -> str:
|
|
219
|
+
if isinstance(resp, StreamingResponse):
|
|
220
|
+
return "".join(resp.response_gen)
|
|
221
|
+
return str(resp.response if hasattr(resp, "response") else resp)
|
|
222
|
+
|
|
223
|
+
def custom_query(self, query_str: str) -> Response:
|
|
224
|
+
nodes = self._retrieve(query_str)
|
|
225
|
+
red_nodes, red_query, ctx = self._redactor.redact(nodes, query_str)
|
|
226
|
+
resp = self.response_synthesizer.synthesize(red_query, nodes=red_nodes)
|
|
227
|
+
answer = self._redactor.restore(self._text(resp), ctx)
|
|
228
|
+
return Response(response=answer, source_nodes=nodes,
|
|
229
|
+
metadata={"obscate_strategy": self.strategy})
|
|
230
|
+
|
|
231
|
+
async def acustom_query(self, query_str: str) -> Response:
|
|
232
|
+
nodes = await self.retriever.aretrieve(query_str)
|
|
233
|
+
for p in self.node_postprocessors:
|
|
234
|
+
nodes = await p.apostprocess_nodes(nodes, query_bundle=QueryBundle(query_str))
|
|
235
|
+
red_nodes, red_query, ctx = await self._redactor.aredact(nodes, query_str)
|
|
236
|
+
resp = await self.response_synthesizer.asynthesize(red_query, nodes=red_nodes)
|
|
237
|
+
if isinstance(resp, StreamingResponse):
|
|
238
|
+
text = "".join(resp.response_gen)
|
|
239
|
+
else:
|
|
240
|
+
text = str(resp.response)
|
|
241
|
+
answer = await self._redactor.arestore(text, ctx)
|
|
242
|
+
return Response(response=answer, source_nodes=nodes, metadata={"obscate_strategy": self.strategy})
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: llama-index-postprocessor-obscate
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Reversible PII redaction for LlamaIndex: redact retrieved nodes and queries before the LLM, restore the answer (Obscate API)
|
|
5
|
+
Project-URL: Homepage, https://obscate.com
|
|
6
|
+
Project-URL: Documentation, https://obscate.com/docs/llamaindex
|
|
7
|
+
Author: Skycast
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: dpdp,gdpr,llama-index,llamaindex,pii,privacy,rag,redaction
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Requires-Dist: httpx>=0.27
|
|
13
|
+
Requires-Dist: llama-index-core<0.15,>=0.12
|
|
14
|
+
Provides-Extra: examples
|
|
15
|
+
Requires-Dist: anthropic<2,>=0.75; extra == 'examples'
|
|
16
|
+
Requires-Dist: llama-index-llms-anthropic>=0.6; extra == 'examples'
|
|
17
|
+
Provides-Extra: test
|
|
18
|
+
Requires-Dist: pytest>=8; extra == 'test'
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# llama-index-postprocessor-obscate
|
|
22
|
+
|
|
23
|
+
Reversible PII redaction for LlamaIndex RAG. The question and the retrieved
|
|
24
|
+
chunks are redacted before the LLM sees them, and the answer comes back with
|
|
25
|
+
the real values restored. Your index and your citations stay untouched.
|
|
26
|
+
|
|
27
|
+
It is a thin wrapper over the [Obscate](https://obscate.com) HTTP API.
|
|
28
|
+
Detection runs in the API, not in this package. Full docs:
|
|
29
|
+
[obscate.com/docs/llamaindex](https://obscate.com/docs/llamaindex).
|
|
30
|
+
|
|
31
|
+
## Known limitation: partial values come back wrong
|
|
32
|
+
|
|
33
|
+
If the model quotes only **part** of a redacted value ("your card ending in
|
|
34
|
+
7588", "the email starting with…"), restore can't fix it: it matches whole
|
|
35
|
+
tokens, and a fragment isn't one. With `fpe` this fails **silently and
|
|
36
|
+
plausibly**. The fake card looks like a real card, so its last four digits
|
|
37
|
+
look like real digits, and your user sees a confident, wrong statement about
|
|
38
|
+
their own data. Assistants paraphrase values like this all the time, so expect
|
|
39
|
+
it in normal use.
|
|
40
|
+
|
|
41
|
+
What we measured with real Claude (`claude-haiku-5-5`, a support-reply prompt,
|
|
42
|
+
3 runs each; a small test, not a guarantee):
|
|
43
|
+
|
|
44
|
+
| setup | replies stating wrong card digits |
|
|
45
|
+
|---|---|
|
|
46
|
+
| `fpe`, plain prompt | 2 of 3 |
|
|
47
|
+
| `fpe` + the instruction below in the system prompt | 0 of 3 |
|
|
48
|
+
| `tokenize`, plain prompt | 0 of 3 |
|
|
49
|
+
|
|
50
|
+
For user-facing output where the model may paraphrase values:
|
|
51
|
+
|
|
52
|
+
- Add to your system prompt: *"When you mention a card number, email or phone
|
|
53
|
+
number, repeat it exactly and in full as given. Never abbreviate, mask or
|
|
54
|
+
quote part of it."*
|
|
55
|
+
- Or use `strategy="tokenize"`: models treat `[CREDIT_CARD_1]` as a unit and
|
|
56
|
+
don't split it, and restore only touches tokens in its map.
|
|
57
|
+
|
|
58
|
+
Partial references are a boundary of any reversible-redaction approach, not
|
|
59
|
+
only ours: once the model only sees a stand-in, a fragment of the stand-in can
|
|
60
|
+
never map back to a fragment of the original.
|
|
61
|
+
|
|
62
|
+
## Quickstart (under 5 minutes)
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
pip install "llama-index-postprocessor-obscate[examples]"
|
|
66
|
+
export OBSCATE_API_KEY=ob_... # free: 1,000 requests/month at https://obscate.com, no card
|
|
67
|
+
export ANTHROPIC_API_KEY=sk-ant-... # optional: without it the example uses a local stand-in LLM
|
|
68
|
+
python examples/quickstart.py
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Real output (stand-in LLM, Obscate engine; redacted values differ per key):
|
|
72
|
+
|
|
73
|
+
```text
|
|
74
|
+
1) Your question:
|
|
75
|
+
What does Ravi Kumar want, and what email should we reply to?
|
|
76
|
+
2) What the LLM receives:
|
|
77
|
+
Ticket 813: Priya Sharma (mlaem@bskcfsg.org) asked to change her delivery address.
|
|
78
|
+
Ticket 812: [PERSON:wdmvms3g…] (seab.yomog@tziwxuo.com, +91 45671 58378) was charged twice on card 7982 1772 2838 5004 ...
|
|
79
|
+
Query: What does [PERSON:wdmvms3g…] want, and what email should we reply to?
|
|
80
|
+
3) Restored answer for your user:
|
|
81
|
+
(stand-in LLM) From the tickets: Ticket 812: Ravi Kumar (ravi.kumar@example.com, +91 98765 43210) was charged twice ...
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
Look at line 2: **"Priya Sharma" was not caught** in that sentence, while
|
|
85
|
+
"Ravi Kumar" was. That is typical. Emails, phones, cards and Indian IDs are
|
|
86
|
+
near-perfect because they use patterns and checksums. Names and places come
|
|
87
|
+
from an NER model and are best-effort. Test on your own data.
|
|
88
|
+
([Detectors](https://obscate.com/docs/detectors), [benchmark](https://obscate.com/docs/benchmark).)
|
|
89
|
+
|
|
90
|
+
## Use it
|
|
91
|
+
|
|
92
|
+
**One step: wrap your query engine.**
|
|
93
|
+
|
|
94
|
+
```python
|
|
95
|
+
from llama_index.postprocessor.obscate import ObscateQueryEngine
|
|
96
|
+
|
|
97
|
+
qe = ObscateQueryEngine.wrap(index.as_query_engine(llm=llm)) # strategy="fpe" by default
|
|
98
|
+
print(qe.query("What did Ravi Kumar ask about?")) # answer restored
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Or build it from parts with
|
|
102
|
+
`ObscateQueryEngine.from_args(retriever, llm=llm, strategy="tokenize")`.
|
|
103
|
+
Retrieval runs on the original query against your own index. Only the
|
|
104
|
+
synthesis LLM sees redacted text. The query and all chunks go in **one**
|
|
105
|
+
Obscate request, so tokens are consistent: the same person gets the same
|
|
106
|
+
token in the question and in the chunks.
|
|
107
|
+
|
|
108
|
+
**Node postprocessor, for pipelines you assemble yourself:**
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
from llama_index.postprocessor.obscate import ObscatePostprocessor
|
|
112
|
+
|
|
113
|
+
obscate = ObscatePostprocessor() # fpe: restore needs no state
|
|
114
|
+
qe = index.as_query_engine(llm=llm, node_postprocessors=[obscate])
|
|
115
|
+
answer = obscate.restore(qe.query("Summarise ticket 812"))
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
A postprocessor can't change the question text, which still reaches the LLM
|
|
119
|
+
unredacted. Use `ObscateQueryEngine` when questions contain PII. With
|
|
120
|
+
`tokenize`/`synthetic`, the postprocessor keeps the last token map per thread
|
|
121
|
+
in memory, so call `restore` in the same thread before the next query.
|
|
122
|
+
|
|
123
|
+
String metadata the LLM would see (for example `author`) is redacted too.
|
|
124
|
+
Keys listed in `excluded_llm_metadata_keys` are not sent at all. Turn this off
|
|
125
|
+
with `redact_metadata=False`.
|
|
126
|
+
|
|
127
|
+
## Strategies
|
|
128
|
+
|
|
129
|
+
`fpe` (default) keeps formats, for example valid-looking card numbers and
|
|
130
|
+
email-shaped addresses. Values that can't keep their format, such as names,
|
|
131
|
+
become bracketed `[PERSON:…]` tokens. Restoring fpe needs only your key.
|
|
132
|
+
`tokenize` (`[EMAIL_1]`) and `synthetic` (realistic fakes) restore from an
|
|
133
|
+
in-memory token map. `mask`, `hash` and `partial` are one-way, so the answer
|
|
134
|
+
stays redacted.
|
|
135
|
+
|
|
136
|
+
**fpe caveat:** an email, phone or ID the model *invents* (never redacted)
|
|
137
|
+
can't be told apart from an fpe token, and restore turns it into a different
|
|
138
|
+
value. Use `tokenize` if that matters.
|
|
139
|
+
|
|
140
|
+
**Key versions are handled for you.** The query engine restores with the
|
|
141
|
+
`key_version` its redaction returned, so a key rotation on Obscate's side
|
|
142
|
+
never breaks an answer. If you keep fpe output to restore later, keep
|
|
143
|
+
`key_version` with it.
|
|
144
|
+
|
|
145
|
+
## What it does not cover
|
|
146
|
+
|
|
147
|
+
- **Embeddings.** Your embedding model sees raw text when you index and query.
|
|
148
|
+
If it is a remote API, use a local embedding model or redact documents
|
|
149
|
+
before indexing.
|
|
150
|
+
- **Streaming.** Answers are read to completion and then restored, because a
|
|
151
|
+
token can be split across chunks.
|
|
152
|
+
- **LlamaIndex's own DEBUG logs.** `llama_index.core.indices.utils` logs
|
|
153
|
+
retrieved chunk text at DEBUG level, which is upstream of this package.
|
|
154
|
+
Keep the `llama_index` logger at INFO or above in production.
|
|
155
|
+
|
|
156
|
+
## Cost, failure and self-hosting
|
|
157
|
+
|
|
158
|
+
- 1 Obscate request per query (1 unit per 100 KB). Restores are free. The
|
|
159
|
+
free tier covers 1,000 queries a month ([pricing](https://obscate.com/pricing)).
|
|
160
|
+
- **Fails closed.** If Obscate errors, the LLM is not called and
|
|
161
|
+
`ObscateError` is raised.
|
|
162
|
+
- Set `OBSCATE_BASE_URL` to use a self-hosted Obscate. The self-hosted
|
|
163
|
+
edition is in pilot; ask contact@obscate.com.
|
|
164
|
+
|
|
165
|
+
## Privacy
|
|
166
|
+
|
|
167
|
+
This package never logs, prints or writes the text, the detected values or
|
|
168
|
+
the token map. Errors carry the HTTP status and the API's fixed message only.
|
|
169
|
+
This is enforced by `tests/test_llama_index_obscate.py::test_privacy_*`.
|
|
170
|
+
|
|
171
|
+
## Develop
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
pip install -e '.[test]' && pytest -q # unit tests against an in-process fake API (no network)
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
License: MIT.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
llama_index/postprocessor/obscate/__init__.py,sha256=YgVN5oj7gwW_IennOm3oRPGdvKvgwAmuH0szdxZ49FA,365
|
|
2
|
+
llama_index/postprocessor/obscate/_client.py,sha256=0FY2Hx4zR8Fkys9o0AHm0BN6hws_WntIth9ZWZ5UkRE,15870
|
|
3
|
+
llama_index/postprocessor/obscate/base.py,sha256=PhQqR6lyQM8p-4AKD76vFdBFv1xeLa8CLdggeihXA10,11485
|
|
4
|
+
llama_index_postprocessor_obscate-0.1.0.dist-info/METADATA,sha256=PZaq2m9OJSH5Grzg3hB_jNEBIvyBm0femn2r3-d4zFs,7702
|
|
5
|
+
llama_index_postprocessor_obscate-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
6
|
+
llama_index_postprocessor_obscate-0.1.0.dist-info/licenses/LICENSE,sha256=EC6nfL-pBCQXYySuPUJfp1myjTawQe2RAFIQ2Xo6mXU,1064
|
|
7
|
+
llama_index_postprocessor_obscate-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Skycast
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|