latet-client 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- latet_client/__init__.py +58 -0
- latet_client/client.py +456 -0
- latet_client/errors.py +105 -0
- latet_client/langchain.py +128 -0
- latet_client/py.typed +0 -0
- latet_client/results.py +153 -0
- latet_client-0.1.0.dist-info/METADATA +184 -0
- latet_client-0.1.0.dist-info/RECORD +10 -0
- latet_client-0.1.0.dist-info/WHEEL +4 -0
- latet_client-0.1.0.dist-info/licenses/LICENSE +21 -0
latet_client/__init__.py
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""A light Python client for the applications API of a Latet server: ``scan``, ``mask`` and ``unmask`` texts with an API
|
|
2
|
+
key's token, so that an application masks its log lines or its documents before they leave, and restores them later;
|
|
3
|
+
``forget`` takes values out of the key's vault.
|
|
4
|
+
|
|
5
|
+
from latet_client import Latet
|
|
6
|
+
|
|
7
|
+
with Latet("https://latet.example.com", token) as latet:
|
|
8
|
+
masked = latet.mask(["Invoice paid by mario.rossi@acme.it"])
|
|
9
|
+
original = latet.unmask(masked[0].text)
|
|
10
|
+
|
|
11
|
+
``latet_client.langchain`` (``pip install "latet-client[langchain]"``) has a document transformer and two runnables
|
|
12
|
+
for LangChain and LangGraph."""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
17
|
+
|
|
18
|
+
from .client import AsyncLatet, Kind, Latet
|
|
19
|
+
from .errors import (
|
|
20
|
+
Blocked,
|
|
21
|
+
ClearTextWarning,
|
|
22
|
+
ConnectionFailed,
|
|
23
|
+
Forbidden,
|
|
24
|
+
LatetError,
|
|
25
|
+
RateLimited,
|
|
26
|
+
TooLarge,
|
|
27
|
+
Unauthorized,
|
|
28
|
+
Unavailable,
|
|
29
|
+
VaultFull,
|
|
30
|
+
)
|
|
31
|
+
from .results import Finding, ForgetResult, MaskResult, ScanResult, UnmaskResult
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
__version__ = version("latet-client")
|
|
35
|
+
except PackageNotFoundError: # run from a copy of the source that was never installed
|
|
36
|
+
__version__ = "0+unknown"
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"AsyncLatet",
|
|
40
|
+
"Blocked",
|
|
41
|
+
"ClearTextWarning",
|
|
42
|
+
"ConnectionFailed",
|
|
43
|
+
"Finding",
|
|
44
|
+
"Forbidden",
|
|
45
|
+
"ForgetResult",
|
|
46
|
+
"Kind",
|
|
47
|
+
"Latet",
|
|
48
|
+
"LatetError",
|
|
49
|
+
"MaskResult",
|
|
50
|
+
"RateLimited",
|
|
51
|
+
"ScanResult",
|
|
52
|
+
"TooLarge",
|
|
53
|
+
"Unauthorized",
|
|
54
|
+
"Unavailable",
|
|
55
|
+
"UnmaskResult",
|
|
56
|
+
"VaultFull",
|
|
57
|
+
"__version__",
|
|
58
|
+
]
|
latet_client/client.py
ADDED
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
"""The client of a Latet server's applications API: ``scan``, ``mask``, ``unmask`` and ``forget`` with an API key's
|
|
2
|
+
token, in :class:`Latet` and, with ``await``, in :class:`AsyncLatet`.
|
|
3
|
+
|
|
4
|
+
A call takes one text and answers one result, or takes a list of texts and answers a list in the same order. A long
|
|
5
|
+
list goes to the server in several calls, each of at most ``max_texts`` texts and ``max_bytes`` of JSON; a text larger
|
|
6
|
+
than ``max_bytes`` goes alone (the server answers 413 if it is past its own limit). The texts of one call are masked
|
|
7
|
+
together, so a value that a later text of the call teaches the server is masked in the earlier ones too; a value that
|
|
8
|
+
only a later call teaches is not, as with any two calls. ``forget`` takes values, each once, sent in calls bound the
|
|
9
|
+
same way, and answers one result: the counts of every call, summed.
|
|
10
|
+
|
|
11
|
+
The token stays in the client: it is in the ``Authorization`` header of each call and nowhere else, not in a ``repr``,
|
|
12
|
+
an exception's message or a log line (the client says what it does in the ``latet_client`` logger, without a token and
|
|
13
|
+
without a text). A call that does not go through raises :class:`ConnectionFailed`, not ``httpx``'s own error, which
|
|
14
|
+
carries the request and so its header."""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import asyncio
|
|
19
|
+
import ipaddress
|
|
20
|
+
import json
|
|
21
|
+
import logging
|
|
22
|
+
import re
|
|
23
|
+
import time
|
|
24
|
+
import warnings
|
|
25
|
+
from collections.abc import Callable, Iterator, Sequence
|
|
26
|
+
from types import TracebackType
|
|
27
|
+
from typing import Any, Literal, TypeVar, overload
|
|
28
|
+
|
|
29
|
+
import httpx
|
|
30
|
+
|
|
31
|
+
from .errors import BY_STATUS, ClearTextWarning, ConnectionFailed, LatetError, RateLimited
|
|
32
|
+
from .results import (
|
|
33
|
+
ForgetResult,
|
|
34
|
+
MaskResult,
|
|
35
|
+
ScanResult,
|
|
36
|
+
UnmaskResult,
|
|
37
|
+
forget_results,
|
|
38
|
+
mask_results,
|
|
39
|
+
scan_results,
|
|
40
|
+
unmask_results,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
__all__ = ["APPS_API", "AsyncLatet", "Kind", "Latet"]
|
|
44
|
+
|
|
45
|
+
log = logging.getLogger(__name__)
|
|
46
|
+
|
|
47
|
+
APPS_API = "/_latet/apps/v1"
|
|
48
|
+
Kind = Literal["prose", "data"]
|
|
49
|
+
KINDS = ("prose", "data")
|
|
50
|
+
MOST_WAIT = 30.0 # seconds waited at most before one more try, whatever the server asked
|
|
51
|
+
MOST_WORDS = 500 # characters kept of the server's words
|
|
52
|
+
_ADDRESS = "the server's address is a URL such as https://latet.example.com, with no user name or password in it"
|
|
53
|
+
_UNSENDABLE = "the token is empty, or holds a character a header cannot carry (a space, a line break)"
|
|
54
|
+
_SAFE = re.compile(r"[\x21-\x7e]+")
|
|
55
|
+
_SECONDS = re.compile(r"\d+(?:\.\d+)?")
|
|
56
|
+
|
|
57
|
+
T = TypeVar("T")
|
|
58
|
+
Reader = Callable[[Any, int], list[T]]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _sleep(seconds: float) -> None:
|
|
62
|
+
time.sleep(seconds)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
async def _asleep(seconds: float) -> None:
|
|
66
|
+
await asyncio.sleep(seconds)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _address(url: str) -> httpx.URL:
|
|
70
|
+
"""The server's address, as ``httpx`` reads it, or ValueError."""
|
|
71
|
+
if not isinstance(url, str):
|
|
72
|
+
raise TypeError(_ADDRESS)
|
|
73
|
+
try:
|
|
74
|
+
found = httpx.URL(url)
|
|
75
|
+
except httpx.InvalidURL:
|
|
76
|
+
raise ValueError(_ADDRESS) from None # never the address as given: it may hold a password
|
|
77
|
+
if found.scheme not in ("http", "https") or not found.host or found.userinfo:
|
|
78
|
+
raise ValueError(_ADDRESS)
|
|
79
|
+
return found
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _secret(token: str) -> str:
|
|
83
|
+
"""The token without the line break a file ends with, or ValueError (that never quotes it)."""
|
|
84
|
+
if not isinstance(token, str):
|
|
85
|
+
raise TypeError("the token is a str")
|
|
86
|
+
token = token.strip()
|
|
87
|
+
if not _SAFE.fullmatch(token):
|
|
88
|
+
raise ValueError(_UNSENDABLE)
|
|
89
|
+
return token
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _listed(texts: str | Sequence[str], field: str = "texts") -> tuple[bool, list[str]]:
|
|
93
|
+
"""Whether ``texts`` is one text, and the texts as a list; a TypeError naming ``field`` for anything else."""
|
|
94
|
+
wrong = f"{field} is a str, or a sequence of str"
|
|
95
|
+
if isinstance(texts, str):
|
|
96
|
+
return True, [texts]
|
|
97
|
+
try:
|
|
98
|
+
listed = list(texts)
|
|
99
|
+
except TypeError:
|
|
100
|
+
raise TypeError(wrong) from None
|
|
101
|
+
if not all(isinstance(text, str) for text in listed):
|
|
102
|
+
raise TypeError(wrong)
|
|
103
|
+
return False, listed
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _seconds(header: str | None) -> float | None:
|
|
107
|
+
"""The seconds a ``Retry-After`` asks for; ``None`` for anything else than a number of them (a date is not read)."""
|
|
108
|
+
found = (header or "").strip()
|
|
109
|
+
return float(found) if _SECONDS.fullmatch(found) else None
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _written(texts: list[str], what: str = "text") -> list[bytes]:
|
|
113
|
+
"""Every text (``what``: or value) as a JSON string in UTF-8, before any call is made: a text UTF-8 cannot write (a
|
|
114
|
+
lone surrogate, as a log read with ``errors="surrogateescape"`` holds) is a ValueError naming its place and nothing
|
|
115
|
+
of it, and nothing has been sent."""
|
|
116
|
+
parts: list[bytes] = []
|
|
117
|
+
unwritable = None
|
|
118
|
+
for index, text in enumerate(texts):
|
|
119
|
+
try:
|
|
120
|
+
parts.append(json.dumps(text, ensure_ascii=False).encode("utf-8"))
|
|
121
|
+
except UnicodeEncodeError:
|
|
122
|
+
unwritable = index
|
|
123
|
+
break
|
|
124
|
+
if unwritable is not None: # outside the handler: the error, which holds the whole text, is not its context
|
|
125
|
+
raise ValueError(f"{what} {unwritable} cannot be written as UTF-8 (a lone surrogate?): nothing was sent")
|
|
126
|
+
return parts
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _bodies(
|
|
130
|
+
parts: list[bytes], kind: Kind | None, max_texts: int, max_bytes: int, field: str = "texts"
|
|
131
|
+
) -> Iterator[tuple[bytes, int]]:
|
|
132
|
+
"""The bodies of the calls that carry the texts (as ``_written`` made them; a forget's values, ``field``) in order,
|
|
133
|
+
each with how many it holds: at most ``max_texts`` of them and ``max_bytes`` bytes of compact JSON in UTF-8, as it
|
|
134
|
+
is sent. A text that is past ``max_bytes`` alone makes a call of its own."""
|
|
135
|
+
head = b'{"' + field.encode() + b'":['
|
|
136
|
+
tail = b"]}" if kind is None else b'],"kind":' + json.dumps(kind).encode() + b"}"
|
|
137
|
+
frame = len(head) + len(tail)
|
|
138
|
+
batch: list[bytes] = []
|
|
139
|
+
size = frame
|
|
140
|
+
for part in parts:
|
|
141
|
+
if batch and (len(batch) >= max_texts or size + 1 + len(part) > max_bytes):
|
|
142
|
+
yield head + b",".join(batch) + tail, len(batch)
|
|
143
|
+
batch, size = [], frame
|
|
144
|
+
size += len(part) + (1 if batch else 0)
|
|
145
|
+
batch.append(part)
|
|
146
|
+
if batch:
|
|
147
|
+
yield head + b",".join(batch) + tail, len(batch)
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _on_this_computer(host: str) -> bool:
|
|
151
|
+
"""Whether ``host`` is this computer: ``localhost`` or a loopback address (``127.0.0.0/8``, ``::1``)."""
|
|
152
|
+
if host == "localhost" or host.endswith(".localhost"):
|
|
153
|
+
return True
|
|
154
|
+
try:
|
|
155
|
+
return ipaddress.ip_address(host).is_loopback
|
|
156
|
+
except ValueError:
|
|
157
|
+
return False
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
class _Wire:
|
|
161
|
+
"""What the two clients share: the address, the token, the limits, and how an answer is judged."""
|
|
162
|
+
|
|
163
|
+
def __init__(self, url: str, token: str, *, timeout: float, retries: int, max_texts: int, max_bytes: int) -> None:
|
|
164
|
+
if timeout <= 0 or retries < 0 or max_texts < 1 or max_bytes < 1:
|
|
165
|
+
raise ValueError("timeout is above zero, retries is zero or more, max_texts and max_bytes are one or more")
|
|
166
|
+
found = _address(url)
|
|
167
|
+
prefix = found.path.rstrip("/")
|
|
168
|
+
self._url = str(found.copy_with(path=prefix, query=None, fragment=None))
|
|
169
|
+
self._urls = {
|
|
170
|
+
operation: str(found.copy_with(path=f"{prefix}{APPS_API}/{operation}", query=None, fragment=None))
|
|
171
|
+
for operation in ("scan", "mask", "unmask", "forget")
|
|
172
|
+
}
|
|
173
|
+
self._token = _secret(token)
|
|
174
|
+
self._host = found.host
|
|
175
|
+
if found.scheme == "http" and not _on_this_computer(found.host):
|
|
176
|
+
message = f"the token and the texts travel in clear to {found.host} over http://: use an https:// address"
|
|
177
|
+
warnings.warn(message, ClearTextWarning, stacklevel=3) # at the line that made the client
|
|
178
|
+
self._timeout = timeout
|
|
179
|
+
self._retries = retries
|
|
180
|
+
self._max_texts = max_texts
|
|
181
|
+
self._max_bytes = max_bytes
|
|
182
|
+
|
|
183
|
+
def __repr__(self) -> str:
|
|
184
|
+
return f"{type(self).__name__}(url={self._url!r})"
|
|
185
|
+
|
|
186
|
+
def _headers(self) -> dict[str, str]:
|
|
187
|
+
return {
|
|
188
|
+
"authorization": f"Bearer {self._token}",
|
|
189
|
+
"content-type": "application/json",
|
|
190
|
+
"accept": "application/json",
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
def _plan(self, texts: str | Sequence[str], kind: Kind | None) -> tuple[bool, Iterator[tuple[bytes, int]]]:
|
|
194
|
+
"""Whether ``texts`` is one text, and the bodies of the calls that carry it; the arguments are checked, and
|
|
195
|
+
every text written, here, before any call."""
|
|
196
|
+
if kind is not None and kind not in KINDS:
|
|
197
|
+
raise ValueError("kind is prose (text people wrote) or data (logs, records, a program's output)")
|
|
198
|
+
single, listed = _listed(texts)
|
|
199
|
+
return single, _bodies(_written(listed), kind, self._max_texts, self._max_bytes)
|
|
200
|
+
|
|
201
|
+
def _forgetting(self, values: str | Sequence[str]) -> Iterator[tuple[bytes, int]]:
|
|
202
|
+
"""The bodies of the calls that carry ``values`` to forget, each once, in the order given (a value sent again
|
|
203
|
+
would be counted by a later call as not held); checked and written here, before any call."""
|
|
204
|
+
_, listed = _listed(values, "values")
|
|
205
|
+
once = list(dict.fromkeys(listed))
|
|
206
|
+
return _bodies(_written(once, "value"), None, self._max_texts, self._max_bytes, "values")
|
|
207
|
+
|
|
208
|
+
@staticmethod
|
|
209
|
+
def _summed(found: list[ForgetResult]) -> ForgetResult:
|
|
210
|
+
"""The counts of every call of a forget, summed."""
|
|
211
|
+
return ForgetResult(sum(r.forgotten for r in found), sum(r.unknown for r in found))
|
|
212
|
+
|
|
213
|
+
def _unreachable(self, failure: str) -> ConnectionFailed:
|
|
214
|
+
"""The error of a call that did not go through, from the name of ``httpx``'s error alone."""
|
|
215
|
+
return ConnectionFailed(0, f"the call to {self._host} did not go through ({failure})")
|
|
216
|
+
|
|
217
|
+
def _after(self, response: httpx.Response, attempt: int) -> float | None:
|
|
218
|
+
"""``None`` for an answer to read; the seconds to wait before trying again, for a 429 or a 503 with tries left
|
|
219
|
+
(the ``Retry-After`` the server gave, at most ``MOST_WAIT``, else 1, 2, 4... seconds); otherwise the refusal,
|
|
220
|
+
raised."""
|
|
221
|
+
status = response.status_code
|
|
222
|
+
if status == 200:
|
|
223
|
+
return None
|
|
224
|
+
asked = _seconds(response.headers.get("retry-after"))
|
|
225
|
+
if status in (429, 503) and attempt < self._retries:
|
|
226
|
+
wait = min(asked if asked is not None else 2.0**attempt, MOST_WAIT)
|
|
227
|
+
log.debug(
|
|
228
|
+
"the server answered %s: trying again in %.1f s (%s of %s)", status, wait, attempt + 1, self._retries
|
|
229
|
+
)
|
|
230
|
+
return wait
|
|
231
|
+
words = self._words(response)
|
|
232
|
+
if status == 429:
|
|
233
|
+
raise RateLimited(status, words, asked)
|
|
234
|
+
raise BY_STATUS.get(status, LatetError)(status, words)
|
|
235
|
+
|
|
236
|
+
def _words(self, response: httpx.Response) -> str:
|
|
237
|
+
"""The server's words for a refusal (``error``), printable and short, and never the token, whatever put it
|
|
238
|
+
there (a proxy that quotes the request); ``HTTP <status>`` when it gave none."""
|
|
239
|
+
try:
|
|
240
|
+
body = response.json()
|
|
241
|
+
except ValueError:
|
|
242
|
+
body = None
|
|
243
|
+
said = body.get("error") if isinstance(body, dict) else None
|
|
244
|
+
words = said if isinstance(said, str) and said.strip() else f"HTTP {response.status_code}"
|
|
245
|
+
if words == f"HTTP {response.status_code}" and 300 <= response.status_code < 400:
|
|
246
|
+
words += (
|
|
247
|
+
": the server redirected the call, which the client does not follow (use the server's final address)"
|
|
248
|
+
)
|
|
249
|
+
words = words.replace(self._token, "[token]")
|
|
250
|
+
return "".join(char for char in words if char.isprintable())[:MOST_WORDS]
|
|
251
|
+
|
|
252
|
+
@staticmethod
|
|
253
|
+
def _read(response: httpx.Response, count: int, read: Reader[T]) -> list[T]:
|
|
254
|
+
"""The ``count`` results an answer holds, or a ``LatetError`` when it is not the API's."""
|
|
255
|
+
try:
|
|
256
|
+
return read(response.json(), count)
|
|
257
|
+
except ValueError: # not JSON, or Malformed
|
|
258
|
+
words = "the answer is not the applications API's: is this the address of a Latet server?"
|
|
259
|
+
raise LatetError(response.status_code, words) from None
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
class Latet(_Wire):
|
|
263
|
+
"""A Latet server's applications API, for one API key's token.
|
|
264
|
+
|
|
265
|
+
``url`` is the server's address, ``token`` an API key's token (``latet_key_...``) with the permission of each
|
|
266
|
+
operation used. ``timeout`` bounds each step of a call (connecting, sending, waiting for the answer), in seconds:
|
|
267
|
+
90 by default, longer than a server waits on its database (60 s a statement, 75 s an answer), so that a call it
|
|
268
|
+
could not answer in time comes back as a 503 rather than a :class:`ConnectionFailed`; a 429 or a 503 is tried again
|
|
269
|
+
``retries`` times after the ``Retry-After`` the server gave (30 s at most each time) before it is raised, and a call
|
|
270
|
+
that does not go through is a :class:`ConnectionFailed`, not tried again. ``client`` is an ``httpx.Client`` of your
|
|
271
|
+
own (a proxy, a certificate, a transport); the client closes only the one it made. A plain ``http://`` address of
|
|
272
|
+
another computer is a :class:`ClearTextWarning`. It is safe to share between threads."""
|
|
273
|
+
|
|
274
|
+
def __init__(
|
|
275
|
+
self,
|
|
276
|
+
url: str,
|
|
277
|
+
token: str,
|
|
278
|
+
*,
|
|
279
|
+
timeout: float = 90.0,
|
|
280
|
+
retries: int = 3,
|
|
281
|
+
max_texts: int = 1000,
|
|
282
|
+
max_bytes: int = 8 * 1024 * 1024,
|
|
283
|
+
client: httpx.Client | None = None,
|
|
284
|
+
) -> None:
|
|
285
|
+
super().__init__(url, token, timeout=timeout, retries=retries, max_texts=max_texts, max_bytes=max_bytes)
|
|
286
|
+
self._owned = client is None
|
|
287
|
+
self._http = httpx.Client() if client is None else client
|
|
288
|
+
|
|
289
|
+
def __enter__(self) -> Latet:
|
|
290
|
+
return self
|
|
291
|
+
|
|
292
|
+
def __exit__(
|
|
293
|
+
self, exc_type: type[BaseException] | None, exc: BaseException | None, trace: TracebackType | None
|
|
294
|
+
) -> None:
|
|
295
|
+
self.close()
|
|
296
|
+
|
|
297
|
+
def close(self) -> None:
|
|
298
|
+
"""Closes the connections of the client it made; one it was given stays open."""
|
|
299
|
+
if self._owned:
|
|
300
|
+
self._http.close()
|
|
301
|
+
|
|
302
|
+
@overload
|
|
303
|
+
def scan(self, texts: str, *, kind: Kind = ...) -> ScanResult: ... # pyright: ignore[reportOverlappingOverload]
|
|
304
|
+
@overload
|
|
305
|
+
def scan(self, texts: Sequence[str], *, kind: Kind = ...) -> list[ScanResult]: ...
|
|
306
|
+
def scan(self, texts: str | Sequence[str], *, kind: Kind = "prose") -> ScanResult | list[ScanResult]:
|
|
307
|
+
"""Where the server's rules find values in each text; nothing is kept. ``kind`` is ``prose`` (text people
|
|
308
|
+
wrote) or ``data`` (logs, records, a program's output)."""
|
|
309
|
+
return self._run("scan", texts, kind, scan_results)
|
|
310
|
+
|
|
311
|
+
@overload
|
|
312
|
+
def mask(self, texts: str, *, kind: Kind = ...) -> MaskResult: ... # pyright: ignore[reportOverlappingOverload]
|
|
313
|
+
@overload
|
|
314
|
+
def mask(self, texts: Sequence[str], *, kind: Kind = ...) -> list[MaskResult]: ...
|
|
315
|
+
def mask(self, texts: str | Sequence[str], *, kind: Kind = "prose") -> MaskResult | list[MaskResult]:
|
|
316
|
+
"""Each text with the fakes of the API key's vault in place of its values: the same value gets the same fake
|
|
317
|
+
in every call. A text the server stopped has no ``text`` and says ``blocked``."""
|
|
318
|
+
return self._run("mask", texts, kind, mask_results)
|
|
319
|
+
|
|
320
|
+
@overload
|
|
321
|
+
def unmask(self, texts: str) -> UnmaskResult: ... # pyright: ignore[reportOverlappingOverload]
|
|
322
|
+
@overload
|
|
323
|
+
def unmask(self, texts: Sequence[str]) -> list[UnmaskResult]: ...
|
|
324
|
+
def unmask(self, texts: str | Sequence[str]) -> UnmaskResult | list[UnmaskResult]:
|
|
325
|
+
"""Each text with every fake of the API key's vault in it replaced by its value, whichever call made the
|
|
326
|
+
fake. The server writes each unmask in its administration log, and refuses one it cannot write (503)."""
|
|
327
|
+
return self._run("unmask", texts, None, unmask_results)
|
|
328
|
+
|
|
329
|
+
def forget(self, values: str | Sequence[str]) -> ForgetResult:
|
|
330
|
+
"""Take values out of the API key's vault (a token with the ``forget`` permission, which no other brings): each
|
|
331
|
+
value found exactly as written, gone with its fake, which restores nothing from then on, and given a new fake
|
|
332
|
+
if it is met again. One result for all of them: how many the vault held, and how many it did not, each value
|
|
333
|
+
counted once. The server writes each call in its administration log with its counts, never a value, and
|
|
334
|
+
forgets nothing it cannot write there (503). A forget is safe to repeat: a value gone is counted as not held."""
|
|
335
|
+
found: list[ForgetResult] = []
|
|
336
|
+
for body, _ in self._forgetting(values):
|
|
337
|
+
found += self._read(self._post("forget", body), 1, forget_results)
|
|
338
|
+
return self._summed(found)
|
|
339
|
+
|
|
340
|
+
def _run(self, operation: str, texts: str | Sequence[str], kind: Kind | None, read: Reader[T]) -> T | list[T]:
|
|
341
|
+
single, planned = self._plan(texts, kind)
|
|
342
|
+
results: list[T] = []
|
|
343
|
+
for body, count in planned:
|
|
344
|
+
results += self._read(self._post(operation, body), count, read)
|
|
345
|
+
return results[0] if single else results
|
|
346
|
+
|
|
347
|
+
def _post(self, operation: str, body: bytes) -> httpx.Response:
|
|
348
|
+
attempt = 0
|
|
349
|
+
while True:
|
|
350
|
+
response, failure = None, ""
|
|
351
|
+
try:
|
|
352
|
+
response = self._http.post(
|
|
353
|
+
self._urls[operation], content=body, headers=self._headers(), timeout=self._timeout
|
|
354
|
+
)
|
|
355
|
+
except httpx.HTTPError as exc:
|
|
356
|
+
failure = type(exc).__name__ # its name alone: it holds the request, and the request the token
|
|
357
|
+
if response is None: # outside the handler, so the error is neither the cause nor the context
|
|
358
|
+
raise self._unreachable(failure)
|
|
359
|
+
wait = self._after(response, attempt)
|
|
360
|
+
if wait is None:
|
|
361
|
+
return response
|
|
362
|
+
_sleep(wait)
|
|
363
|
+
attempt += 1
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
class AsyncLatet(_Wire):
|
|
367
|
+
""":class:`Latet` with ``await``: the same arguments, results, batches, errors and retries, over an
|
|
368
|
+
``httpx.AsyncClient``. The batches of one call are sent one after the other."""
|
|
369
|
+
|
|
370
|
+
def __init__(
|
|
371
|
+
self,
|
|
372
|
+
url: str,
|
|
373
|
+
token: str,
|
|
374
|
+
*,
|
|
375
|
+
timeout: float = 90.0,
|
|
376
|
+
retries: int = 3,
|
|
377
|
+
max_texts: int = 1000,
|
|
378
|
+
max_bytes: int = 8 * 1024 * 1024,
|
|
379
|
+
client: httpx.AsyncClient | None = None,
|
|
380
|
+
) -> None:
|
|
381
|
+
super().__init__(url, token, timeout=timeout, retries=retries, max_texts=max_texts, max_bytes=max_bytes)
|
|
382
|
+
self._owned = client is None
|
|
383
|
+
self._http = httpx.AsyncClient() if client is None else client
|
|
384
|
+
|
|
385
|
+
async def __aenter__(self) -> AsyncLatet:
|
|
386
|
+
return self
|
|
387
|
+
|
|
388
|
+
async def __aexit__(
|
|
389
|
+
self, exc_type: type[BaseException] | None, exc: BaseException | None, trace: TracebackType | None
|
|
390
|
+
) -> None:
|
|
391
|
+
await self.aclose()
|
|
392
|
+
|
|
393
|
+
async def aclose(self) -> None:
|
|
394
|
+
"""Closes the connections of the client it made; one it was given stays open."""
|
|
395
|
+
if self._owned:
|
|
396
|
+
await self._http.aclose()
|
|
397
|
+
|
|
398
|
+
async def close(self) -> None:
|
|
399
|
+
"""The same as ``aclose``."""
|
|
400
|
+
await self.aclose()
|
|
401
|
+
|
|
402
|
+
@overload
|
|
403
|
+
async def scan(self, texts: str, *, kind: Kind = ...) -> ScanResult: ... # pyright: ignore[reportOverlappingOverload]
|
|
404
|
+
@overload
|
|
405
|
+
async def scan(self, texts: Sequence[str], *, kind: Kind = ...) -> list[ScanResult]: ...
|
|
406
|
+
async def scan(self, texts: str | Sequence[str], *, kind: Kind = "prose") -> ScanResult | list[ScanResult]:
|
|
407
|
+
"""As :meth:`Latet.scan`."""
|
|
408
|
+
return await self._run("scan", texts, kind, scan_results)
|
|
409
|
+
|
|
410
|
+
@overload
|
|
411
|
+
async def mask(self, texts: str, *, kind: Kind = ...) -> MaskResult: ... # pyright: ignore[reportOverlappingOverload]
|
|
412
|
+
@overload
|
|
413
|
+
async def mask(self, texts: Sequence[str], *, kind: Kind = ...) -> list[MaskResult]: ...
|
|
414
|
+
async def mask(self, texts: str | Sequence[str], *, kind: Kind = "prose") -> MaskResult | list[MaskResult]:
|
|
415
|
+
"""As :meth:`Latet.mask`."""
|
|
416
|
+
return await self._run("mask", texts, kind, mask_results)
|
|
417
|
+
|
|
418
|
+
@overload
|
|
419
|
+
async def unmask(self, texts: str) -> UnmaskResult: ... # pyright: ignore[reportOverlappingOverload]
|
|
420
|
+
@overload
|
|
421
|
+
async def unmask(self, texts: Sequence[str]) -> list[UnmaskResult]: ...
|
|
422
|
+
async def unmask(self, texts: str | Sequence[str]) -> UnmaskResult | list[UnmaskResult]:
|
|
423
|
+
"""As :meth:`Latet.unmask`."""
|
|
424
|
+
return await self._run("unmask", texts, None, unmask_results)
|
|
425
|
+
|
|
426
|
+
async def forget(self, values: str | Sequence[str]) -> ForgetResult:
|
|
427
|
+
"""As :meth:`Latet.forget`."""
|
|
428
|
+
found: list[ForgetResult] = []
|
|
429
|
+
for body, _ in self._forgetting(values):
|
|
430
|
+
found += self._read(await self._post("forget", body), 1, forget_results)
|
|
431
|
+
return self._summed(found)
|
|
432
|
+
|
|
433
|
+
async def _run(self, operation: str, texts: str | Sequence[str], kind: Kind | None, read: Reader[T]) -> T | list[T]:
|
|
434
|
+
single, planned = self._plan(texts, kind)
|
|
435
|
+
results: list[T] = []
|
|
436
|
+
for body, count in planned:
|
|
437
|
+
results += self._read(await self._post(operation, body), count, read)
|
|
438
|
+
return results[0] if single else results
|
|
439
|
+
|
|
440
|
+
async def _post(self, operation: str, body: bytes) -> httpx.Response:
|
|
441
|
+
attempt = 0
|
|
442
|
+
while True:
|
|
443
|
+
response, failure = None, ""
|
|
444
|
+
try:
|
|
445
|
+
response = await self._http.post(
|
|
446
|
+
self._urls[operation], content=body, headers=self._headers(), timeout=self._timeout
|
|
447
|
+
)
|
|
448
|
+
except httpx.HTTPError as exc:
|
|
449
|
+
failure = type(exc).__name__ # its name alone: it holds the request, and the request the token
|
|
450
|
+
if response is None: # outside the handler, so the error is neither the cause nor the context
|
|
451
|
+
raise self._unreachable(failure)
|
|
452
|
+
wait = self._after(response, attempt)
|
|
453
|
+
if wait is None:
|
|
454
|
+
return response
|
|
455
|
+
await _asleep(wait)
|
|
456
|
+
attempt += 1
|
latet_client/errors.py
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""What goes wrong in a call, as errors. The server's refusal is a :class:`LatetError` of the HTTP status it answered
|
|
2
|
+
and the words it gave; the statuses a program acts on are classes of their own; a call that does not reach the server is
|
|
3
|
+
a :class:`ConnectionFailed`. The token of a call is in none of them: not in the message, not in the arguments, not in a
|
|
4
|
+
``repr``, and not in anything an error holds (a failed connection is not chained, because ``httpx``'s own error carries
|
|
5
|
+
the request and its ``Authorization`` header)."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
__all__ = [
|
|
10
|
+
"Blocked",
|
|
11
|
+
"ClearTextWarning",
|
|
12
|
+
"ConnectionFailed",
|
|
13
|
+
"Forbidden",
|
|
14
|
+
"LatetError",
|
|
15
|
+
"RateLimited",
|
|
16
|
+
"TooLarge",
|
|
17
|
+
"Unauthorized",
|
|
18
|
+
"Unavailable",
|
|
19
|
+
"VaultFull",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class LatetError(Exception):
|
|
24
|
+
"""The server refused a call, or answered something the applications API does not answer. ``status`` is the HTTP
|
|
25
|
+
status (200 for an answer that was not the API's) and ``message`` the server's words for it, or ``HTTP <status>``
|
|
26
|
+
when it gave none. A 400, 404, 405, 422 or 500 is this class itself; the statuses below are its subclasses."""
|
|
27
|
+
|
|
28
|
+
def __init__(self, status: int, message: str) -> None:
|
|
29
|
+
super().__init__(status, message)
|
|
30
|
+
self.status = status
|
|
31
|
+
self.message = message
|
|
32
|
+
|
|
33
|
+
def __str__(self) -> str:
|
|
34
|
+
return f"{self.status}: {self.message}"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ConnectionFailed(LatetError):
|
|
38
|
+
"""The call did not go through: the server could not be reached, the connection broke or the call timed out. The
|
|
39
|
+
status is 0 and the message names the kind of failure (``ConnectError``, ``ReadTimeout``...) and the server's host.
|
|
40
|
+
It is not tried again by the client, and it is raised without ``__cause__`` or ``__context__``: ``httpx``'s own
|
|
41
|
+
error holds the request, and the request holds the token."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Unauthorized(LatetError):
|
|
45
|
+
"""401: no token, or one the server does not know (revoked, rotated, mistyped)."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class Forbidden(LatetError):
|
|
49
|
+
"""403: the token lacks the permission of the operation (scan, mask or unmask), or it is a person's token, which
|
|
50
|
+
the applications API does not take."""
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class TooLarge(LatetError):
|
|
54
|
+
"""413: the call is larger than the server takes (its ``apps_request_mb``, 8 MiB unless its settings say otherwise,
|
|
55
|
+
never more than its ``max_request_mb``). Send fewer bytes in a call: the client's ``max_bytes``."""
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class RateLimited(LatetError):
|
|
59
|
+
"""429: too many calls, or too many refused tokens, from this address. ``retry_after`` is the seconds the server
|
|
60
|
+
asked to wait, ``None`` when it did not say. The client has waited and tried again ``retries`` times before it
|
|
61
|
+
raises this."""
|
|
62
|
+
|
|
63
|
+
def __init__(self, status: int, message: str, retry_after: float | None = None) -> None:
|
|
64
|
+
super().__init__(status, message)
|
|
65
|
+
self.retry_after = retry_after
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class Unavailable(LatetError):
|
|
69
|
+
"""503: a store of the server does not answer, or its administration log did not take an unmask (nothing was
|
|
70
|
+
restored). The client has waited and tried again ``retries`` times before it raises this."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class VaultFull(LatetError):
|
|
74
|
+
"""507: the API key's vault holds the most values it may, so the call masked nothing and nothing left in clear. An
|
|
75
|
+
administrator raises the limit (``latet server keys limit``)."""
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Blocked(Exception):
|
|
79
|
+
"""A text the server did not answer: its local reviewer stopped it, or its guard against dumps withheld it.
|
|
80
|
+
``index`` is the place of the text in what was sent, ``reason`` the server's words (they name no value)."""
|
|
81
|
+
|
|
82
|
+
def __init__(self, index: int, reason: str) -> None:
|
|
83
|
+
super().__init__(index, reason)
|
|
84
|
+
self.index = index
|
|
85
|
+
self.reason = reason
|
|
86
|
+
|
|
87
|
+
def __str__(self) -> str:
|
|
88
|
+
return f"text {self.index} was blocked: {self.reason}"
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class ClearTextWarning(UserWarning):
|
|
92
|
+
"""The client was made for a plain ``http://`` address of another computer: the token and the texts travel in
|
|
93
|
+
clear. Use an ``https://`` address. An address of this computer (``localhost``, ``127.0.0.1``, ``::1``) is not
|
|
94
|
+
warned of."""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
# The class of each status that has one.
|
|
98
|
+
BY_STATUS: dict[int, type[LatetError]] = {
|
|
99
|
+
401: Unauthorized,
|
|
100
|
+
403: Forbidden,
|
|
101
|
+
413: TooLarge,
|
|
102
|
+
429: RateLimited,
|
|
103
|
+
503: Unavailable,
|
|
104
|
+
507: VaultFull,
|
|
105
|
+
}
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
"""The LangChain extra (``pip install "latet-client[langchain]"``): :class:`LatetMasker`, a document transformer that
|
|
2
|
+
masks documents before they are indexed, and :func:`mask_runnable` and :func:`unmask_runnable`, runnables that mask a
|
|
3
|
+
question and restore an answer, for a chain or a node of a LangGraph graph.
|
|
4
|
+
|
|
5
|
+
masked = LatetMasker(latet).transform_documents(documents) # before indexing them
|
|
6
|
+
# the model sees fakes, the person reads values
|
|
7
|
+
chain = mask_runnable(latet) | model | StrOutputParser() | unmask_runnable(latet)
|
|
8
|
+
|
|
9
|
+
A chat model answers a message (``AIMessage``): ``StrOutputParser`` (``langchain_core.output_parsers``) gives its text,
|
|
10
|
+
and ``unmask_runnable`` takes the message too, and gives its text back restored.
|
|
11
|
+
|
|
12
|
+
Only a document's text is masked: its metadata is kept as it is, and holds ``latet_counts``, how many values of each
|
|
13
|
+
type were masked in the text."""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from collections.abc import Sequence
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
try:
|
|
21
|
+
from langchain_core.documents import BaseDocumentTransformer, Document
|
|
22
|
+
from langchain_core.messages import BaseMessage
|
|
23
|
+
from langchain_core.runnables import Runnable, RunnableLambda
|
|
24
|
+
except ImportError as exc: # pragma: no cover - the message is the point
|
|
25
|
+
raise ImportError('latet_client.langchain needs LangChain\'s core: pip install "latet-client[langchain]"') from exc
|
|
26
|
+
|
|
27
|
+
from .client import AsyncLatet, Kind, Latet
|
|
28
|
+
from .errors import Blocked
|
|
29
|
+
from .results import MaskResult
|
|
30
|
+
|
|
31
|
+
__all__ = ["Blocked", "LatetMasker", "mask_runnable", "unmask_runnable"]
|
|
32
|
+
|
|
33
|
+
COUNTS = "latet_counts" # the metadata a masked document gets
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _masked(results: Sequence[MaskResult]) -> list[str]:
|
|
37
|
+
"""The masked texts; ``Blocked`` for the first the server stopped."""
|
|
38
|
+
texts = []
|
|
39
|
+
for index, result in enumerate(results):
|
|
40
|
+
if result.text is None:
|
|
41
|
+
raise Blocked(index, result.blocked or "the server did not answer this text")
|
|
42
|
+
texts.append(result.text)
|
|
43
|
+
return texts
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class LatetMasker(BaseDocumentTransformer):
|
|
47
|
+
"""Documents masked by a Latet server, for a pipeline that indexes them: the same value gets the same fake in every
|
|
48
|
+
document, today and in a year, and the question that asks about it (``mask_runnable``) the same one.
|
|
49
|
+
|
|
50
|
+
``transform_documents`` returns copies with the masked text and ``metadata["latet_counts"]``, and leaves the
|
|
51
|
+
documents it was given as they are. A document the server stopped raises :class:`Blocked` with its index and
|
|
52
|
+
nothing is returned, so no document of that batch is indexed unmasked.
|
|
53
|
+
|
|
54
|
+
``client`` is a :class:`Latet` for ``transform_documents`` and ``atransform_documents`` (which then runs it in a
|
|
55
|
+
thread), or an :class:`AsyncLatet` for ``atransform_documents`` alone. ``kind`` is ``prose`` for text people wrote,
|
|
56
|
+
``data`` for logs or records."""
|
|
57
|
+
|
|
58
|
+
def __init__(self, client: Latet | AsyncLatet, *, kind: Kind = "prose") -> None:
|
|
59
|
+
self.client = client
|
|
60
|
+
self.kind: Kind = kind
|
|
61
|
+
|
|
62
|
+
def transform_documents(self, documents: Sequence[Document], **kwargs: Any) -> Sequence[Document]:
|
|
63
|
+
if not isinstance(self.client, Latet):
|
|
64
|
+
raise TypeError("this masker has an AsyncLatet: use atransform_documents")
|
|
65
|
+
results = self.client.mask([document.page_content for document in documents], kind=self.kind)
|
|
66
|
+
return self._copies(documents, results)
|
|
67
|
+
|
|
68
|
+
async def atransform_documents(self, documents: Sequence[Document], **kwargs: Any) -> Sequence[Document]:
|
|
69
|
+
if not isinstance(self.client, AsyncLatet):
|
|
70
|
+
return await super().atransform_documents(documents, **kwargs) # the sync client, in a thread
|
|
71
|
+
results = await self.client.mask([document.page_content for document in documents], kind=self.kind)
|
|
72
|
+
return self._copies(documents, results)
|
|
73
|
+
|
|
74
|
+
@staticmethod
|
|
75
|
+
def _copies(documents: Sequence[Document], results: Sequence[MaskResult]) -> list[Document]:
|
|
76
|
+
texts = _masked(results)
|
|
77
|
+
return [
|
|
78
|
+
document.model_copy(
|
|
79
|
+
update={
|
|
80
|
+
"page_content": text,
|
|
81
|
+
"metadata": {**document.metadata, COUNTS: dict(result.counts)},
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
for document, text, result in zip(documents, texts, results, strict=True)
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def mask_runnable(client: Latet | AsyncLatet, *, kind: Kind = "prose") -> Runnable[str, str]:
|
|
89
|
+
"""A runnable from a text to the same text masked, for the question of a RAG chain. A text the server stopped
|
|
90
|
+
raises :class:`Blocked`. With an :class:`AsyncLatet` it can only be awaited (``ainvoke``)."""
|
|
91
|
+
if isinstance(client, AsyncLatet):
|
|
92
|
+
|
|
93
|
+
async def amask(text: str) -> str:
|
|
94
|
+
return _masked([await client.mask(text, kind=kind)])[0]
|
|
95
|
+
|
|
96
|
+
return RunnableLambda(amask, name="latet_mask")
|
|
97
|
+
|
|
98
|
+
def mask(text: str) -> str:
|
|
99
|
+
return _masked([client.mask(text, kind=kind)])[0]
|
|
100
|
+
|
|
101
|
+
return RunnableLambda(mask, name="latet_mask")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _text(answer: str | BaseMessage) -> str:
|
|
105
|
+
"""The text of a model's answer: a text as it is, a chat model's message (``AIMessage``) by its content, which must
|
|
106
|
+
be a text (a message of several parts, an image's, raises TypeError)."""
|
|
107
|
+
if isinstance(answer, BaseMessage):
|
|
108
|
+
if not isinstance(answer.content, str):
|
|
109
|
+
raise TypeError("unmask_runnable takes a message whose content is a text: put StrOutputParser() before it")
|
|
110
|
+
return answer.content
|
|
111
|
+
return answer
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def unmask_runnable(client: Latet | AsyncLatet) -> Runnable[str | BaseMessage, str]:
|
|
115
|
+
"""A runnable from a text to the same text with every fake of the API key's vault replaced by its value, for the
|
|
116
|
+
answer of a model that was shown fakes: a text, or a chat model's message, whose text it gives back restored. With
|
|
117
|
+
an :class:`AsyncLatet` it can only be awaited (``ainvoke``)."""
|
|
118
|
+
if isinstance(client, AsyncLatet):
|
|
119
|
+
|
|
120
|
+
async def aunmask(answer: str | BaseMessage) -> str:
|
|
121
|
+
return (await client.unmask(_text(answer))).text
|
|
122
|
+
|
|
123
|
+
return RunnableLambda(aunmask, name="latet_unmask")
|
|
124
|
+
|
|
125
|
+
def unmask(answer: str | BaseMessage) -> str:
|
|
126
|
+
return client.unmask(_text(answer)).text
|
|
127
|
+
|
|
128
|
+
return RunnableLambda(unmask, name="latet_unmask")
|
latet_client/py.typed
ADDED
|
File without changes
|
latet_client/results.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""What a call answers, one result for each text sent. ``scan`` says where the values are, ``mask`` the text with fakes
|
|
2
|
+
in their place (and where they are in it), ``unmask`` the text with the real values back; ``forget`` how many values it
|
|
3
|
+
took out of the vault, and how many the vault did not hold. Positions are Unicode code points, which is what a Python
|
|
4
|
+
string indexes by."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
__all__ = ["Finding", "ForgetResult", "MaskResult", "ScanResult", "UnmaskResult"]
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass(frozen=True, slots=True)
|
|
15
|
+
class Finding:
|
|
16
|
+
"""A value found in a text, or a fake put in its place: its ``type`` (``EMAIL``, ``IBAN``...), its
|
|
17
|
+
``entity_class`` (``pii``, ``secret``... the JSON's ``class``) and where it is, ``text[start:end]``."""
|
|
18
|
+
|
|
19
|
+
type: str
|
|
20
|
+
entity_class: str
|
|
21
|
+
start: int
|
|
22
|
+
end: int
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass(frozen=True, slots=True)
|
|
26
|
+
class ScanResult:
|
|
27
|
+
"""Where the server's rules find values in a text: nothing is kept, no fake is made. ``counts`` is how many of each
|
|
28
|
+
type."""
|
|
29
|
+
|
|
30
|
+
findings: list[Finding]
|
|
31
|
+
counts: dict[str, int]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True, slots=True)
|
|
35
|
+
class MaskResult:
|
|
36
|
+
"""A text masked: ``text`` has the fakes of the API key's vault in place of its values, ``replacements`` says where
|
|
37
|
+
(in ``text``), ``counts`` how many of each type. When the local reviewer or the server's guard stopped the text,
|
|
38
|
+
``text`` is ``None`` and ``blocked`` says why. ``degraded`` names what ran weaker on the call (a detector that did
|
|
39
|
+
not read a text, a review that did not end), the same on every result of one call; empty when nothing did."""
|
|
40
|
+
|
|
41
|
+
text: str | None
|
|
42
|
+
replacements: list[Finding] = field(default_factory=list)
|
|
43
|
+
counts: dict[str, int] = field(default_factory=dict)
|
|
44
|
+
blocked: str | None = None
|
|
45
|
+
degraded: tuple[str, ...] = ()
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True, slots=True)
|
|
49
|
+
class UnmaskResult:
|
|
50
|
+
"""A text with every fake of the API key's vault in it replaced by its value; ``restored`` is how many."""
|
|
51
|
+
|
|
52
|
+
text: str
|
|
53
|
+
restored: int
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True, slots=True)
|
|
57
|
+
class ForgetResult:
|
|
58
|
+
"""Values taken out of the API key's vault: ``forgotten`` is how many it held (gone, with their fakes), ``unknown``
|
|
59
|
+
how many it did not hold, each value counted once."""
|
|
60
|
+
|
|
61
|
+
forgotten: int
|
|
62
|
+
unknown: int
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Malformed(ValueError):
|
|
66
|
+
"""An answer that is not the applications API's (a proxy's page, another service). The client never shows it: it
|
|
67
|
+
raises a ``LatetError`` in words of its own."""
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _object(value: Any) -> dict[str, Any]:
|
|
71
|
+
if not isinstance(value, dict):
|
|
72
|
+
raise Malformed
|
|
73
|
+
return value
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _text(value: Any) -> str:
|
|
77
|
+
if not isinstance(value, str):
|
|
78
|
+
raise Malformed
|
|
79
|
+
return value
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _number(value: Any) -> int:
|
|
83
|
+
if not isinstance(value, int) or isinstance(value, bool):
|
|
84
|
+
raise Malformed
|
|
85
|
+
return value
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _counts(value: Any) -> dict[str, int]:
|
|
89
|
+
return {_text(name): _number(count) for name, count in _object(value).items()}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _places(value: Any) -> list[Finding]:
|
|
93
|
+
if not isinstance(value, list):
|
|
94
|
+
raise Malformed
|
|
95
|
+
places = []
|
|
96
|
+
for item in value:
|
|
97
|
+
found = _object(item)
|
|
98
|
+
place = (
|
|
99
|
+
_text(found.get("type")),
|
|
100
|
+
_text(found.get("class")),
|
|
101
|
+
_number(found.get("start")),
|
|
102
|
+
_number(found.get("end")),
|
|
103
|
+
)
|
|
104
|
+
places.append(Finding(*place))
|
|
105
|
+
return places
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _items(body: Any, count: int) -> list[dict[str, Any]]:
|
|
109
|
+
items = _object(body).get("results")
|
|
110
|
+
if not isinstance(items, list) or len(items) != count:
|
|
111
|
+
raise Malformed
|
|
112
|
+
return [_object(item) for item in items]
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def scan_results(body: Any, count: int) -> list[ScanResult]:
|
|
116
|
+
"""The ``count`` results of a ``scan`` answer; ``Malformed`` when it is not one."""
|
|
117
|
+
return [ScanResult(_places(item.get("findings")), _counts(item.get("counts"))) for item in _items(body, count)]
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def mask_results(body: Any, count: int) -> list[MaskResult]:
|
|
121
|
+
"""The ``count`` results of a ``mask`` answer, each with the call's ``degraded``; ``Malformed`` if it is not one."""
|
|
122
|
+
items = _items(body, count)
|
|
123
|
+
degraded = _object(body).get("degraded", [])
|
|
124
|
+
if not isinstance(degraded, list):
|
|
125
|
+
raise Malformed
|
|
126
|
+
weaker = tuple(_text(name) for name in degraded)
|
|
127
|
+
results = []
|
|
128
|
+
for item in items:
|
|
129
|
+
if "blocked" in item:
|
|
130
|
+
results.append(MaskResult(None, blocked=_text(item["blocked"]), degraded=weaker))
|
|
131
|
+
else:
|
|
132
|
+
text, places, counts = (
|
|
133
|
+
_text(item.get("text")),
|
|
134
|
+
_places(item.get("replacements")),
|
|
135
|
+
_counts(item.get("counts")),
|
|
136
|
+
)
|
|
137
|
+
results.append(MaskResult(text, places, counts, degraded=weaker))
|
|
138
|
+
return results
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def unmask_results(body: Any, count: int) -> list[UnmaskResult]:
|
|
142
|
+
"""The ``count`` results of an ``unmask`` answer; ``Malformed`` when it is not one."""
|
|
143
|
+
return [UnmaskResult(_text(item.get("text")), _number(item.get("restored"))) for item in _items(body, count)]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def forget_results(body: Any, count: int) -> list[ForgetResult]:
|
|
147
|
+
"""The one result of a ``forget`` answer, whatever how many values it carried (``count``); ``Malformed`` when it
|
|
148
|
+
is not one."""
|
|
149
|
+
found = _object(body)
|
|
150
|
+
forgotten, unknown = _number(found.get("forgotten")), _number(found.get("unknown"))
|
|
151
|
+
if forgotten < 0 or unknown < 0:
|
|
152
|
+
raise Malformed
|
|
153
|
+
return [ForgetResult(forgotten, unknown)]
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: latet-client
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A light Python client for the applications API of a Latet server: scan, mask and unmask texts with an API key's token, with a LangChain extra.
|
|
5
|
+
Project-URL: Homepage, https://latet.ai
|
|
6
|
+
Project-URL: Documentation, https://docs.latet.ai
|
|
7
|
+
Project-URL: Repository, https://github.com/Latet-AI/latet
|
|
8
|
+
Project-URL: Issues, https://github.com/Latet-AI/latet/issues
|
|
9
|
+
Author-email: Emanuele Casabona <siphion@gmail.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: langchain,latet,llm,logs,pii,privacy,pseudonymization,rag,redaction
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Security
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: httpx>=0.27
|
|
26
|
+
Provides-Extra: langchain
|
|
27
|
+
Requires-Dist: langchain-core>=0.3; extra == 'langchain'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# latet-client
|
|
31
|
+
|
|
32
|
+
A light Python client for the applications API of a [Latet](https://latet.ai) server. An application masks its log
|
|
33
|
+
lines or its documents with an API key's token before they leave, and a person or a program with the right token
|
|
34
|
+
restores them later. The package only talks HTTP (`httpx`): it has no detector and no model, and the server does the
|
|
35
|
+
work.
|
|
36
|
+
|
|
37
|
+
It is released on its own, under the MIT license, so that any application can embed it. Latet itself is under its own
|
|
38
|
+
license: see [latet.ai](https://latet.ai).
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install latet-client
|
|
44
|
+
pip install "latet-client[langchain]" # with the LangChain transformer and runnables
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Python 3.10 or newer. You need a Latet server, an API key on it, and a token of that key with the permissions of the
|
|
48
|
+
operations you call: `scan`, `mask`, `unmask` or `forget` (a token with `mask` can `scan` too; `forget` is given on its
|
|
49
|
+
own, and the key's first token does not have it).
|
|
50
|
+
|
|
51
|
+
## Use
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from latet_client import Latet
|
|
55
|
+
|
|
56
|
+
with Latet("https://latet.example.com", token) as latet:
|
|
57
|
+
masked = latet.mask("Invoice 12 was paid by mario.rossi@acme.it")
|
|
58
|
+
print(masked.text) # Invoice 12 was paid by a fake address
|
|
59
|
+
print(masked.counts) # {'EMAIL': 1}
|
|
60
|
+
|
|
61
|
+
lines = latet.mask(log_lines, kind="data") # a list in, a list out, in the same order
|
|
62
|
+
original = latet.unmask(lines[500].text) # needs a token with the unmask permission
|
|
63
|
+
print(original.text, original.restored)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
One text in, one result out; a list in, a list out. `kind` is `"prose"` (text people wrote, the default) or `"data"`
|
|
67
|
+
(logs, records, a program's output).
|
|
68
|
+
|
|
69
|
+
- `scan(texts)` says where the server's rules find values, and keeps nothing: `ScanResult(findings, counts)`, each
|
|
70
|
+
finding `Finding(type, entity_class, start, end)` with positions in the text you sent.
|
|
71
|
+
- `mask(texts)` gives each text with the fakes of the API key's vault in place of its values:
|
|
72
|
+
`MaskResult(text, replacements, counts, blocked, degraded)`, the positions of the replacements in the text it
|
|
73
|
+
answers. The same value gets the same fake in every call, on every instance of the server, for as long as the vault
|
|
74
|
+
keeps it. When the server's local reviewer stops a text, `text` is `None` and `blocked` says why. `degraded` names
|
|
75
|
+
what ran weaker on the call, if anything did.
|
|
76
|
+
- `unmask(texts)` gives each text with every fake of the API key's vault replaced by its value, whichever call made it:
|
|
77
|
+
`UnmaskResult(text, restored)`. The server writes each unmask in its administration log.
|
|
78
|
+
- `forget(values)` takes values out of the API key's vault: `ForgetResult(forgotten, unknown)`, how many the vault
|
|
79
|
+
held and how many it did not (see below).
|
|
80
|
+
|
|
81
|
+
`AsyncLatet` is the same with `await` (`async with AsyncLatet(url, token) as latet`, `await latet.mask(...)`), over
|
|
82
|
+
`httpx.AsyncClient`.
|
|
83
|
+
|
|
84
|
+
### Forgetting values
|
|
85
|
+
|
|
86
|
+
When a person asks that their data leave your logs' vault, a token with the `forget` permission takes their values out
|
|
87
|
+
of it:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
with Latet("https://latet.example.com", eraser_token) as latet:
|
|
91
|
+
found = latet.forget(["mario.rossi@acme.it", "+39 347 123 4567"])
|
|
92
|
+
print(found.forgotten, found.unknown) # how many the vault held, how many it did not
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Each value is found exactly as you write it: another case, other spaces or another written form (`%40` for `@`) is
|
|
96
|
+
another value, which the vault keeps as an entry of its own if it met it. A value forgotten goes with its fake: the
|
|
97
|
+
fake restores nothing from then on, and the value gets a new fake if a text holds it again. An address at a company's
|
|
98
|
+
domain keeps the domain as a value of its own (its fake is the domain of the address's fake): forget it too if it should
|
|
99
|
+
go. A list goes in calls of `max_texts` values at most, each value once, and the counts are summed. The server writes
|
|
100
|
+
each call in its administration log with its counts, never a value, and forgets nothing it cannot write there. A forget
|
|
101
|
+
is safe to repeat: a value already gone is counted as not held. When a call of a long list fails, its error is raised
|
|
102
|
+
and the counts of the calls before it are not returned: what they forgot stays forgotten, and sending the whole list
|
|
103
|
+
again forgets the rest. A key's first forget after the server's upgrade first indexes what the key's local reviewer
|
|
104
|
+
remembered and its history kept before it: on a key with much history it can take minutes, longer than the client's
|
|
105
|
+
`timeout` (90 seconds). Give the client a longer `timeout` for it, or send the list again after a `ConnectionFailed`:
|
|
106
|
+
what the server forgot meanwhile stays forgotten.
|
|
107
|
+
|
|
108
|
+
The answer says whether the vault held each value, so a token with `forget` learns which values the vault knows, as one
|
|
109
|
+
with `scan` or `mask` can: give it only to what needs it.
|
|
110
|
+
|
|
111
|
+
### Long lists
|
|
112
|
+
|
|
113
|
+
A list goes to the server in calls of at most `max_texts` texts (1000) and `max_bytes` bytes of JSON (8 MiB); a text
|
|
114
|
+
larger than `max_bytes` goes alone, and the server answers 413 if it is past its own limit. Every text is checked and
|
|
115
|
+
written before the first call: a text that cannot be written as UTF-8 (a lone surrogate, as a log read with
|
|
116
|
+
`errors="surrogateescape"` can hold) is a `ValueError` naming its place, and nothing is sent.
|
|
117
|
+
|
|
118
|
+
The calls of one list are sent one after the other, in order. The texts of one call are masked together: a value that a
|
|
119
|
+
text of the call teaches the server is masked in the other texts of the call too. A value that only a later call
|
|
120
|
+
teaches is not applied to the texts of the calls before it, which are already masked and answered: if that matters,
|
|
121
|
+
keep the texts that must be masked together in one call (raise `max_texts` and `max_bytes`, up to what the server
|
|
122
|
+
takes).
|
|
123
|
+
|
|
124
|
+
### Errors and retries
|
|
125
|
+
|
|
126
|
+
The server's refusal is a `LatetError(status, message)`, with the server's words as `message`:
|
|
127
|
+
|
|
128
|
+
| Status | Class | Meaning |
|
|
129
|
+
| --- | --- | --- |
|
|
130
|
+
| 401 | `Unauthorized` | no token, or a token the server does not know |
|
|
131
|
+
| 403 | `Forbidden` | the token lacks the permission, or it is a person's token |
|
|
132
|
+
| 413 | `TooLarge` | the call is larger than the server takes |
|
|
133
|
+
| 429 | `RateLimited` | too many calls or refused tokens; `retry_after` is the seconds the server asked |
|
|
134
|
+
| 503 | `Unavailable` | a store of the server does not answer, or its log did not take an unmask or a forget |
|
|
135
|
+
| 507 | `VaultFull` | the key's vault holds the most values it may; nothing was masked |
|
|
136
|
+
| 0 | `ConnectionFailed` | the call did not go through: no connection, a broken one, a timeout |
|
|
137
|
+
|
|
138
|
+
Only a 429 and a 503 are tried again: `retries` times (3), after the `Retry-After` the server gave (seconds; 30 at most
|
|
139
|
+
each time; without it 1, 2, 4 seconds), and they are raised after that. Any other status is raised at once, and so is
|
|
140
|
+
`ConnectionFailed`: a 422 means the texts name more of the vault's values than one call may weigh (send fewer texts in a
|
|
141
|
+
call), and the call masked nothing; a 400, 404, 405 or 500 is a plain `LatetError`. A masking call is safe to repeat (the
|
|
142
|
+
same value gets the same fake), and so are an unmask and a forget, so a program may try again after a `ConnectionFailed`
|
|
143
|
+
if it wants to.
|
|
144
|
+
|
|
145
|
+
`ConnectionFailed` has status 0 and a message that names the kind of failure and the server's host
|
|
146
|
+
(`the call to latet.example.com did not go through (ConnectTimeout)`). It replaces `httpx`'s own error, which carries the
|
|
147
|
+
request and so its `Authorization` header, and it is raised without it as `__cause__` or `__context__`.
|
|
148
|
+
|
|
149
|
+
### The token
|
|
150
|
+
|
|
151
|
+
The token is in the `Authorization` header of each call and nowhere else: not in an exception's message, not in a
|
|
152
|
+
`repr`, not in a log line, and not in anything an error the client raises holds. Use an `https://` address for any
|
|
153
|
+
server that is not on your own computer: a plain `http://` address of another computer makes the client give a
|
|
154
|
+
`ClearTextWarning` (a Python warning), because the token and the texts then travel in clear. Pass your own `httpx.Client`
|
|
155
|
+
(a proxy, a certificate) as `client=`; the client closes only the one it made.
|
|
156
|
+
|
|
157
|
+
## LangChain and LangGraph
|
|
158
|
+
|
|
159
|
+
`latet_client.langchain` masks the documents of a RAG pipeline before they are indexed, masks the question, and
|
|
160
|
+
restores the model's answer:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from langchain_core.runnables import RunnableLambda
|
|
164
|
+
from latet_client import Latet
|
|
165
|
+
from latet_client.langchain import LatetMasker, mask_runnable, unmask_runnable
|
|
166
|
+
|
|
167
|
+
latet = Latet("https://latet.example.com", token)
|
|
168
|
+
|
|
169
|
+
masked = LatetMasker(latet).transform_documents(documents) # index these, not the originals
|
|
170
|
+
# each has metadata["latet_counts"]: how many values of each type were masked; a blocked document raises Blocked
|
|
171
|
+
|
|
172
|
+
chain = mask_runnable(latet) | RunnableLambda(ask_the_model) | unmask_runnable(latet)
|
|
173
|
+
answer = chain.invoke("What did mario.rossi@acme.it pay?") # the model sees fakes, you read the real values
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
The masked documents and the masked question share their fakes, because they come from the same API key's vault, so the
|
|
177
|
+
model can tell that they speak of the same person. `unmask_runnable` takes a chat model's message (`AIMessage`) too,
|
|
178
|
+
and gives its text back restored. `mask_runnable` and `unmask_runnable` are plain runnables: use them as nodes of a
|
|
179
|
+
LangGraph graph as well. With an `AsyncLatet` the transformer has `atransform_documents` and the runnables
|
|
180
|
+
`ainvoke`. Only a document's text is masked, not its metadata.
|
|
181
|
+
|
|
182
|
+
## License
|
|
183
|
+
|
|
184
|
+
MIT, see `LICENSE`.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
latet_client/__init__.py,sha256=MZknrM6DG05aI334fyO0z3FpRyMLBSTP87bAByESZMg,1589
|
|
2
|
+
latet_client/client.py,sha256=Ya57xE-olVBdBofUiexdMNu3zpNtfwq6ahWgUN8iI7U,21733
|
|
3
|
+
latet_client/errors.py,sha256=c8jBZUE4LBA8TN1gCDsxpjLLYlrErJvyRk5XWpSykpY,4335
|
|
4
|
+
latet_client/langchain.py,sha256=OGCuorZ57osj-cELtnp5LwJ5EtK81Vm_cAvAEH9EH8g,6205
|
|
5
|
+
latet_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
latet_client/results.py,sha256=QlpXjgXimj78zQ_MIMDombulLqJ9rAG8HiKhgqiikoE,5493
|
|
7
|
+
latet_client-0.1.0.dist-info/METADATA,sha256=P-w_keH_5xoEdzJwJjan6IIjrnSSD1ut7q7LxIF0sjo,10583
|
|
8
|
+
latet_client-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
9
|
+
latet_client-0.1.0.dist-info/licenses/LICENSE,sha256=OEkim3VRFk2OOsURDoHxw5_droM_kSTz6i5P_xg9IV4,1093
|
|
10
|
+
latet_client-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Emanuele Casabona (https://latet.ai)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|