latet-client 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- latet_client-0.1.0/.gitignore +50 -0
- latet_client-0.1.0/LICENSE +21 -0
- latet_client-0.1.0/PKG-INFO +184 -0
- latet_client-0.1.0/README.md +155 -0
- latet_client-0.1.0/pyproject.toml +61 -0
- latet_client-0.1.0/src/latet_client/__init__.py +58 -0
- latet_client-0.1.0/src/latet_client/client.py +456 -0
- latet_client-0.1.0/src/latet_client/errors.py +105 -0
- latet_client-0.1.0/src/latet_client/langchain.py +128 -0
- latet_client-0.1.0/src/latet_client/py.typed +0 -0
- latet_client-0.1.0/src/latet_client/results.py +153 -0
- latet_client-0.1.0/tests/conftest.py +29 -0
- latet_client-0.1.0/tests/latet_stand_in.py +144 -0
- latet_client-0.1.0/tests/test_async.py +113 -0
- latet_client-0.1.0/tests/test_client.py +412 -0
- latet_client-0.1.0/tests/test_failures.py +138 -0
- latet_client-0.1.0/tests/test_forget.py +103 -0
- latet_client-0.1.0/tests/test_langchain.py +169 -0
- latet_client-0.1.0/tests/test_without_langchain.py +25 -0
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.egg-info/
|
|
5
|
+
.venv/
|
|
6
|
+
build/
|
|
7
|
+
dist/
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.ruff_cache/
|
|
10
|
+
.mypy_cache/
|
|
11
|
+
.hypothesis/
|
|
12
|
+
.coverage
|
|
13
|
+
htmlcov/
|
|
14
|
+
|
|
15
|
+
# Node
|
|
16
|
+
node_modules/
|
|
17
|
+
dashboard/dist/
|
|
18
|
+
docs/dist/
|
|
19
|
+
docs/.astro/
|
|
20
|
+
|
|
21
|
+
# Built dashboard copied into the wheel (CI artifact)
|
|
22
|
+
src/latet/web/*
|
|
23
|
+
!src/latet/web/.gitkeep
|
|
24
|
+
# The documentation Latet serves itself, built into the wheel (`npm run build:bundle` in docs/)
|
|
25
|
+
src/latet/docs/
|
|
26
|
+
|
|
27
|
+
# Claude Code mod type declarations (generated on load)
|
|
28
|
+
**/.claude-plugin/types/
|
|
29
|
+
|
|
30
|
+
# Local data, captures and secrets: never commit
|
|
31
|
+
notes/captures/
|
|
32
|
+
*.sqlite
|
|
33
|
+
*.sqlite-*
|
|
34
|
+
*.db
|
|
35
|
+
.env
|
|
36
|
+
.env.*
|
|
37
|
+
!.env.example
|
|
38
|
+
*.pem
|
|
39
|
+
*.key
|
|
40
|
+
secrets/
|
|
41
|
+
certs/
|
|
42
|
+
|
|
43
|
+
# OS / editors
|
|
44
|
+
.DS_Store
|
|
45
|
+
Thumbs.db
|
|
46
|
+
.idea/
|
|
47
|
+
.vscode/
|
|
48
|
+
|
|
49
|
+
# Auto-generated by Claude Code when loading a mod with --plugin-dir
|
|
50
|
+
notes/spike/mod/tsconfig.json
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Emanuele Casabona (https://latet.ai)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: latet-client
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: A light Python client for the applications API of a Latet server: scan, mask and unmask texts with an API key's token, with a LangChain extra.
|
|
5
|
+
Project-URL: Homepage, https://latet.ai
|
|
6
|
+
Project-URL: Documentation, https://docs.latet.ai
|
|
7
|
+
Project-URL: Repository, https://github.com/Latet-AI/latet
|
|
8
|
+
Project-URL: Issues, https://github.com/Latet-AI/latet/issues
|
|
9
|
+
Author-email: Emanuele Casabona <siphion@gmail.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: langchain,latet,llm,logs,pii,privacy,pseudonymization,rag,redaction
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Security
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
23
|
+
Classifier: Typing :: Typed
|
|
24
|
+
Requires-Python: >=3.10
|
|
25
|
+
Requires-Dist: httpx>=0.27
|
|
26
|
+
Provides-Extra: langchain
|
|
27
|
+
Requires-Dist: langchain-core>=0.3; extra == 'langchain'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# latet-client
|
|
31
|
+
|
|
32
|
+
A light Python client for the applications API of a [Latet](https://latet.ai) server. An application masks its log
|
|
33
|
+
lines or its documents with an API key's token before they leave, and a person or a program with the right token
|
|
34
|
+
restores them later. The package only talks HTTP (`httpx`): it has no detector and no model, and the server does the
|
|
35
|
+
work.
|
|
36
|
+
|
|
37
|
+
It is released on its own, under the MIT license, so that any application can embed it. Latet itself is under its own
|
|
38
|
+
license: see [latet.ai](https://latet.ai).
|
|
39
|
+
|
|
40
|
+
## Install
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
pip install latet-client
|
|
44
|
+
pip install "latet-client[langchain]" # with the LangChain transformer and runnables
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Python 3.10 or newer. You need a Latet server, an API key on it, and a token of that key with the permissions of the
|
|
48
|
+
operations you call: `scan`, `mask`, `unmask` or `forget` (a token with `mask` can `scan` too; `forget` is given on its
|
|
49
|
+
own, and the key's first token does not have it).
|
|
50
|
+
|
|
51
|
+
## Use
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from latet_client import Latet
|
|
55
|
+
|
|
56
|
+
with Latet("https://latet.example.com", token) as latet:
|
|
57
|
+
masked = latet.mask("Invoice 12 was paid by mario.rossi@acme.it")
|
|
58
|
+
print(masked.text) # Invoice 12 was paid by a fake address
|
|
59
|
+
print(masked.counts) # {'EMAIL': 1}
|
|
60
|
+
|
|
61
|
+
lines = latet.mask(log_lines, kind="data") # a list in, a list out, in the same order
|
|
62
|
+
original = latet.unmask(lines[500].text) # needs a token with the unmask permission
|
|
63
|
+
print(original.text, original.restored)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
One text in, one result out; a list in, a list out. `kind` is `"prose"` (text people wrote, the default) or `"data"`
|
|
67
|
+
(logs, records, a program's output).
|
|
68
|
+
|
|
69
|
+
- `scan(texts)` says where the server's rules find values, and keeps nothing: `ScanResult(findings, counts)`, each
|
|
70
|
+
finding `Finding(type, entity_class, start, end)` with positions in the text you sent.
|
|
71
|
+
- `mask(texts)` gives each text with the fakes of the API key's vault in place of its values:
|
|
72
|
+
`MaskResult(text, replacements, counts, blocked, degraded)`, the positions of the replacements in the text it
|
|
73
|
+
answers. The same value gets the same fake in every call, on every instance of the server, for as long as the vault
|
|
74
|
+
keeps it. When the server's local reviewer stops a text, `text` is `None` and `blocked` says why. `degraded` names
|
|
75
|
+
what ran weaker on the call, if anything did.
|
|
76
|
+
- `unmask(texts)` gives each text with every fake of the API key's vault replaced by its value, whichever call made it:
|
|
77
|
+
`UnmaskResult(text, restored)`. The server writes each unmask in its administration log.
|
|
78
|
+
- `forget(values)` takes values out of the API key's vault: `ForgetResult(forgotten, unknown)`, how many the vault
|
|
79
|
+
held and how many it did not (see below).
|
|
80
|
+
|
|
81
|
+
`AsyncLatet` is the same with `await` (`async with AsyncLatet(url, token) as latet`, `await latet.mask(...)`), over
|
|
82
|
+
`httpx.AsyncClient`.
|
|
83
|
+
|
|
84
|
+
### Forgetting values
|
|
85
|
+
|
|
86
|
+
When a person asks that their data leave your logs' vault, a token with the `forget` permission takes their values out
|
|
87
|
+
of it:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
with Latet("https://latet.example.com", eraser_token) as latet:
|
|
91
|
+
found = latet.forget(["mario.rossi@acme.it", "+39 347 123 4567"])
|
|
92
|
+
print(found.forgotten, found.unknown) # how many the vault held, how many it did not
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Each value is found exactly as you write it: another case, other spaces or another written form (`%40` for `@`) is
|
|
96
|
+
another value, which the vault keeps as an entry of its own if it met it. A value forgotten goes with its fake: the
|
|
97
|
+
fake restores nothing from then on, and the value gets a new fake if a text holds it again. An address at a company's
|
|
98
|
+
domain keeps the domain as a value of its own (its fake is the domain of the address's fake): forget it too if it should
|
|
99
|
+
go. A list goes in calls of `max_texts` values at most, each value once, and the counts are summed. The server writes
|
|
100
|
+
each call in its administration log with its counts, never a value, and forgets nothing it cannot write there. A forget
|
|
101
|
+
is safe to repeat: a value already gone is counted as not held. When a call of a long list fails, its error is raised
|
|
102
|
+
and the counts of the calls before it are not returned: what they forgot stays forgotten, and sending the whole list
|
|
103
|
+
again forgets the rest. A key's first forget after the server's upgrade first indexes what the key's local reviewer
|
|
104
|
+
remembered and its history kept before it: on a key with much history it can take minutes, longer than the client's
|
|
105
|
+
`timeout` (90 seconds). Give the client a longer `timeout` for it, or send the list again after a `ConnectionFailed`:
|
|
106
|
+
what the server forgot meanwhile stays forgotten.
|
|
107
|
+
|
|
108
|
+
The answer says whether the vault held each value, so a token with `forget` learns which values the vault knows, as one
|
|
109
|
+
with `scan` or `mask` can: give it only to what needs it.
|
|
110
|
+
|
|
111
|
+
### Long lists
|
|
112
|
+
|
|
113
|
+
A list goes to the server in calls of at most `max_texts` texts (1000) and `max_bytes` bytes of JSON (8 MiB); a text
|
|
114
|
+
larger than `max_bytes` goes alone, and the server answers 413 if it is past its own limit. Every text is checked and
|
|
115
|
+
written before the first call: a text that cannot be written as UTF-8 (a lone surrogate, as a log read with
|
|
116
|
+
`errors="surrogateescape"` can hold) is a `ValueError` naming its place, and nothing is sent.
|
|
117
|
+
|
|
118
|
+
The calls of one list are sent one after the other, in order. The texts of one call are masked together: a value that a
|
|
119
|
+
text of the call teaches the server is masked in the other texts of the call too. A value that only a later call
|
|
120
|
+
teaches is not applied to the texts of the calls before it, which are already masked and answered: if that matters,
|
|
121
|
+
keep the texts that must be masked together in one call (raise `max_texts` and `max_bytes`, up to what the server
|
|
122
|
+
takes).
|
|
123
|
+
|
|
124
|
+
### Errors and retries
|
|
125
|
+
|
|
126
|
+
The server's refusal is a `LatetError(status, message)`, with the server's words as `message`:
|
|
127
|
+
|
|
128
|
+
| Status | Class | Meaning |
|
|
129
|
+
| --- | --- | --- |
|
|
130
|
+
| 401 | `Unauthorized` | no token, or a token the server does not know |
|
|
131
|
+
| 403 | `Forbidden` | the token lacks the permission, or it is a person's token |
|
|
132
|
+
| 413 | `TooLarge` | the call is larger than the server takes |
|
|
133
|
+
| 429 | `RateLimited` | too many calls or refused tokens; `retry_after` is the seconds the server asked |
|
|
134
|
+
| 503 | `Unavailable` | a store of the server does not answer, or its log did not take an unmask or a forget |
|
|
135
|
+
| 507 | `VaultFull` | the key's vault holds the most values it may; nothing was masked |
|
|
136
|
+
| 0 | `ConnectionFailed` | the call did not go through: no connection, a broken one, a timeout |
|
|
137
|
+
|
|
138
|
+
Only a 429 and a 503 are tried again: `retries` times (3), after the `Retry-After` the server gave (seconds; 30 at most
|
|
139
|
+
each time; without it 1, 2, 4 seconds), and they are raised after that. Any other status is raised at once, and so is
|
|
140
|
+
`ConnectionFailed`: a 422 means the texts name more of the vault's values than one call may weigh (send fewer texts in a
|
|
141
|
+
call), and the call masked nothing; a 400, 404, 405 or 500 is a plain `LatetError`. A masking call is safe to repeat (the
|
|
142
|
+
same value gets the same fake), and so are an unmask and a forget, so a program may try again after a `ConnectionFailed`
|
|
143
|
+
if it wants to.
|
|
144
|
+
|
|
145
|
+
`ConnectionFailed` has status 0 and a message that names the kind of failure and the server's host
|
|
146
|
+
(`the call to latet.example.com did not go through (ConnectTimeout)`). It replaces `httpx`'s own error, which carries the
|
|
147
|
+
request and so its `Authorization` header, and it is raised without it as `__cause__` or `__context__`.
|
|
148
|
+
|
|
149
|
+
### The token
|
|
150
|
+
|
|
151
|
+
The token is in the `Authorization` header of each call and nowhere else: not in an exception's message, not in a
|
|
152
|
+
`repr`, not in a log line, and not in anything an error the client raises holds. Use an `https://` address for any
|
|
153
|
+
server that is not on your own computer: a plain `http://` address of another computer makes the client give a
|
|
154
|
+
`ClearTextWarning` (a Python warning), because the token and the texts then travel in clear. Pass your own `httpx.Client`
|
|
155
|
+
(a proxy, a certificate) as `client=`; the client closes only the one it made.
|
|
156
|
+
|
|
157
|
+
## LangChain and LangGraph
|
|
158
|
+
|
|
159
|
+
`latet_client.langchain` masks the documents of a RAG pipeline before they are indexed, masks the question, and
|
|
160
|
+
restores the model's answer:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from langchain_core.runnables import RunnableLambda
|
|
164
|
+
from latet_client import Latet
|
|
165
|
+
from latet_client.langchain import LatetMasker, mask_runnable, unmask_runnable
|
|
166
|
+
|
|
167
|
+
latet = Latet("https://latet.example.com", token)
|
|
168
|
+
|
|
169
|
+
masked = LatetMasker(latet).transform_documents(documents) # index these, not the originals
|
|
170
|
+
# each has metadata["latet_counts"]: how many values of each type were masked; a blocked document raises Blocked
|
|
171
|
+
|
|
172
|
+
chain = mask_runnable(latet) | RunnableLambda(ask_the_model) | unmask_runnable(latet)
|
|
173
|
+
answer = chain.invoke("What did mario.rossi@acme.it pay?") # the model sees fakes, you read the real values
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
The masked documents and the masked question share their fakes, because they come from the same API key's vault, so the
|
|
177
|
+
model can tell that they speak of the same person. `unmask_runnable` takes a chat model's message (`AIMessage`) too,
|
|
178
|
+
and gives its text back restored. `mask_runnable` and `unmask_runnable` are plain runnables: use them as nodes of a
|
|
179
|
+
LangGraph graph as well. With an `AsyncLatet` the transformer has `atransform_documents` and the runnables
|
|
180
|
+
`ainvoke`. Only a document's text is masked, not its metadata.
|
|
181
|
+
|
|
182
|
+
## License
|
|
183
|
+
|
|
184
|
+
MIT, see `LICENSE`.
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# latet-client
|
|
2
|
+
|
|
3
|
+
A light Python client for the applications API of a [Latet](https://latet.ai) server. An application masks its log
|
|
4
|
+
lines or its documents with an API key's token before they leave, and a person or a program with the right token
|
|
5
|
+
restores them later. The package only talks HTTP (`httpx`): it has no detector and no model, and the server does the
|
|
6
|
+
work.
|
|
7
|
+
|
|
8
|
+
It is released on its own, under the MIT license, so that any application can embed it. Latet itself is under its own
|
|
9
|
+
license: see [latet.ai](https://latet.ai).
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pip install latet-client
|
|
15
|
+
pip install "latet-client[langchain]" # with the LangChain transformer and runnables
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Python 3.10 or newer. You need a Latet server, an API key on it, and a token of that key with the permissions of the
|
|
19
|
+
operations you call: `scan`, `mask`, `unmask` or `forget` (a token with `mask` can `scan` too; `forget` is given on its
|
|
20
|
+
own, and the key's first token does not have it).
|
|
21
|
+
|
|
22
|
+
## Use
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from latet_client import Latet
|
|
26
|
+
|
|
27
|
+
with Latet("https://latet.example.com", token) as latet:
|
|
28
|
+
masked = latet.mask("Invoice 12 was paid by mario.rossi@acme.it")
|
|
29
|
+
print(masked.text) # Invoice 12 was paid by a fake address
|
|
30
|
+
print(masked.counts) # {'EMAIL': 1}
|
|
31
|
+
|
|
32
|
+
lines = latet.mask(log_lines, kind="data") # a list in, a list out, in the same order
|
|
33
|
+
original = latet.unmask(lines[500].text) # needs a token with the unmask permission
|
|
34
|
+
print(original.text, original.restored)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
One text in, one result out; a list in, a list out. `kind` is `"prose"` (text people wrote, the default) or `"data"`
|
|
38
|
+
(logs, records, a program's output).
|
|
39
|
+
|
|
40
|
+
- `scan(texts)` says where the server's rules find values, and keeps nothing: `ScanResult(findings, counts)`, each
|
|
41
|
+
finding `Finding(type, entity_class, start, end)` with positions in the text you sent.
|
|
42
|
+
- `mask(texts)` gives each text with the fakes of the API key's vault in place of its values:
|
|
43
|
+
`MaskResult(text, replacements, counts, blocked, degraded)`, the positions of the replacements in the text it
|
|
44
|
+
answers. The same value gets the same fake in every call, on every instance of the server, for as long as the vault
|
|
45
|
+
keeps it. When the server's local reviewer stops a text, `text` is `None` and `blocked` says why. `degraded` names
|
|
46
|
+
what ran weaker on the call, if anything did.
|
|
47
|
+
- `unmask(texts)` gives each text with every fake of the API key's vault replaced by its value, whichever call made it:
|
|
48
|
+
`UnmaskResult(text, restored)`. The server writes each unmask in its administration log.
|
|
49
|
+
- `forget(values)` takes values out of the API key's vault: `ForgetResult(forgotten, unknown)`, how many the vault
|
|
50
|
+
held and how many it did not (see below).
|
|
51
|
+
|
|
52
|
+
`AsyncLatet` is the same with `await` (`async with AsyncLatet(url, token) as latet`, `await latet.mask(...)`), over
|
|
53
|
+
`httpx.AsyncClient`.
|
|
54
|
+
|
|
55
|
+
### Forgetting values
|
|
56
|
+
|
|
57
|
+
When a person asks that their data leave your logs' vault, a token with the `forget` permission takes their values out
|
|
58
|
+
of it:
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
with Latet("https://latet.example.com", eraser_token) as latet:
|
|
62
|
+
found = latet.forget(["mario.rossi@acme.it", "+39 347 123 4567"])
|
|
63
|
+
print(found.forgotten, found.unknown) # how many the vault held, how many it did not
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Each value is found exactly as you write it: another case, other spaces or another written form (`%40` for `@`) is
|
|
67
|
+
another value, which the vault keeps as an entry of its own if it met it. A value forgotten goes with its fake: the
|
|
68
|
+
fake restores nothing from then on, and the value gets a new fake if a text holds it again. An address at a company's
|
|
69
|
+
domain keeps the domain as a value of its own (its fake is the domain of the address's fake): forget it too if it should
|
|
70
|
+
go. A list goes in calls of `max_texts` values at most, each value once, and the counts are summed. The server writes
|
|
71
|
+
each call in its administration log with its counts, never a value, and forgets nothing it cannot write there. A forget
|
|
72
|
+
is safe to repeat: a value already gone is counted as not held. When a call of a long list fails, its error is raised
|
|
73
|
+
and the counts of the calls before it are not returned: what they forgot stays forgotten, and sending the whole list
|
|
74
|
+
again forgets the rest. A key's first forget after the server's upgrade first indexes what the key's local reviewer
|
|
75
|
+
remembered and its history kept before it: on a key with much history it can take minutes, longer than the client's
|
|
76
|
+
`timeout` (90 seconds). Give the client a longer `timeout` for it, or send the list again after a `ConnectionFailed`:
|
|
77
|
+
what the server forgot meanwhile stays forgotten.
|
|
78
|
+
|
|
79
|
+
The answer says whether the vault held each value, so a token with `forget` learns which values the vault knows, as one
|
|
80
|
+
with `scan` or `mask` can: give it only to what needs it.
|
|
81
|
+
|
|
82
|
+
### Long lists
|
|
83
|
+
|
|
84
|
+
A list goes to the server in calls of at most `max_texts` texts (1000) and `max_bytes` bytes of JSON (8 MiB); a text
|
|
85
|
+
larger than `max_bytes` goes alone, and the server answers 413 if it is past its own limit. Every text is checked and
|
|
86
|
+
written before the first call: a text that cannot be written as UTF-8 (a lone surrogate, as a log read with
|
|
87
|
+
`errors="surrogateescape"` can hold) is a `ValueError` naming its place, and nothing is sent.
|
|
88
|
+
|
|
89
|
+
The calls of one list are sent one after the other, in order. The texts of one call are masked together: a value that a
|
|
90
|
+
text of the call teaches the server is masked in the other texts of the call too. A value that only a later call
|
|
91
|
+
teaches is not applied to the texts of the calls before it, which are already masked and answered: if that matters,
|
|
92
|
+
keep the texts that must be masked together in one call (raise `max_texts` and `max_bytes`, up to what the server
|
|
93
|
+
takes).
|
|
94
|
+
|
|
95
|
+
### Errors and retries
|
|
96
|
+
|
|
97
|
+
The server's refusal is a `LatetError(status, message)`, with the server's words as `message`:
|
|
98
|
+
|
|
99
|
+
| Status | Class | Meaning |
|
|
100
|
+
| --- | --- | --- |
|
|
101
|
+
| 401 | `Unauthorized` | no token, or a token the server does not know |
|
|
102
|
+
| 403 | `Forbidden` | the token lacks the permission, or it is a person's token |
|
|
103
|
+
| 413 | `TooLarge` | the call is larger than the server takes |
|
|
104
|
+
| 429 | `RateLimited` | too many calls or refused tokens; `retry_after` is the seconds the server asked |
|
|
105
|
+
| 503 | `Unavailable` | a store of the server does not answer, or its log did not take an unmask or a forget |
|
|
106
|
+
| 507 | `VaultFull` | the key's vault holds the most values it may; nothing was masked |
|
|
107
|
+
| 0 | `ConnectionFailed` | the call did not go through: no connection, a broken one, a timeout |
|
|
108
|
+
|
|
109
|
+
Only a 429 and a 503 are tried again: `retries` times (3), after the `Retry-After` the server gave (seconds; 30 at most
|
|
110
|
+
each time; without it 1, 2, 4 seconds), and they are raised after that. Any other status is raised at once, and so is
|
|
111
|
+
`ConnectionFailed`: a 422 means the texts name more of the vault's values than one call may weigh (send fewer texts in a
|
|
112
|
+
call), and the call masked nothing; a 400, 404, 405 or 500 is a plain `LatetError`. A masking call is safe to repeat (the
|
|
113
|
+
same value gets the same fake), and so are an unmask and a forget, so a program may try again after a `ConnectionFailed`
|
|
114
|
+
if it wants to.
|
|
115
|
+
|
|
116
|
+
`ConnectionFailed` has status 0 and a message that names the kind of failure and the server's host
|
|
117
|
+
(`the call to latet.example.com did not go through (ConnectTimeout)`). It replaces `httpx`'s own error, which carries the
|
|
118
|
+
request and so its `Authorization` header, and it is raised without it as `__cause__` or `__context__`.
|
|
119
|
+
|
|
120
|
+
### The token
|
|
121
|
+
|
|
122
|
+
The token is in the `Authorization` header of each call and nowhere else: not in an exception's message, not in a
|
|
123
|
+
`repr`, not in a log line, and not in anything an error the client raises holds. Use an `https://` address for any
|
|
124
|
+
server that is not on your own computer: a plain `http://` address of another computer makes the client give a
|
|
125
|
+
`ClearTextWarning` (a Python warning), because the token and the texts then travel in clear. Pass your own `httpx.Client`
|
|
126
|
+
(a proxy, a certificate) as `client=`; the client closes only the one it made.
|
|
127
|
+
|
|
128
|
+
## LangChain and LangGraph
|
|
129
|
+
|
|
130
|
+
`latet_client.langchain` masks the documents of a RAG pipeline before they are indexed, masks the question, and
|
|
131
|
+
restores the model's answer:
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
from langchain_core.runnables import RunnableLambda
|
|
135
|
+
from latet_client import Latet
|
|
136
|
+
from latet_client.langchain import LatetMasker, mask_runnable, unmask_runnable
|
|
137
|
+
|
|
138
|
+
latet = Latet("https://latet.example.com", token)
|
|
139
|
+
|
|
140
|
+
masked = LatetMasker(latet).transform_documents(documents) # index these, not the originals
|
|
141
|
+
# each has metadata["latet_counts"]: how many values of each type were masked; a blocked document raises Blocked
|
|
142
|
+
|
|
143
|
+
chain = mask_runnable(latet) | RunnableLambda(ask_the_model) | unmask_runnable(latet)
|
|
144
|
+
answer = chain.invoke("What did mario.rossi@acme.it pay?") # the model sees fakes, you read the real values
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
The masked documents and the masked question share their fakes, because they come from the same API key's vault, so the
|
|
148
|
+
model can tell that they speak of the same person. `unmask_runnable` takes a chat model's message (`AIMessage`) too,
|
|
149
|
+
and gives its text back restored. `mask_runnable` and `unmask_runnable` are plain runnables: use them as nodes of a
|
|
150
|
+
LangGraph graph as well. With an `AsyncLatet` the transformer has `atransform_documents` and the runnables
|
|
151
|
+
`ainvoke`. Only a document's text is masked, not its metadata.
|
|
152
|
+
|
|
153
|
+
## License
|
|
154
|
+
|
|
155
|
+
MIT, see `LICENSE`.
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "latet-client"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "A light Python client for the applications API of a Latet server: scan, mask and unmask texts with an API key's token, with a LangChain extra."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Emanuele Casabona", email = "siphion@gmail.com" }]
|
|
14
|
+
keywords = ["privacy", "llm", "pii", "redaction", "pseudonymization", "latet", "langchain", "logs", "rag"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Programming Language :: Python :: 3.10",
|
|
21
|
+
"Programming Language :: Python :: 3.11",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Topic :: Security",
|
|
25
|
+
"Topic :: Software Development :: Libraries",
|
|
26
|
+
"Typing :: Typed",
|
|
27
|
+
]
|
|
28
|
+
dependencies = ["httpx>=0.27"]
|
|
29
|
+
|
|
30
|
+
[project.optional-dependencies]
|
|
31
|
+
# `LatetMasker` and the two runnables in `latet_client.langchain`.
|
|
32
|
+
langchain = ["langchain-core>=0.3"]
|
|
33
|
+
|
|
34
|
+
[project.urls]
|
|
35
|
+
Homepage = "https://latet.ai"
|
|
36
|
+
Documentation = "https://docs.latet.ai"
|
|
37
|
+
Repository = "https://github.com/Latet-AI/latet"
|
|
38
|
+
Issues = "https://github.com/Latet-AI/latet/issues"
|
|
39
|
+
|
|
40
|
+
[tool.hatch.build.targets.wheel]
|
|
41
|
+
packages = ["src/latet_client"]
|
|
42
|
+
|
|
43
|
+
[tool.hatch.build.targets.sdist]
|
|
44
|
+
include = ["src", "tests", "README.md", "LICENSE", "pyproject.toml"]
|
|
45
|
+
|
|
46
|
+
# The repository's rules, for the lowest Python the package supports.
|
|
47
|
+
[tool.ruff]
|
|
48
|
+
extend = "../../pyproject.toml"
|
|
49
|
+
target-version = "py310"
|
|
50
|
+
src = ["src", "tests"]
|
|
51
|
+
|
|
52
|
+
[tool.ruff.lint.per-file-ignores]
|
|
53
|
+
"tests/**" = ["S", "B011", "PT018", "E501"]
|
|
54
|
+
|
|
55
|
+
# The package's own tests, for a run inside this folder (`pytest` from `sdk/python`): the repository's run takes the
|
|
56
|
+
# same options from its root.
|
|
57
|
+
[tool.pytest.ini_options]
|
|
58
|
+
testpaths = ["tests"]
|
|
59
|
+
asyncio_mode = "auto"
|
|
60
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
61
|
+
addopts = "-ra --strict-markers --disable-socket --allow-hosts=127.0.0.1,::1"
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""A light Python client for the applications API of a Latet server: ``scan``, ``mask`` and ``unmask`` texts with an API
|
|
2
|
+
key's token, so that an application masks its log lines or its documents before they leave, and restores them later;
|
|
3
|
+
``forget`` takes values out of the key's vault.
|
|
4
|
+
|
|
5
|
+
from latet_client import Latet
|
|
6
|
+
|
|
7
|
+
with Latet("https://latet.example.com", token) as latet:
|
|
8
|
+
masked = latet.mask(["Invoice paid by mario.rossi@acme.it"])
|
|
9
|
+
original = latet.unmask(masked[0].text)
|
|
10
|
+
|
|
11
|
+
``latet_client.langchain`` (``pip install "latet-client[langchain]"``) has a document transformer and two runnables
|
|
12
|
+
for LangChain and LangGraph."""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
17
|
+
|
|
18
|
+
from .client import AsyncLatet, Kind, Latet
|
|
19
|
+
from .errors import (
|
|
20
|
+
Blocked,
|
|
21
|
+
ClearTextWarning,
|
|
22
|
+
ConnectionFailed,
|
|
23
|
+
Forbidden,
|
|
24
|
+
LatetError,
|
|
25
|
+
RateLimited,
|
|
26
|
+
TooLarge,
|
|
27
|
+
Unauthorized,
|
|
28
|
+
Unavailable,
|
|
29
|
+
VaultFull,
|
|
30
|
+
)
|
|
31
|
+
from .results import Finding, ForgetResult, MaskResult, ScanResult, UnmaskResult
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
__version__ = version("latet-client")
|
|
35
|
+
except PackageNotFoundError: # run from a copy of the source that was never installed
|
|
36
|
+
__version__ = "0+unknown"
|
|
37
|
+
|
|
38
|
+
__all__ = [
|
|
39
|
+
"AsyncLatet",
|
|
40
|
+
"Blocked",
|
|
41
|
+
"ClearTextWarning",
|
|
42
|
+
"ConnectionFailed",
|
|
43
|
+
"Finding",
|
|
44
|
+
"Forbidden",
|
|
45
|
+
"ForgetResult",
|
|
46
|
+
"Kind",
|
|
47
|
+
"Latet",
|
|
48
|
+
"LatetError",
|
|
49
|
+
"MaskResult",
|
|
50
|
+
"RateLimited",
|
|
51
|
+
"ScanResult",
|
|
52
|
+
"TooLarge",
|
|
53
|
+
"Unauthorized",
|
|
54
|
+
"Unavailable",
|
|
55
|
+
"UnmaskResult",
|
|
56
|
+
"VaultFull",
|
|
57
|
+
"__version__",
|
|
58
|
+
]
|