exegete 0.14.1a0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- exegete/__init__.py +16 -0
- exegete/coder_comparison.py +208 -0
- exegete/cursors.py +218 -0
- exegete/database.py +12621 -0
- exegete/env_settings.py +174 -0
- exegete/memo_privacy.py +215 -0
- exegete/names.py +68 -0
- exegete/new_project.py +1027 -0
- exegete/preview_tokens.py +613 -0
- exegete/project_settings.py +1082 -0
- exegete/pseudonymise.py +2796 -0
- exegete/refi_export.py +645 -0
- exegete/server.py +18899 -0
- exegete/sessions.py +1153 -0
- exegete/state_folder.py +360 -0
- exegete/transition.py +1049 -0
- exegete-0.14.1a0.dev1.dist-info/METADATA +515 -0
- exegete-0.14.1a0.dev1.dist-info/RECORD +24 -0
- exegete-0.14.1a0.dev1.dist-info/WHEEL +5 -0
- exegete-0.14.1a0.dev1.dist-info/entry_points.txt +2 -0
- exegete-0.14.1a0.dev1.dist-info/licenses/COPYING.LESSER +165 -0
- exegete-0.14.1a0.dev1.dist-info/licenses/NOTICE +773 -0
- exegete-0.14.1a0.dev1.dist-info/licenses/legal/GPL-3.0.txt +674 -0
- exegete-0.14.1a0.dev1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,613 @@
|
|
|
1
|
+
# SPDX-License-Identifier: LGPL-3.0-or-later
|
|
2
|
+
"""Authorisation tokens for destructive operations (v0.12, D3; H1).
|
|
3
|
+
|
|
4
|
+
A destructive tool executes only with proof that a preview of EXACTLY
|
|
5
|
+
this operation was computed and that the rows it covers have not changed
|
|
6
|
+
since. The proof is a token the preview issues and the execute presents.
|
|
7
|
+
|
|
8
|
+
Why a token rather than a `confirm` flag: `confirm=true` says "yes" to
|
|
9
|
+
whatever the tool is asked to do now, which may not be what the preview
|
|
10
|
+
the user saw described. A token is bound to the tool, to the arguments
|
|
11
|
+
that decide the effect, to the project, and to a fingerprint of the rows
|
|
12
|
+
the operation would touch, so a stale preview cannot authorise a changed
|
|
13
|
+
operation. It is also stateless: the token carries its own claims and the
|
|
14
|
+
server verifies them by recomputation, so a host that recycles the server
|
|
15
|
+
process between the preview and the execute changes nothing.
|
|
16
|
+
|
|
17
|
+
This module is the ONLY authorisation-token codec in this server. Every
|
|
18
|
+
gated tool is one row in `REGISTRY` below, which says how to canonicalise
|
|
19
|
+
its arguments; there is no tool-specific branch anywhere else, so adding
|
|
20
|
+
a tool (the flagship's `pseudonymise_source` next) is one row. A second
|
|
21
|
+
table, `KEYED_BIND`, names the tools whose bound arguments carry a value
|
|
22
|
+
the conversation never supplied; for those the public `bind` is keyed
|
|
23
|
+
with the secret, because an unkeyed digest of a secret is a confirmation
|
|
24
|
+
oracle for it (fix round 3, S2).
|
|
25
|
+
|
|
26
|
+
B4's cursor tokens are a different thing and deliberately do not share
|
|
27
|
+
this codec (D4 3.11): a cursor is a position and is replayable by design;
|
|
28
|
+
these are authorisations. The prefixes `qcp1.` and `c1.` keep them
|
|
29
|
+
distinguishable at a glance.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
import hashlib
|
|
33
|
+
import hmac
|
|
34
|
+
import json
|
|
35
|
+
import logging
|
|
36
|
+
import os
|
|
37
|
+
import re
|
|
38
|
+
import secrets
|
|
39
|
+
import stat
|
|
40
|
+
import tempfile
|
|
41
|
+
import time
|
|
42
|
+
from pathlib import Path
|
|
43
|
+
from typing import Any, Dict, Optional, Sequence, Tuple
|
|
44
|
+
|
|
45
|
+
from . import names, state_folder
|
|
46
|
+
|
|
47
|
+
logger = logging.getLogger(__name__)
|
|
48
|
+
|
|
49
|
+
TOKEN_PREFIX = "qcp1"
|
|
50
|
+
# The grammar of a token this server issues, so exactly one spelling of
|
|
51
|
+
# each AUTHENTICATED field verifies. `bind` is outside that claim by
|
|
52
|
+
# design: it is non-authoritative (D3 3.2), the MAC does not cover it,
|
|
53
|
+
# and a token with `bind` overwritten still verifies, which a gate
|
|
54
|
+
# confirmed by experiment. Nothing downstream reads it from the token;
|
|
55
|
+
# the flagship recomputes it server-side. Public for the six codebook
|
|
56
|
+
# tools, whose bound arguments are ids the conversation already holds;
|
|
57
|
+
# keyed with the secret for the flagship, whose bound mapping can be the
|
|
58
|
+
# researcher's reverse key (`KEYED_BIND`). `[0-9]`, never `\d`: in
|
|
59
|
+
# a str pattern `\d`
|
|
60
|
+
# matches every Unicode Nd digit and `int()` decodes fullwidth,
|
|
61
|
+
# Arabic-Indic, Devanagari and the rest to the same integer, so a live
|
|
62
|
+
# token re-spelled in another digit script used to verify OK, because
|
|
63
|
+
# the MAC is computed over the DECODED integer. The 12-digit bound also
|
|
64
|
+
# makes `int()` below total (CPython refuses an int-str conversion past
|
|
65
|
+
# 4300 digits), and the hex runs are lower-case because that is what
|
|
66
|
+
# `hexdigest()` produces (fix round 4).
|
|
67
|
+
_ISSUED_RE = re.compile(r"[0-9]{1,12}")
|
|
68
|
+
_BIND_RE = re.compile(r"[0-9a-f]{8}")
|
|
69
|
+
_MAC_RE = re.compile(r"[0-9a-f]{32}")
|
|
70
|
+
TOKEN_VERSION = 1
|
|
71
|
+
# Sixty minutes back, five forward: enough for a researcher to read a
|
|
72
|
+
# preview and decide, and enough tolerance for a host whose clock is a
|
|
73
|
+
# few minutes off without making a stale preview usable tomorrow.
|
|
74
|
+
TOKEN_MAX_AGE_SECONDS = 60 * 60
|
|
75
|
+
TOKEN_MAX_SKEW_SECONDS = 5 * 60
|
|
76
|
+
TOKEN_VALID_FOR_MINUTES = 60
|
|
77
|
+
|
|
78
|
+
SECRET_FILENAME = "preview_secret"
|
|
79
|
+
SECRET_READ_MAX_BYTES = 4096
|
|
80
|
+
SECRET_HEX_CHARS = 64
|
|
81
|
+
|
|
82
|
+
# Where the secret lives (v0.14.1): chosen at CALL time, never fixed at
|
|
83
|
+
# import. None means the state folder of this run, ~/.exegete (or
|
|
84
|
+
# ~/.qualcoder_mcp for a run whose move could not be made, see
|
|
85
|
+
# state_folder); the MRU hint and the AI coding sessions are derived from
|
|
86
|
+
# the same choice, so the three can never end up in different folders.
|
|
87
|
+
# The test suite sets it, and OLD_STATE_HOME, to folders of its own.
|
|
88
|
+
STATE_HOME: Optional[Path] = None
|
|
89
|
+
# The folder the state folder was moved from, which the guards refuse as
|
|
90
|
+
# well, for good (an older copy of the server may recreate it). None
|
|
91
|
+
# means ~/.qualcoder_mcp.
|
|
92
|
+
OLD_STATE_HOME: Optional[Path] = None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class PreviewSecretUnavailable(Exception):
|
|
96
|
+
"""The secret could not be created or read; no token can be trusted."""
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
SECRET_UNAVAILABLE_MESSAGE = (
|
|
100
|
+
"Could not read or create the preview-token secret in "
|
|
101
|
+
f"~/{names.STATE_FOLDER}: check permissions; nothing was changed.")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def secret_unavailable_message() -> str:
|
|
105
|
+
"""SECRET_UNAVAILABLE_MESSAGE, naming the state folder this run uses
|
|
106
|
+
(~/.qualcoder_mcp in a run whose move could not be made)."""
|
|
107
|
+
return state_folder.in_this_run(SECRET_UNAVAILABLE_MESSAGE)
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def state_home() -> Path:
|
|
111
|
+
"""The state folder, chosen at call time.
|
|
112
|
+
|
|
113
|
+
A function rather than a constant, so a test that isolates STATE_HOME
|
|
114
|
+
isolates it for every caller, including the export-path guard in the
|
|
115
|
+
server layer, and so a run whose move fell back to the old folder uses
|
|
116
|
+
it for everything.
|
|
117
|
+
"""
|
|
118
|
+
if STATE_HOME is not None:
|
|
119
|
+
return Path(STATE_HOME)
|
|
120
|
+
return state_folder.current()
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def old_state_home() -> Path:
|
|
124
|
+
"""The other of the two state folders, which the guards refuse as
|
|
125
|
+
well: the one the state folder was moved from (~/.qualcoder_mcp), or,
|
|
126
|
+
in a run whose move could not be made and which therefore uses that
|
|
127
|
+
one, ~/.exegete. So both are refused in every run."""
|
|
128
|
+
if OLD_STATE_HOME is not None:
|
|
129
|
+
return Path(OLD_STATE_HOME)
|
|
130
|
+
return state_folder.other()
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _now() -> int:
|
|
134
|
+
"""Unix seconds. Injectable so no test depends on the wall clock."""
|
|
135
|
+
return int(time.time())
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def canonical(obj: Any) -> str:
|
|
139
|
+
"""The canonical JSON this module signs.
|
|
140
|
+
|
|
141
|
+
Sorted keys and compact separators, so the same operation described
|
|
142
|
+
in a different argument order signs identically; `ensure_ascii=False`
|
|
143
|
+
with an explicit UTF-8 encode, so a non-ASCII name signs as itself
|
|
144
|
+
rather than as an escape sequence (upstream signs its own records the
|
|
145
|
+
same way, ai_mcp_server.py:322, :342 at 9bddf17).
|
|
146
|
+
"""
|
|
147
|
+
return json.dumps(obj, sort_keys=True, separators=(",", ":"),
|
|
148
|
+
ensure_ascii=False)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _secret_path() -> Path:
|
|
152
|
+
return state_home() / SECRET_FILENAME
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _valid_secret(raw: str) -> Optional[str]:
|
|
156
|
+
"""The secret if it is 64 hex characters, else None."""
|
|
157
|
+
text = raw.strip()
|
|
158
|
+
if len(text) != SECRET_HEX_CHARS:
|
|
159
|
+
return None
|
|
160
|
+
try:
|
|
161
|
+
int(text, 16)
|
|
162
|
+
except ValueError:
|
|
163
|
+
return None
|
|
164
|
+
return text.lower()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def ensure_state_dir(path: Path) -> None:
|
|
168
|
+
"""Create one of this server's state folders, owner-only, and
|
|
169
|
+
tighten a wider one.
|
|
170
|
+
|
|
171
|
+
It holds the token secret and the MRU pointer, so the group and
|
|
172
|
+
other bits have no business being set. `mkdir` alone applies the
|
|
173
|
+
umask, which on a default macOS or Linux account leaves 0755: every
|
|
174
|
+
local account could list the folder and stat the secret. Creating at
|
|
175
|
+
0700 and narrowing an existing folder costs nothing and is not
|
|
176
|
+
undone by the next start (fix round 4).
|
|
177
|
+
|
|
178
|
+
Mode bits are meaningless on Windows, where this is a no-op beyond
|
|
179
|
+
the mkdir.
|
|
180
|
+
"""
|
|
181
|
+
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
182
|
+
if os.name == "nt":
|
|
183
|
+
return
|
|
184
|
+
try:
|
|
185
|
+
mode = stat.S_IMODE(os.stat(str(path)).st_mode)
|
|
186
|
+
except OSError:
|
|
187
|
+
return
|
|
188
|
+
if mode & 0o077:
|
|
189
|
+
try:
|
|
190
|
+
os.chmod(str(path), mode & ~0o077)
|
|
191
|
+
except OSError:
|
|
192
|
+
logger.warning(state_folder.in_this_run(
|
|
193
|
+
f"The server's state folder (~/{names.STATE_FOLDER}) is "
|
|
194
|
+
"readable by other users on this machine and could not be "
|
|
195
|
+
"narrowed."))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def ensure_state_home() -> None:
|
|
199
|
+
"""`ensure_state_dir` for the token secret's own folder."""
|
|
200
|
+
ensure_state_dir(state_home())
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _publish_exclusive(tmp_name: str, path: Path,
|
|
204
|
+
windows: Optional[bool] = None) -> None:
|
|
205
|
+
"""Give a complete temp file the final name, or raise FileExistsError.
|
|
206
|
+
|
|
207
|
+
Two spellings of one rule, because the platforms differ on what
|
|
208
|
+
"create only if absent" costs:
|
|
209
|
+
|
|
210
|
+
- POSIX: `os.link`, which never replaces and raises FileExistsError.
|
|
211
|
+
`os.rename` would silently replace, and replacing is exactly what
|
|
212
|
+
D3 3.3 refuses on create.
|
|
213
|
+
- Windows: `os.rename`, which ALREADY raises when the destination
|
|
214
|
+
exists, and which works on every filesystem. `os.link` there needs
|
|
215
|
+
NTFS and the privilege to make hard links, so a user on a FAT or
|
|
216
|
+
exFAT profile would get "the preview-token secret could not be
|
|
217
|
+
created" on a machine where nothing is wrong (fix round 4).
|
|
218
|
+
|
|
219
|
+
The caller unlinks the temp either way; on Windows the rename has
|
|
220
|
+
already consumed it, which its own unlink tolerates.
|
|
221
|
+
|
|
222
|
+
`windows` is an argument rather than a read of `os.name` inside the
|
|
223
|
+
branch so a test can drive the other platform's spelling without
|
|
224
|
+
patching `os.name`, which makes `pathlib.Path()` try to build a
|
|
225
|
+
WindowsPath and takes pytest itself down with it on 3.11.
|
|
226
|
+
"""
|
|
227
|
+
if windows is None:
|
|
228
|
+
windows = os.name == "nt"
|
|
229
|
+
if windows:
|
|
230
|
+
os.rename(tmp_name, str(path))
|
|
231
|
+
else:
|
|
232
|
+
os.link(tmp_name, str(path))
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _write_new_secret(path: Path, exclusive: bool) -> str:
|
|
236
|
+
"""Create the secret. `exclusive` refuses to replace an existing one.
|
|
237
|
+
|
|
238
|
+
BOTH paths publish a COMPLETE file: the bytes are written to a
|
|
239
|
+
private temp file, fsynced, and only then given the final name.
|
|
240
|
+
First creation used to `os.open` the final path with O_CREAT|O_EXCL
|
|
241
|
+
and write afterwards, which is atomic in EXISTENCE but not in
|
|
242
|
+
CONTENT: a second server starting inside that window saw a
|
|
243
|
+
zero-length file, judged it malformed and ROTATED over the winner's
|
|
244
|
+
secret, whose own write then landed on an unlinked inode. Measured
|
|
245
|
+
at 24 simultaneous starts on a fresh state home: three distinct
|
|
246
|
+
secrets in one run.
|
|
247
|
+
|
|
248
|
+
The publish step is what differs. First creation uses `os.link`,
|
|
249
|
+
which creates the final name only if it does not exist and raises
|
|
250
|
+
FileExistsError otherwise, so first-writer-wins still holds and the
|
|
251
|
+
loser reads the winner's secret. A ROTATION (an existing file we
|
|
252
|
+
cannot read) uses `os.replace`, because the final path is already
|
|
253
|
+
taken. D3 3.3 rejects replace-on-create for a real reason: last
|
|
254
|
+
writer wins, and the first server's outstanding tokens are orphaned.
|
|
255
|
+
"""
|
|
256
|
+
value = secrets.token_hex(32)
|
|
257
|
+
ensure_state_home()
|
|
258
|
+
fd, tmp_name = tempfile.mkstemp(dir=str(state_home()),
|
|
259
|
+
prefix=f"{SECRET_FILENAME}.",
|
|
260
|
+
suffix=".tmp")
|
|
261
|
+
tmp: Optional[Path] = Path(tmp_name)
|
|
262
|
+
try:
|
|
263
|
+
# If `os.fdopen` raises, the descriptor mkstemp returned is
|
|
264
|
+
# still open and nothing owns it. POSIX lets the cleanup below
|
|
265
|
+
# unlink an open file, so the leak was invisible here; Windows
|
|
266
|
+
# refuses (ERROR_SHARING_VIOLATION), so the temp file survived
|
|
267
|
+
# the failure and the state folder was left with litter. Hand
|
|
268
|
+
# the descriptor to the file object or close it; never neither
|
|
269
|
+
# (fix round 5).
|
|
270
|
+
try:
|
|
271
|
+
handle = os.fdopen(fd, "w", encoding="ascii")
|
|
272
|
+
except BaseException:
|
|
273
|
+
os.close(fd)
|
|
274
|
+
raise
|
|
275
|
+
with handle as f:
|
|
276
|
+
f.write(value + "\n")
|
|
277
|
+
f.flush()
|
|
278
|
+
os.fsync(f.fileno())
|
|
279
|
+
if os.name != "nt":
|
|
280
|
+
os.chmod(tmp_name, 0o600) # mkstemp already does; be sure
|
|
281
|
+
if exclusive:
|
|
282
|
+
_publish_exclusive(tmp_name, path)
|
|
283
|
+
# the link left the temp behind; the finally clause takes it
|
|
284
|
+
else:
|
|
285
|
+
os.replace(tmp_name, str(path))
|
|
286
|
+
tmp = None # the replace consumed it
|
|
287
|
+
except BaseException:
|
|
288
|
+
if tmp is not None:
|
|
289
|
+
try:
|
|
290
|
+
tmp.unlink()
|
|
291
|
+
except OSError:
|
|
292
|
+
pass
|
|
293
|
+
tmp = None
|
|
294
|
+
raise
|
|
295
|
+
finally:
|
|
296
|
+
if tmp is not None:
|
|
297
|
+
try:
|
|
298
|
+
tmp.unlink() # the link left the temp behind
|
|
299
|
+
except OSError:
|
|
300
|
+
pass
|
|
301
|
+
return value
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def load_secret() -> str:
|
|
305
|
+
"""The per-user HMAC secret, read from disk on every use.
|
|
306
|
+
|
|
307
|
+
Never cached: several servers on one machine must agree, and a
|
|
308
|
+
rotation by one must be seen by the others. A symlink at the path is
|
|
309
|
+
refused rather than followed (the pre-planted-symlink class the MRU
|
|
310
|
+
reader already guards against), an unreadable or malformed file is
|
|
311
|
+
ROTATED and logged at warning level (outstanding tokens then fail as
|
|
312
|
+
"a different operation", which is the safe direction), and any
|
|
313
|
+
failure to create or read raises rather than falling back to a
|
|
314
|
+
token-less execute.
|
|
315
|
+
|
|
316
|
+
Raises:
|
|
317
|
+
PreviewSecretUnavailable: with the fixed caller-facing message.
|
|
318
|
+
"""
|
|
319
|
+
path = _secret_path()
|
|
320
|
+
try:
|
|
321
|
+
if not os.path.lexists(path):
|
|
322
|
+
try:
|
|
323
|
+
return _write_new_secret(path, exclusive=True)
|
|
324
|
+
except FileExistsError:
|
|
325
|
+
pass # another server won the race
|
|
326
|
+
st = os.lstat(path)
|
|
327
|
+
if stat.S_ISLNK(st.st_mode) or not stat.S_ISREG(st.st_mode):
|
|
328
|
+
raise PreviewSecretUnavailable(secret_unavailable_message())
|
|
329
|
+
# The mode was set at creation and never looked at again, so a
|
|
330
|
+
# secret that had been widened since (by a restore, a copy, a
|
|
331
|
+
# sync tool, or another local account) was used as though it
|
|
332
|
+
# were still private. A secret any local user can read is a
|
|
333
|
+
# secret any local user can mint tokens with, so it is treated
|
|
334
|
+
# like any other unusable one and ROTATED, which writes a fresh
|
|
335
|
+
# 0600 file; tightening the old one in place would leave a value
|
|
336
|
+
# that may already have been read (fix round 4).
|
|
337
|
+
widened = os.name != "nt" and bool(stat.S_IMODE(st.st_mode) & 0o077)
|
|
338
|
+
if widened:
|
|
339
|
+
logger.warning(
|
|
340
|
+
"The preview-token secret was readable by other users on "
|
|
341
|
+
"this machine and has been rotated; outstanding preview "
|
|
342
|
+
"tokens will be refused.")
|
|
343
|
+
return _write_new_secret(path, exclusive=False)
|
|
344
|
+
with open(path, "r", encoding="ascii", errors="replace") as f:
|
|
345
|
+
raw = f.read(SECRET_READ_MAX_BYTES + 1)
|
|
346
|
+
value = _valid_secret(raw)
|
|
347
|
+
if value is None:
|
|
348
|
+
logger.warning(
|
|
349
|
+
"The preview-token secret was not usable and has been "
|
|
350
|
+
"rotated; outstanding preview tokens will be refused.")
|
|
351
|
+
return _write_new_secret(path, exclusive=False)
|
|
352
|
+
return value
|
|
353
|
+
except PreviewSecretUnavailable:
|
|
354
|
+
raise
|
|
355
|
+
except OSError as e:
|
|
356
|
+
logger.error("Preview secret unavailable: %s", type(e).__name__)
|
|
357
|
+
raise PreviewSecretUnavailable(secret_unavailable_message()) from e
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
# ---------------------------------------------------------------------------
|
|
361
|
+
# The registration table (H1): one row per gated tool, no branches elsewhere
|
|
362
|
+
# ---------------------------------------------------------------------------
|
|
363
|
+
# `args` maps the tool's call arguments to the canonical `A` that decides
|
|
364
|
+
# the EFFECT of the operation. Arguments that do not change what the
|
|
365
|
+
# operation does to the data are deliberately absent: `preview_token` and
|
|
366
|
+
# `confirm` never appear, and `cascade` is not bound because the preview
|
|
367
|
+
# always reports the whole branch, so the impact shown does not depend on
|
|
368
|
+
# it (D3 3.2); `cascade` still gates the execute on its own.
|
|
369
|
+
|
|
370
|
+
def _args_merge_codes(kwargs):
|
|
371
|
+
return {"from_code_id": int(kwargs["from_code_id"]),
|
|
372
|
+
"into_code_id": int(kwargs["into_code_id"])}
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _args_delete_code(kwargs):
|
|
376
|
+
return {"code_id": int(kwargs["code_id"])}
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _args_delete_category(kwargs):
|
|
380
|
+
return {"category_id": int(kwargs["category_id"])}
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _args_merge_category(kwargs):
|
|
384
|
+
target = kwargs.get("into_category_id")
|
|
385
|
+
return {"from_category_id": int(kwargs["from_category_id"]),
|
|
386
|
+
# The RESOLVED id, never the name: if the name is given to a
|
|
387
|
+
# different category between preview and execute, the binding
|
|
388
|
+
# differs and the model is told to preview again.
|
|
389
|
+
"into_category_id": None if target is None else int(target)}
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _args_restore_backup(kwargs):
|
|
393
|
+
return {"backup": os.path.normcase(str(kwargs["backup"]))}
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _args_prune_backups(kwargs):
|
|
397
|
+
keep = kwargs.get("keep_last")
|
|
398
|
+
older = kwargs.get("older_than_days")
|
|
399
|
+
return {"keep_last": None if keep is None else int(keep),
|
|
400
|
+
"older_than_days": None if older is None else float(older)}
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _args_pseudonymise_source(kwargs):
|
|
404
|
+
"""The arguments that decide what a pseudonymisation run does.
|
|
405
|
+
|
|
406
|
+
`mapping` arrives already in its canonical form (entries sorted by
|
|
407
|
+
original, variants sorted, every string NFC-normalised), the same way
|
|
408
|
+
`merge_category` passes the RESOLVED category id rather than the name
|
|
409
|
+
it was given: the canonicalisation belongs to the module that knows
|
|
410
|
+
what a mapping is, and what reaches this table is the settled value.
|
|
411
|
+
Canonicalising it there is also what keeps `use_project_pseudonyms`
|
|
412
|
+
OUT of the binding, so the identical mapping binds the same whether
|
|
413
|
+
it was typed out or read from the project's own `pseudonyms.json`.
|
|
414
|
+
|
|
415
|
+
The preview-only arguments are absent, as `cascade` is: `include_context`,
|
|
416
|
+
`context_chars`, `scan_residue`, `residue_detail` and
|
|
417
|
+
`max_spans_per_entry` change what the preview SHOWS, and
|
|
418
|
+
`record_in_journal` changes only whether the run records itself, so
|
|
419
|
+
none of them changes what happens to the text or to a single row.
|
|
420
|
+
|
|
421
|
+
`rewrite_memos` changes what happens to rows (the public part of
|
|
422
|
+
every note in the project), so it is bound (v0.13, Brief 2).
|
|
423
|
+
`save_mapping_to_project` writes a file of real names into the
|
|
424
|
+
project folder, a side effect the human must have seen in the
|
|
425
|
+
preview, so it is bound. `researcher_keeps_mapping` is an attestation
|
|
426
|
+
with no side effect, in the class of `record_in_journal`, and is not.
|
|
427
|
+
|
|
428
|
+
`file_id` is one id since v0.13 (one file per call), bound as the
|
|
429
|
+
integer the server validated. `TOKEN_VERSION` is not bumped for it:
|
|
430
|
+
a token lives sixty minutes, and one issued by 0.12 for `file_ids`
|
|
431
|
+
that did reach this binding would fail as `token_other_operation`,
|
|
432
|
+
which is what it is. The `bool()` on each switch is so that a truthy
|
|
433
|
+
value that is not `True` cannot bind differently from `True`.
|
|
434
|
+
"""
|
|
435
|
+
return {"file_id": int(kwargs["file_id"]),
|
|
436
|
+
"mapping": kwargs["mapping"],
|
|
437
|
+
"case_mode": str(kwargs["case_mode"]),
|
|
438
|
+
"overlap_policy": str(kwargs["overlap_policy"]),
|
|
439
|
+
"rewrite_memos": bool(kwargs["rewrite_memos"]),
|
|
440
|
+
"save_mapping_to_project": bool(
|
|
441
|
+
kwargs["save_mapping_to_project"])}
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
REGISTRY: Dict[str, Any] = {
|
|
445
|
+
"merge_codes": _args_merge_codes,
|
|
446
|
+
"delete_code": _args_delete_code,
|
|
447
|
+
"delete_category": _args_delete_category,
|
|
448
|
+
"merge_category": _args_merge_category,
|
|
449
|
+
"restore_backup": _args_restore_backup,
|
|
450
|
+
"prune_backups": _args_prune_backups,
|
|
451
|
+
"pseudonymise_source": _args_pseudonymise_source,
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
# The tools whose public `bind` is keyed with the secret. D3 3.2 declared
|
|
455
|
+
# `bind` public on the premise that every bound argument was already in
|
|
456
|
+
# the conversation, which is true of a code id and false of a mapping
|
|
457
|
+
# read from the project's own `pseudonyms.json`: with the pseudonym and
|
|
458
|
+
# the other three arguments visible in the preview, an unkeyed sha256
|
|
459
|
+
# over the canonical mapping let a dictionary of first names confirm
|
|
460
|
+
# the original in twenty guesses. Keying it costs nothing the verifier
|
|
461
|
+
# does not already have (it holds the secret), so `verify` still tells
|
|
462
|
+
# `project_changed` from `token_other_operation`, and nobody without the
|
|
463
|
+
# secret can confirm a guess (fix round 3, S2).
|
|
464
|
+
KEYED_BIND = frozenset({"pseudonymise_source"})
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def canonical_args(tool: str, **kwargs) -> Dict[str, Any]:
|
|
468
|
+
"""The canonical arguments for a gated tool (the only place they live)."""
|
|
469
|
+
if tool not in REGISTRY:
|
|
470
|
+
raise KeyError(f"{tool} is not a token-gated tool")
|
|
471
|
+
return REGISTRY[tool](kwargs)
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def fingerprint_rows(preview: Any, rows: Any) -> str:
|
|
475
|
+
"""The state element `S`: what the preview said, and which rows it covers.
|
|
476
|
+
|
|
477
|
+
The digest binds BOTH the numbers the user was shown and the identity
|
|
478
|
+
of the rows behind them, so neither a changed count nor a swapped row
|
|
479
|
+
can slip through a token issued for the earlier state. Row tuples
|
|
480
|
+
carry ids, positions and ownership plus `has_memo` and `has_private`
|
|
481
|
+
booleans; note rows (v0.13, `rewrite_memos`) carry their key, the
|
|
482
|
+
private-part flag and the public part's length. Memo text never
|
|
483
|
+
enters the payload, on principle, even though an HMAC would not
|
|
484
|
+
reveal it.
|
|
485
|
+
"""
|
|
486
|
+
return hashlib.sha256(
|
|
487
|
+
canonical({"preview": preview, "rows": rows}).encode("utf-8")
|
|
488
|
+
).hexdigest()
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _binding(tool: str, args: Dict[str, Any], project: str) -> Dict[str, Any]:
|
|
492
|
+
return {"v": TOKEN_VERSION, "tool": tool, "args": args,
|
|
493
|
+
"project": project}
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def bind_id(tool: str, args: Dict[str, Any], project: str,
|
|
497
|
+
secret: Optional[str] = None) -> str:
|
|
498
|
+
"""The non-authoritative operation id carried in the token.
|
|
499
|
+
|
|
500
|
+
Eight hex characters over the binding, so the verifier can tell "this
|
|
501
|
+
token is for another operation" from "the project changed under this
|
|
502
|
+
one" and say the more useful of the two. It proves nothing by itself;
|
|
503
|
+
the MAC does that.
|
|
504
|
+
|
|
505
|
+
A plain digest for the tools whose arguments the conversation already
|
|
506
|
+
holds; HMAC under the secret for the tools in `KEYED_BIND`, whose
|
|
507
|
+
arguments it may not. For those the secret is the caller's to pass
|
|
508
|
+
(`issue`, `verify` and the flagship all hold it), never loaded here
|
|
509
|
+
on the caller's behalf: a keyed bind computed by any code running
|
|
510
|
+
as the researcher must say so, so that a verifier's recomputation
|
|
511
|
+
cannot be mistaken for an attacker's (fix round 4, L1).
|
|
512
|
+
"""
|
|
513
|
+
payload = canonical(_binding(tool, args, project)).encode("utf-8")
|
|
514
|
+
if tool in KEYED_BIND:
|
|
515
|
+
if secret is None:
|
|
516
|
+
raise TypeError(
|
|
517
|
+
f"bind_id needs the secret for {tool!r}: its bind is "
|
|
518
|
+
f"keyed, and the caller passes the secret it holds")
|
|
519
|
+
return hmac.new(secret.encode("ascii"), payload,
|
|
520
|
+
hashlib.sha256).hexdigest()[:8]
|
|
521
|
+
return hashlib.sha256(payload).hexdigest()[:8]
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def _mac(secret: str, tool: str, args: Dict[str, Any], project: str,
|
|
525
|
+
state: str, issued: int) -> str:
|
|
526
|
+
payload = _binding(tool, args, project)
|
|
527
|
+
payload["state"] = state
|
|
528
|
+
payload["issued"] = issued
|
|
529
|
+
return hmac.new(secret.encode("ascii"),
|
|
530
|
+
canonical(payload).encode("utf-8"),
|
|
531
|
+
hashlib.sha256).hexdigest()[:32]
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def issue(tool: str, args: Dict[str, Any], project: str, state: str,
|
|
535
|
+
now: Optional[int] = None) -> str:
|
|
536
|
+
"""Mint a token for one previewed operation."""
|
|
537
|
+
issued = _now() if now is None else int(now)
|
|
538
|
+
secret = load_secret()
|
|
539
|
+
bind = bind_id(tool, args, project, secret)
|
|
540
|
+
token = (f"{TOKEN_PREFIX}.{issued}.{bind}."
|
|
541
|
+
f"{_mac(secret, tool, args, project, state, issued)}")
|
|
542
|
+
logger.debug("Issued preview token for %s (%s)", tool, bind)
|
|
543
|
+
return token
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
# Verification outcomes. The caller turns these into the fixed texts of
|
|
547
|
+
# D3 3.5; this module decides WHICH failure it was and nothing else.
|
|
548
|
+
OK = "ok"
|
|
549
|
+
MALFORMED = "token_malformed"
|
|
550
|
+
EXPIRED = "token_expired"
|
|
551
|
+
OTHER_OPERATION = "token_other_operation"
|
|
552
|
+
PROJECT_CHANGED = "project_changed"
|
|
553
|
+
|
|
554
|
+
|
|
555
|
+
def verify(token: Any, tool: str, args: Dict[str, Any], project: str,
|
|
556
|
+
state: str, now: Optional[int] = None) -> str:
|
|
557
|
+
"""Check a token against the operation it is being used for.
|
|
558
|
+
|
|
559
|
+
Returns one of OK, MALFORMED, EXPIRED, PROJECT_CHANGED or
|
|
560
|
+
OTHER_OPERATION. A token whose MAC does not match is a refusal either
|
|
561
|
+
way; the public `bind` decides which of the two explanations the
|
|
562
|
+
caller gets, since a matching bind means this token WAS issued for
|
|
563
|
+
this operation and the rows have moved since. Expiry is checked
|
|
564
|
+
before the MAC so an old token for the right operation gets the more
|
|
565
|
+
useful message. Every comparison uses `hmac.compare_digest`.
|
|
566
|
+
"""
|
|
567
|
+
if not isinstance(token, str):
|
|
568
|
+
return MALFORMED
|
|
569
|
+
token = token.strip()
|
|
570
|
+
# Every field of a token this server issues is ASCII: the prefix, a
|
|
571
|
+
# decimal timestamp and two lower-case hex runs. Gate on that once,
|
|
572
|
+
# here, rather than per field: hmac.compare_digest REFUSES to
|
|
573
|
+
# compare strings with non-ASCII characters and raises TypeError,
|
|
574
|
+
# which would leave verify() by raising instead of returning
|
|
575
|
+
# MALFORMED, and the caller would lose the fixed refusal envelope
|
|
576
|
+
# for a raw Python message (fix round 4).
|
|
577
|
+
if not token.isascii():
|
|
578
|
+
return MALFORMED
|
|
579
|
+
parts = token.split(".")
|
|
580
|
+
if len(parts) != 4 or parts[0] != TOKEN_PREFIX:
|
|
581
|
+
return MALFORMED
|
|
582
|
+
_, issued_text, bind, mac = parts
|
|
583
|
+
# A grammar, not a predicate: str.isdigit() is True for superscripts
|
|
584
|
+
# and circled digits that int() then REJECTS, raising ValueError
|
|
585
|
+
# past every caller, and int(x, 16) accepts spellings that are not
|
|
586
|
+
# what this server writes.
|
|
587
|
+
if not (_ISSUED_RE.fullmatch(issued_text) and _BIND_RE.fullmatch(bind)
|
|
588
|
+
and _MAC_RE.fullmatch(mac)):
|
|
589
|
+
return MALFORMED
|
|
590
|
+
issued = int(issued_text)
|
|
591
|
+
# One spelling only: "0000001700000000" decodes to the same integer
|
|
592
|
+
# and would verify under the same MAC.
|
|
593
|
+
if str(issued) != issued_text:
|
|
594
|
+
return MALFORMED
|
|
595
|
+
current = _now() if now is None else int(now)
|
|
596
|
+
if issued > current + TOKEN_MAX_SKEW_SECONDS:
|
|
597
|
+
return EXPIRED
|
|
598
|
+
if issued < current - TOKEN_MAX_AGE_SECONDS:
|
|
599
|
+
return EXPIRED
|
|
600
|
+
secret = load_secret()
|
|
601
|
+
expected = _mac(secret, tool, args, project, state, issued)
|
|
602
|
+
if not hmac.compare_digest(mac, expected):
|
|
603
|
+
# The MAC covers the state as well as the operation, so a failure
|
|
604
|
+
# means one of the two moved. `bind` is exactly what tells them
|
|
605
|
+
# apart: it covers the tool, the arguments and the project and
|
|
606
|
+
# nothing else, so a token whose bind still matches was issued
|
|
607
|
+
# for THIS operation and the project has changed under it, which
|
|
608
|
+
# is the more useful thing to say. It is public and proves
|
|
609
|
+
# nothing on its own; the MAC has already refused either way.
|
|
610
|
+
if hmac.compare_digest(bind, bind_id(tool, args, project, secret)):
|
|
611
|
+
return PROJECT_CHANGED
|
|
612
|
+
return OTHER_OPERATION
|
|
613
|
+
return OK
|