lorebase-keeper 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lorebase_keeper/__init__.py +1 -0
- lorebase_keeper/__main__.py +456 -0
- lorebase_keeper/agents/__init__.py +1 -0
- lorebase_keeper/agents/budget.py +247 -0
- lorebase_keeper/agents/context.py +529 -0
- lorebase_keeper/agents/explain.py +582 -0
- lorebase_keeper/agents/feed.py +259 -0
- lorebase_keeper/agents/markdown.py +219 -0
- lorebase_keeper/agents/pack.py +403 -0
- lorebase_keeper/agents/settings.py +111 -0
- lorebase_keeper/app.py +373 -0
- lorebase_keeper/auth/__init__.py +4 -0
- lorebase_keeper/auth/api_keys.py +127 -0
- lorebase_keeper/auth/audit.py +356 -0
- lorebase_keeper/auth/cache.py +53 -0
- lorebase_keeper/auth/config.py +618 -0
- lorebase_keeper/auth/deps.py +183 -0
- lorebase_keeper/auth/keyring.py +14 -0
- lorebase_keeper/auth/oidc.py +242 -0
- lorebase_keeper/auth/permissions.py +142 -0
- lorebase_keeper/auth/ports.py +212 -0
- lorebase_keeper/auth/principals.py +377 -0
- lorebase_keeper/auth/provider.py +250 -0
- lorebase_keeper/auth/rbac.py +247 -0
- lorebase_keeper/auth/read_audit.py +357 -0
- lorebase_keeper/auth/runtime.py +113 -0
- lorebase_keeper/auth/sessions.py +452 -0
- lorebase_keeper/bootstrap.py +151 -0
- lorebase_keeper/client_address.py +129 -0
- lorebase_keeper/context.py +226 -0
- lorebase_keeper/db.py +93 -0
- lorebase_keeper/deps.py +47 -0
- lorebase_keeper/errors.py +257 -0
- lorebase_keeper/health.py +61 -0
- lorebase_keeper/kindhealth/__init__.py +6 -0
- lorebase_keeper/kindhealth/config.py +51 -0
- lorebase_keeper/kindhealth/markdown.py +163 -0
- lorebase_keeper/kindhealth/models.py +179 -0
- lorebase_keeper/kindhealth/report.py +154 -0
- lorebase_keeper/kindhealth/scan.py +512 -0
- lorebase_keeper/middleware.py +285 -0
- lorebase_keeper/pagination.py +87 -0
- lorebase_keeper/ratelimit/__init__.py +10 -0
- lorebase_keeper/ratelimit/bucket.py +118 -0
- lorebase_keeper/ratelimit/config.py +283 -0
- lorebase_keeper/ratelimit/enforce.py +393 -0
- lorebase_keeper/ratelimit/limiter.py +149 -0
- lorebase_keeper/ratelimit/quotas.py +204 -0
- lorebase_keeper/ratelimit/shared.py +242 -0
- lorebase_keeper/readiness.py +257 -0
- lorebase_keeper/reading/__init__.py +1 -0
- lorebase_keeper/reading/caching.py +251 -0
- lorebase_keeper/reading/compiler.py +678 -0
- lorebase_keeper/reading/cursors.py +56 -0
- lorebase_keeper/reading/diff.py +408 -0
- lorebase_keeper/reading/graph.py +268 -0
- lorebase_keeper/reading/history.py +152 -0
- lorebase_keeper/reading/points.py +410 -0
- lorebase_keeper/reading/problems.py +74 -0
- lorebase_keeper/reading/record_history.py +722 -0
- lorebase_keeper/reading/records.py +780 -0
- lorebase_keeper/reading/requests.py +107 -0
- lorebase_keeper/reading/runtime.py +145 -0
- lorebase_keeper/reading/search.py +708 -0
- lorebase_keeper/reading/sensitivity.py +882 -0
- lorebase_keeper/reading/settings.py +339 -0
- lorebase_keeper/reading/snapshots.py +148 -0
- lorebase_keeper/reading/tree.py +98 -0
- lorebase_keeper/reading/views.py +264 -0
- lorebase_keeper/reading/visibility.py +93 -0
- lorebase_keeper/routes/__init__.py +1 -0
- lorebase_keeper/routes/admin.py +597 -0
- lorebase_keeper/routes/agents.py +160 -0
- lorebase_keeper/routes/branches.py +438 -0
- lorebase_keeper/routes/derivations.py +215 -0
- lorebase_keeper/routes/documents.py +497 -0
- lorebase_keeper/routes/domain_api.py +59 -0
- lorebase_keeper/routes/domains.py +220 -0
- lorebase_keeper/routes/events.py +187 -0
- lorebase_keeper/routes/kind_health.py +166 -0
- lorebase_keeper/routes/operations.py +392 -0
- lorebase_keeper/routes/proposal_items.py +563 -0
- lorebase_keeper/routes/proposals.py +1107 -0
- lorebase_keeper/routes/read.py +493 -0
- lorebase_keeper/routes/revisions.py +523 -0
- lorebase_keeper/routes/schema.py +682 -0
- lorebase_keeper/routes/schema_inference.py +437 -0
- lorebase_keeper/routes/schema_suggestions.py +328 -0
- lorebase_keeper/routes/sessions.py +286 -0
- lorebase_keeper/routes/source_preview.py +232 -0
- lorebase_keeper/routes/sources.py +200 -0
- lorebase_keeper/routes/trace.py +506 -0
- lorebase_keeper/routes/vault.py +332 -0
- lorebase_keeper/routes/webhooks.py +298 -0
- lorebase_keeper/routes/whoami.py +88 -0
- lorebase_keeper/services/__init__.py +1 -0
- lorebase_keeper/services/agent_writes.py +1035 -0
- lorebase_keeper/services/branches.py +337 -0
- lorebase_keeper/services/config.py +431 -0
- lorebase_keeper/services/derivation_preview.py +279 -0
- lorebase_keeper/services/document_urls.py +114 -0
- lorebase_keeper/services/domains.py +118 -0
- lorebase_keeper/services/errors.py +196 -0
- lorebase_keeper/services/operation_catalog.py +806 -0
- lorebase_keeper/services/operation_events.py +199 -0
- lorebase_keeper/services/operations.py +1229 -0
- lorebase_keeper/services/pii_import.py +222 -0
- lorebase_keeper/services/proposal_reads.py +158 -0
- lorebase_keeper/services/proposals.py +850 -0
- lorebase_keeper/services/reads.py +286 -0
- lorebase_keeper/services/schema.py +601 -0
- lorebase_keeper/services/schema_inference.py +390 -0
- lorebase_keeper/services/schema_suggestions.py +418 -0
- lorebase_keeper/services/source_preview.py +127 -0
- lorebase_keeper/settings.py +313 -0
- lorebase_keeper/tracing/__init__.py +1 -0
- lorebase_keeper/tracing/capture.py +350 -0
- lorebase_keeper/tracing/config.py +157 -0
- lorebase_keeper/tracing/explain.py +97 -0
- lorebase_keeper/tracing/middleware.py +359 -0
- lorebase_keeper/tracing/notes.py +80 -0
- lorebase_keeper/tracing/recorder.py +180 -0
- lorebase_keeper-0.2.3.dist-info/METADATA +21 -0
- lorebase_keeper-0.2.3.dist-info/RECORD +126 -0
- lorebase_keeper-0.2.3.dist-info/WHEEL +4 -0
- lorebase_keeper-0.2.3.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
"""Erasure of personal data held in the vault (design 13.5, AC-22).
|
|
2
|
+
|
|
3
|
+
`POST /domains/{d}/vault/erase` erases the vaulted values of one data subject, needing
|
|
4
|
+
`pii.erase` on the whole Domain (only `admin` holds it among the built-in roles):
|
|
5
|
+
|
|
6
|
+
- by Record: `kind` and `key`, optionally limited to some vault `attributes`; every value the
|
|
7
|
+
Record ever held in them, under every key it had (key changes), on every branch and Revision;
|
|
8
|
+
- by value: `kind`, a vault `attribute` and the `value`; every Record of the Kind holding it
|
|
9
|
+
or a case variant of it (same matching form, `lorebase_core.types.matching`), on every branch
|
|
10
|
+
and Revision.
|
|
11
|
+
|
|
12
|
+
Revisions keep their hashes and structure: Record objects hold tokens, and reads of an erased
|
|
13
|
+
token answer `keeper.reads.sensitivity.erased_value`. Erasing again changes nothing (`erased`
|
|
14
|
+
counts the values erased by this request). A later write of an erased value is stored erased,
|
|
15
|
+
refused or stored again, as `vault.erased_values` says. The erasure and its audit entry (action
|
|
16
|
+
`vault.erase`, with the Kind, a keyed digest of the key (`lorebase_store.digests`) or the
|
|
17
|
+
Attribute, and the count, never the key or the value) commit in one transaction. Attribute and
|
|
18
|
+
Kind names are read in the schema of `branch` (default: the Domain's default branch).
|
|
19
|
+
|
|
20
|
+
Vault keys rotate (`lorebase_store.vault.rotation`), under `vault.manage`, which installation
|
|
21
|
+
administrators hold: `GET /vault/keys` counts the rows stored under each encryption key and
|
|
22
|
+
fingerprint key and tells which earlier keys no row uses any more; `POST /vault/rotate` queues
|
|
23
|
+
the `vault.rotate` job that makes the outdated rows of every Domain again under the current
|
|
24
|
+
keys, in batches, continuing where it stopped.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import logging
|
|
30
|
+
from typing import Annotated, Any, Literal, Self
|
|
31
|
+
|
|
32
|
+
from fastapi import APIRouter, Depends, Request
|
|
33
|
+
from pydantic import BaseModel, Field, JsonValue, model_validator
|
|
34
|
+
|
|
35
|
+
from lorebase_core.model import Kind, Schema
|
|
36
|
+
from lorebase_core.types import EncodingError, encode_value
|
|
37
|
+
from lorebase_keeper.auth.audit import annotate, suppress
|
|
38
|
+
from lorebase_keeper.auth.deps import Access, require, require_installation
|
|
39
|
+
from lorebase_keeper.auth.permissions import Permission
|
|
40
|
+
from lorebase_keeper.context import StoreDep, require_domain
|
|
41
|
+
from lorebase_keeper.errors import (
|
|
42
|
+
CONFLICT,
|
|
43
|
+
NOT_FOUND,
|
|
44
|
+
UNAVAILABLE,
|
|
45
|
+
Problem,
|
|
46
|
+
ProblemError,
|
|
47
|
+
ValidationProblem,
|
|
48
|
+
)
|
|
49
|
+
from lorebase_keeper.reading.problems import invalid
|
|
50
|
+
from lorebase_store.auth import AuditRepository, NewAuditEntry
|
|
51
|
+
from lorebase_store.commit.schema import load_compiled
|
|
52
|
+
from lorebase_store.jobs import VAULT_ROTATE_JOB, enqueue_job
|
|
53
|
+
from lorebase_store.repositories import MaterializedRepository
|
|
54
|
+
from lorebase_store.vault import erase_matching, erase_record, usage
|
|
55
|
+
from lorebase_store.vault.store import VaultLockTimeoutError
|
|
56
|
+
|
|
57
|
+
router = APIRouter(tags=["vault"])
|
|
58
|
+
log = logging.getLogger("lorebase.keeper.vault")
|
|
59
|
+
|
|
60
|
+
ERASE_ACTION = "vault.erase"
|
|
61
|
+
KEY_DIGEST = "erased_record_key"
|
|
62
|
+
"""Purpose of the keyed digest of `<kind id>/<key>` held by the audit entry of an erasure."""
|
|
63
|
+
|
|
64
|
+
_ERRORS: dict[int | str, dict[str, Any]] = {
|
|
65
|
+
403: {"model": Problem, "description": "The caller lacks pii.erase on the Domain."},
|
|
66
|
+
404: {"model": Problem, "description": "Domain, branch or Kind not found."},
|
|
67
|
+
409: {"model": Problem, "description": "Erasure by value needs the vault keys."},
|
|
68
|
+
503: {"model": Problem, "description": "The vault of the Domain stayed locked."},
|
|
69
|
+
422: {"model": ValidationProblem, "description": "Invalid request."},
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
Erase = Annotated[Access, Depends(require(Permission.PII_ERASE))]
|
|
73
|
+
Manage = Annotated[Access, Depends(require_installation(Permission.VAULT_MANAGE))]
|
|
74
|
+
|
|
75
|
+
ROTATE_ACTION = "vault.rotate"
|
|
76
|
+
|
|
77
|
+
_KEY_ERRORS: dict[int | str, dict[str, Any]] = {
|
|
78
|
+
403: {"model": Problem, "description": "The caller is not an installation administrator."},
|
|
79
|
+
409: {"model": Problem, "description": "The vault is not configured."},
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class EraseIn(BaseModel):
|
|
84
|
+
"""Whose values to erase: a Record (`key`, optional `attributes`) or a value
|
|
85
|
+
(`attribute` and `value`)."""
|
|
86
|
+
|
|
87
|
+
kind: str = Field(description="Kind api name.")
|
|
88
|
+
key: str | None = Field(default=None, description="Key of the Record whose values to erase.")
|
|
89
|
+
attributes: list[str] | None = Field(
|
|
90
|
+
default=None,
|
|
91
|
+
min_length=1,
|
|
92
|
+
description="With `key`: only these vault Attributes (api names); default: all.",
|
|
93
|
+
)
|
|
94
|
+
attribute: str | None = Field(
|
|
95
|
+
default=None, description="Vault Attribute (api name) whose `value` to erase."
|
|
96
|
+
)
|
|
97
|
+
value: JsonValue = Field(
|
|
98
|
+
default=None, description="With `attribute`: the value to erase in every Record."
|
|
99
|
+
)
|
|
100
|
+
branch: str | None = Field(
|
|
101
|
+
default=None, description="Branch whose schema names Kinds and Attributes."
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
@model_validator(mode="after")
|
|
105
|
+
def _one_subject(self) -> Self:
|
|
106
|
+
by_key = self.key is not None
|
|
107
|
+
by_value = self.attribute is not None
|
|
108
|
+
if by_key == by_value:
|
|
109
|
+
raise ValueError("give either key (and attributes) or attribute and value")
|
|
110
|
+
if by_value and (self.value is None or self.attributes is not None):
|
|
111
|
+
raise ValueError("erasure by value takes attribute and value only")
|
|
112
|
+
return self
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class EraseOut(BaseModel):
|
|
116
|
+
domain: str
|
|
117
|
+
kind: str
|
|
118
|
+
mode: Literal["record", "value"]
|
|
119
|
+
erased: int = Field(description="Vault values erased by this request.")
|
|
120
|
+
unopenable: int = Field(
|
|
121
|
+
default=0,
|
|
122
|
+
description="Rows looked at that are sealed under a key no longer configured: their "
|
|
123
|
+
"fingerprints could not be made again, so a case variant among them may be missed.",
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
@router.post(
|
|
128
|
+
"/domains/{domain_id}/vault/erase",
|
|
129
|
+
summary="Erase personal data from the vault",
|
|
130
|
+
response_model=EraseOut,
|
|
131
|
+
responses=_ERRORS,
|
|
132
|
+
)
|
|
133
|
+
async def erase(
|
|
134
|
+
domain_id: str, body: EraseIn, access: Erase, store: StoreDep, request: Request
|
|
135
|
+
) -> EraseOut:
|
|
136
|
+
"""Erase the vaulted values of a Record or a value in a Kind, on every Revision."""
|
|
137
|
+
annotate(request, action=ERASE_ACTION, target=body.kind)
|
|
138
|
+
try:
|
|
139
|
+
async with store.sessions() as session, session.begin():
|
|
140
|
+
domain = await require_domain(session, domain_id)
|
|
141
|
+
branch = body.branch or domain.default_branch
|
|
142
|
+
compiled = await MaterializedRepository(
|
|
143
|
+
session, domain_id, branch, writes=store.runtime.settings.writes
|
|
144
|
+
).schema()
|
|
145
|
+
if compiled is None:
|
|
146
|
+
raise ProblemError(NOT_FOUND, f"Branch {branch!r} not found or not materialized.")
|
|
147
|
+
schema = load_compiled(compiled.compiled)
|
|
148
|
+
kind = _kind(schema, body.kind)
|
|
149
|
+
at = store.clock()
|
|
150
|
+
if body.key is not None:
|
|
151
|
+
attribute_ids = (
|
|
152
|
+
None
|
|
153
|
+
if body.attributes is None
|
|
154
|
+
else [
|
|
155
|
+
_vault_attribute(kind, name, f"/body/attributes/{index}")
|
|
156
|
+
for index, name in enumerate(body.attributes)
|
|
157
|
+
]
|
|
158
|
+
)
|
|
159
|
+
erasure = await erase_record(
|
|
160
|
+
session,
|
|
161
|
+
domain_id,
|
|
162
|
+
kind.id,
|
|
163
|
+
body.key,
|
|
164
|
+
attribute_ids,
|
|
165
|
+
keys=store.runtime.vault,
|
|
166
|
+
at=at,
|
|
167
|
+
lock_timeout_ms=store.runtime.vault_settings.erase_lock_timeout_ms,
|
|
168
|
+
)
|
|
169
|
+
mode: Literal["record", "value"] = "record"
|
|
170
|
+
digests = await store.runtime.digests.ready(session)
|
|
171
|
+
key_digest = digests.digest(domain_id, KEY_DIGEST, f"{kind.id}/{body.key}".encode())
|
|
172
|
+
target = f"{kind.api_name}/#{key_digest[:16]}"
|
|
173
|
+
details: dict[str, Any] = {"kind": kind.id, "key_digest": key_digest}
|
|
174
|
+
if body.attributes is not None:
|
|
175
|
+
details["attributes"] = attribute_ids
|
|
176
|
+
else:
|
|
177
|
+
assert body.attribute is not None
|
|
178
|
+
attribute_id = _vault_attribute(kind, body.attribute, "/body/attribute")
|
|
179
|
+
keys = store.runtime.vault
|
|
180
|
+
if keys is None:
|
|
181
|
+
raise ProblemError(
|
|
182
|
+
CONFLICT, "Erasure by value needs vault.token_key and vault.encryption_key."
|
|
183
|
+
)
|
|
184
|
+
try:
|
|
185
|
+
canonical = encode_value(kind.attributes[attribute_id].type, body.value)
|
|
186
|
+
except EncodingError as error:
|
|
187
|
+
raise invalid("/body/value", str(error), "value_invalid") from None
|
|
188
|
+
erasure = await erase_matching(
|
|
189
|
+
session,
|
|
190
|
+
domain_id,
|
|
191
|
+
kind.id,
|
|
192
|
+
attribute_id,
|
|
193
|
+
canonical,
|
|
194
|
+
keys,
|
|
195
|
+
at=at,
|
|
196
|
+
lock_timeout_ms=store.runtime.vault_settings.erase_lock_timeout_ms,
|
|
197
|
+
)
|
|
198
|
+
mode = "value"
|
|
199
|
+
target = f"{kind.api_name}.{body.attribute}"
|
|
200
|
+
details = {"kind": kind.id, "attribute": attribute_id}
|
|
201
|
+
details["erased"] = erasure.erased
|
|
202
|
+
if erasure.unopenable:
|
|
203
|
+
details["unopenable"] = erasure.unopenable
|
|
204
|
+
if erasure.linked:
|
|
205
|
+
# Flagged for an administrator: the subject's keys were changed unusually often.
|
|
206
|
+
details["linked_keys"] = erasure.keys
|
|
207
|
+
log.warning(
|
|
208
|
+
"erasure followed many key changes",
|
|
209
|
+
extra={"domain_id": domain_id, "kind": kind.id, "linked_keys": erasure.keys},
|
|
210
|
+
)
|
|
211
|
+
await AuditRepository(session).append(
|
|
212
|
+
NewAuditEntry(
|
|
213
|
+
actor=access.principal.id,
|
|
214
|
+
action=ERASE_ACTION,
|
|
215
|
+
domain_id=domain_id,
|
|
216
|
+
target=target,
|
|
217
|
+
details={"mode": mode, **details},
|
|
218
|
+
at=at,
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
except VaultLockTimeoutError as error:
|
|
222
|
+
raise ProblemError(
|
|
223
|
+
UNAVAILABLE,
|
|
224
|
+
f"{error}; retry later.",
|
|
225
|
+
headers={"Retry-After": "5"},
|
|
226
|
+
) from None
|
|
227
|
+
# The entry above is the audit record of the erasure; the request's own entry would repeat it.
|
|
228
|
+
suppress(request)
|
|
229
|
+
return EraseOut(
|
|
230
|
+
domain=domain_id,
|
|
231
|
+
kind=kind.api_name,
|
|
232
|
+
mode=mode,
|
|
233
|
+
erased=erasure.erased,
|
|
234
|
+
unopenable=erasure.unopenable,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
class VaultKeysOut(BaseModel):
|
|
239
|
+
"""Which keys the vault rows use."""
|
|
240
|
+
|
|
241
|
+
key_id: str = Field(description="Encryption key new values are sealed under.")
|
|
242
|
+
token_key_id: str = Field(description="Token key new tokens and fingerprints use.")
|
|
243
|
+
fingerprint_key: str = Field(description="Current fingerprint key: token key id/scheme.")
|
|
244
|
+
encryption_keys: dict[str, int] = Field(description="Live rows by encryption key id.")
|
|
245
|
+
fingerprint_keys: dict[str, int] = Field(
|
|
246
|
+
description="Live rows by fingerprint key (`legacy`: written before rotation)."
|
|
247
|
+
)
|
|
248
|
+
tombstone_keys: dict[str, int] = Field(
|
|
249
|
+
description="Fingerprints of erased values by fingerprint key."
|
|
250
|
+
)
|
|
251
|
+
outdated: int = Field(description="Live rows the rotation job still has to make again.")
|
|
252
|
+
removable_keys: list[str] = Field(
|
|
253
|
+
description="Ids in vault.previous_keys that no row uses: they can be removed."
|
|
254
|
+
)
|
|
255
|
+
removable_token_keys: list[str] = Field(
|
|
256
|
+
description="Ids in vault.previous_token_keys that no row or tombstone uses."
|
|
257
|
+
)
|
|
258
|
+
digests: str = Field(
|
|
259
|
+
description="Key of the keyed digests: vault.digest_key, vault.token_key or "
|
|
260
|
+
"installation (the database's own secret)."
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class RotationQueued(BaseModel):
|
|
265
|
+
job_id: int = Field(description="Id of the queued (or already pending) rotation job.")
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
@router.get(
|
|
269
|
+
"/vault/keys",
|
|
270
|
+
summary="Show which keys the vault rows use",
|
|
271
|
+
response_model=VaultKeysOut,
|
|
272
|
+
responses=_KEY_ERRORS,
|
|
273
|
+
)
|
|
274
|
+
async def vault_keys(access: Manage, store: StoreDep) -> VaultKeysOut:
|
|
275
|
+
"""Rows by encryption key and fingerprint key, and the earlier keys that can be removed."""
|
|
276
|
+
keys = store.runtime.vault
|
|
277
|
+
if keys is None:
|
|
278
|
+
raise ProblemError(CONFLICT, "The vault is not configured.")
|
|
279
|
+
settings = store.runtime.vault_settings
|
|
280
|
+
async with store.sessions() as session, session.begin():
|
|
281
|
+
found = await usage(session)
|
|
282
|
+
token_ids_in_use = {k.split("/", 1)[0] for k in found.fingerprint_keys} | {
|
|
283
|
+
k.split("/", 1)[0] for k in found.tombstone_keys
|
|
284
|
+
}
|
|
285
|
+
return VaultKeysOut(
|
|
286
|
+
key_id=keys.key_id,
|
|
287
|
+
token_key_id=keys.token_key_id,
|
|
288
|
+
fingerprint_key=keys.fingerprint_key,
|
|
289
|
+
encryption_keys=found.encryption_keys,
|
|
290
|
+
fingerprint_keys=found.fingerprint_keys,
|
|
291
|
+
tombstone_keys=found.tombstone_keys,
|
|
292
|
+
outdated=found.outdated(keys),
|
|
293
|
+
removable_keys=sorted(k for k in settings.previous_keys if k not in found.encryption_keys),
|
|
294
|
+
removable_token_keys=sorted(
|
|
295
|
+
k
|
|
296
|
+
for k in settings.previous_token_keys
|
|
297
|
+
if k not in token_ids_in_use and "legacy" not in token_ids_in_use
|
|
298
|
+
),
|
|
299
|
+
digests=store.runtime.digests.source,
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
@router.post(
|
|
304
|
+
"/vault/rotate",
|
|
305
|
+
status_code=202,
|
|
306
|
+
summary="Queue a vault key rotation",
|
|
307
|
+
response_model=RotationQueued,
|
|
308
|
+
responses=_KEY_ERRORS,
|
|
309
|
+
)
|
|
310
|
+
async def rotate_vault(access: Manage, store: StoreDep, request: Request) -> RotationQueued:
|
|
311
|
+
"""Queue the job that makes the rows stored under earlier keys again under the current
|
|
312
|
+
ones."""
|
|
313
|
+
if store.runtime.vault is None:
|
|
314
|
+
raise ProblemError(CONFLICT, "The vault is not configured.")
|
|
315
|
+
async with store.sessions() as session, session.begin():
|
|
316
|
+
job_id = await enqueue_job(session, VAULT_ROTATE_JOB, payload={}, unique=True)
|
|
317
|
+
annotate(request, action=ROTATE_ACTION, job=job_id)
|
|
318
|
+
return RotationQueued(job_id=job_id)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _kind(schema: Schema, name: str) -> Kind:
|
|
322
|
+
kind = schema.kind_by_api_name(name)
|
|
323
|
+
if kind is None:
|
|
324
|
+
raise ProblemError(NOT_FOUND, f"Kind {name} not found.")
|
|
325
|
+
return kind
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _vault_attribute(kind: Kind, name: str, pointer: str) -> str:
|
|
329
|
+
attribute = kind.attribute_by_api_name(name)
|
|
330
|
+
if attribute is None or not attribute.in_vault:
|
|
331
|
+
raise invalid(pointer, f"{name} is not a vault Attribute of {kind.api_name}", "not_vault")
|
|
332
|
+
return attribute.id
|
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
"""Webhooks of a Domain (design 10.5): create, list, update clearances, delete.
|
|
2
|
+
|
|
3
|
+
Managing webhooks needs `domain.settings` (the `admin` role). A new webhook's secret is
|
|
4
|
+
returned once; only its digest is stored (`lorebase_store.webhooks.signing`), sealed under
|
|
5
|
+
`webhooks.encryption_key` (`lorebase_store.webhooks.keys`). Without that key a webhook is
|
|
6
|
+
created only when `webhooks.allow_unsealed_keys` is set (local development). Every change is
|
|
7
|
+
written to the audit log (`webhook.create`, `webhook.update`, `webhook.delete`) without the
|
|
8
|
+
secret or the full URL. Deliveries are made by `lorebase_store.webhooks.deliver_pending`, run by
|
|
9
|
+
a worker.
|
|
10
|
+
|
|
11
|
+
A webhook subscribes to `revision.committed`, to `operation.<side effect>` events or to every
|
|
12
|
+
Operation event (`operation.*`). Sensitive values in Operation events reach it masked unless it
|
|
13
|
+
holds `clearances` for their tags; granting one needs the matching clearance permission
|
|
14
|
+
(`pii.read`, `confidential.read`, `restricted.read`) on the whole Domain, so an administrator
|
|
15
|
+
cannot route values to a webhook that they could not read themselves. `PATCH` replaces the
|
|
16
|
+
clearances: the permission is needed only for those it adds, so anyone managing webhooks can
|
|
17
|
+
narrow one.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
from collections.abc import Iterable
|
|
23
|
+
from datetime import datetime
|
|
24
|
+
from typing import Annotated, Any, Literal, cast
|
|
25
|
+
from urllib.parse import urlsplit
|
|
26
|
+
|
|
27
|
+
from fastapi import APIRouter, Depends, Request, Response
|
|
28
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
29
|
+
|
|
30
|
+
from lorebase_core.model import Sensitivity
|
|
31
|
+
from lorebase_keeper.auth.audit import annotate
|
|
32
|
+
from lorebase_keeper.auth.deps import Access, AuthDep, require
|
|
33
|
+
from lorebase_keeper.auth.permissions import Permission
|
|
34
|
+
from lorebase_keeper.context import StoreDep, require_domain
|
|
35
|
+
from lorebase_keeper.errors import (
|
|
36
|
+
CONFLICT,
|
|
37
|
+
FORBIDDEN,
|
|
38
|
+
NOT_FOUND,
|
|
39
|
+
Problem,
|
|
40
|
+
ProblemError,
|
|
41
|
+
ValidationProblem,
|
|
42
|
+
)
|
|
43
|
+
from lorebase_keeper.reading.problems import invalid
|
|
44
|
+
from lorebase_keeper.reading.sensitivity import CLEARANCES, Clearance, clearance_of
|
|
45
|
+
from lorebase_platform.settings import Settings
|
|
46
|
+
from lorebase_store.outbox_events import EVENT_TYPES, event_type_problem
|
|
47
|
+
from lorebase_store.webhooks import (
|
|
48
|
+
WebhookRepository,
|
|
49
|
+
WebhookRow,
|
|
50
|
+
WebhookSettings,
|
|
51
|
+
new_secret,
|
|
52
|
+
signing_key,
|
|
53
|
+
webhook_settings,
|
|
54
|
+
)
|
|
55
|
+
from lorebase_store.webhooks.delivery import scheme_problem
|
|
56
|
+
from lorebase_store.webhooks.keys import SigningKeys
|
|
57
|
+
|
|
58
|
+
router = APIRouter(tags=["webhooks"])
|
|
59
|
+
|
|
60
|
+
BASE = "/domains/{domain_id}/webhooks"
|
|
61
|
+
_ERRORS: dict[int | str, dict[str, Any]] = {
|
|
62
|
+
403: {"model": Problem, "description": "The caller lacks domain.settings."},
|
|
63
|
+
404: {"model": Problem, "description": "Domain or webhook not found."},
|
|
64
|
+
422: {"model": ValidationProblem, "description": "Invalid request."},
|
|
65
|
+
}
|
|
66
|
+
_CLEARANCE_ERRORS: dict[int | str, dict[str, Any]] = {
|
|
67
|
+
**_ERRORS,
|
|
68
|
+
403: {
|
|
69
|
+
"model": Problem,
|
|
70
|
+
"description": "The caller lacks domain.settings, or a clearance it gives the webhook.",
|
|
71
|
+
},
|
|
72
|
+
}
|
|
73
|
+
_CREATE_ERRORS: dict[int | str, dict[str, Any]] = {
|
|
74
|
+
**_CLEARANCE_ERRORS,
|
|
75
|
+
409: {
|
|
76
|
+
"model": Problem,
|
|
77
|
+
"description": "The Domain has the most webhooks allowed, or webhooks.encryption_key "
|
|
78
|
+
"is not configured.",
|
|
79
|
+
},
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
Manage = Annotated[Access, Depends(require(Permission.DOMAIN_SETTINGS))]
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def get_webhook_settings(request: Request) -> WebhookSettings:
|
|
86
|
+
return webhook_settings(cast(Settings, request.app.state.settings))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
WebhookSettingsDep = Annotated[WebhookSettings, Depends(get_webhook_settings)]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
class CreateWebhook(BaseModel):
|
|
93
|
+
model_config = ConfigDict(extra="forbid")
|
|
94
|
+
|
|
95
|
+
url: str = Field(min_length=1, max_length=2048, description="Target of the POST requests.")
|
|
96
|
+
events: list[str] | None = Field(
|
|
97
|
+
default=None,
|
|
98
|
+
min_length=1,
|
|
99
|
+
description="Event types delivered: revision.committed, operation.<side effect>, or "
|
|
100
|
+
"operation.* for every Operation event; default: revision.committed and operation.*.",
|
|
101
|
+
)
|
|
102
|
+
branches: list[str] | None = Field(
|
|
103
|
+
default=None, min_length=1, description="Only events of these branches; default: all."
|
|
104
|
+
)
|
|
105
|
+
description: str | None = Field(default=None, max_length=500)
|
|
106
|
+
clearances: list[Literal["pii", "confidential", "restricted"]] = Field(
|
|
107
|
+
default_factory=list,
|
|
108
|
+
description="Sensitivity tags whose values the webhook receives in clear; others are "
|
|
109
|
+
"masked. Each needs the matching clearance permission on the Domain.",
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
class UpdateWebhook(BaseModel):
|
|
114
|
+
model_config = ConfigDict(extra="forbid")
|
|
115
|
+
|
|
116
|
+
clearances: list[Literal["pii", "confidential", "restricted"]] = Field(
|
|
117
|
+
description="The new clearances, replacing the current ones. Each one added needs the "
|
|
118
|
+
"matching clearance permission on the Domain; removing needs only domain.settings."
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class WebhookOut(BaseModel):
|
|
123
|
+
id: str
|
|
124
|
+
url: str
|
|
125
|
+
description: str | None = None
|
|
126
|
+
events: list[str]
|
|
127
|
+
branches: list[str] | None = None
|
|
128
|
+
clearances: list[str] = Field(default_factory=list)
|
|
129
|
+
created_by: str
|
|
130
|
+
created_at: datetime
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
class WebhookCreated(WebhookOut):
|
|
134
|
+
"""A new webhook; `secret` is shown this once."""
|
|
135
|
+
|
|
136
|
+
secret: str = Field(
|
|
137
|
+
description="Signing secret. Requests carry `t=<unix>,v1=<hex HMAC-SHA256>` computed "
|
|
138
|
+
"over `<t>.<body>` with the key SHA-256(secret)."
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class WebhookList(BaseModel):
|
|
143
|
+
items: list[WebhookOut]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _out(row: WebhookRow) -> dict[str, Any]:
|
|
147
|
+
return {
|
|
148
|
+
"id": row.id,
|
|
149
|
+
"url": row.url,
|
|
150
|
+
"description": row.description,
|
|
151
|
+
"events": list(row.event_types),
|
|
152
|
+
"branches": None if row.branches is None else list(row.branches),
|
|
153
|
+
"clearances": list(row.clearances),
|
|
154
|
+
"created_by": row.created_by,
|
|
155
|
+
"created_at": row.created_at,
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _require_sealing(settings: WebhookSettings) -> None:
|
|
160
|
+
"""409 unless signing keys are sealed, or storing them usable is explicitly allowed."""
|
|
161
|
+
if settings.encryption_key is None and not settings.allow_unsealed_keys:
|
|
162
|
+
raise ProblemError(
|
|
163
|
+
CONFLICT,
|
|
164
|
+
"Creating a webhook needs webhooks.encryption_key, which seals its signing key "
|
|
165
|
+
"(webhooks.allow_unsealed_keys allows it without, for local development only).",
|
|
166
|
+
extensions={"code": "encryption_key_required", "setting": "webhooks.encryption_key"},
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _require_clearances(held: Clearance, tags: Iterable[str]) -> None:
|
|
171
|
+
"""403 unless `held` (the caller's clearance) covers, Domain-wide, every tag."""
|
|
172
|
+
wanted = set(tags)
|
|
173
|
+
missing = sorted(
|
|
174
|
+
CLEARANCES[Sensitivity(tag)].value
|
|
175
|
+
for tag in wanted
|
|
176
|
+
if Sensitivity(tag) not in held.everywhere
|
|
177
|
+
)
|
|
178
|
+
if missing:
|
|
179
|
+
raise ProblemError(
|
|
180
|
+
FORBIDDEN,
|
|
181
|
+
f"Clearing a webhook for sensitive values needs {' and '.join(missing)} on the Domain.",
|
|
182
|
+
extensions={"code": "clearance_required", "permission": missing[0]},
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
@router.post(BASE, status_code=201, summary="Create a webhook", responses=_CREATE_ERRORS)
|
|
187
|
+
async def create_webhook(
|
|
188
|
+
domain_id: str,
|
|
189
|
+
body: CreateWebhook,
|
|
190
|
+
access: Manage,
|
|
191
|
+
store: StoreDep,
|
|
192
|
+
settings: WebhookSettingsDep,
|
|
193
|
+
request: Request,
|
|
194
|
+
auth: AuthDep,
|
|
195
|
+
) -> WebhookCreated:
|
|
196
|
+
"""Register an endpoint for the Domain's events; the secret is returned once."""
|
|
197
|
+
_require_sealing(settings)
|
|
198
|
+
problem = scheme_problem(body.url, settings)
|
|
199
|
+
if problem is not None:
|
|
200
|
+
raise invalid("/body/url", problem, "invalid_target")
|
|
201
|
+
events = body.events or list(EVENT_TYPES)
|
|
202
|
+
for index, event in enumerate(events):
|
|
203
|
+
unknown = event_type_problem(event)
|
|
204
|
+
if unknown is not None:
|
|
205
|
+
raise invalid(f"/body/events/{index}", unknown, "unknown")
|
|
206
|
+
if body.clearances:
|
|
207
|
+
held = await clearance_of(auth.authorizer, access.principal, domain_id, None)
|
|
208
|
+
_require_clearances(held, body.clearances)
|
|
209
|
+
secret = new_secret(settings.secret_bytes)
|
|
210
|
+
async with store.sessions() as session, session.begin():
|
|
211
|
+
await require_domain(session, domain_id)
|
|
212
|
+
webhooks = WebhookRepository(session, domain_id)
|
|
213
|
+
if await webhooks.count() >= settings.max_per_domain:
|
|
214
|
+
raise ProblemError(
|
|
215
|
+
CONFLICT, f"The Domain already has {settings.max_per_domain} webhooks."
|
|
216
|
+
)
|
|
217
|
+
row = await webhooks.add(
|
|
218
|
+
url=body.url,
|
|
219
|
+
description=body.description,
|
|
220
|
+
event_types=sorted(set(events)),
|
|
221
|
+
branches=None if body.branches is None else sorted(set(body.branches)),
|
|
222
|
+
signing_key=SigningKeys.from_settings(settings).store(signing_key(secret)),
|
|
223
|
+
created_by=access.principal.id,
|
|
224
|
+
clearances=body.clearances,
|
|
225
|
+
)
|
|
226
|
+
annotate(
|
|
227
|
+
request,
|
|
228
|
+
action="webhook.create",
|
|
229
|
+
target=row.id,
|
|
230
|
+
host=urlsplit(row.url).hostname,
|
|
231
|
+
events=list(row.event_types),
|
|
232
|
+
branches=None if row.branches is None else list(row.branches),
|
|
233
|
+
clearances=list(row.clearances),
|
|
234
|
+
)
|
|
235
|
+
return WebhookCreated(**_out(row), secret=secret)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
@router.get(BASE, summary="List webhooks", response_model=WebhookList, responses=_ERRORS)
|
|
239
|
+
async def list_webhooks(domain_id: str, access: Manage, store: StoreDep) -> WebhookList:
|
|
240
|
+
"""Webhooks of the Domain, oldest first (secrets are never shown again)."""
|
|
241
|
+
async with store.sessions() as session, session.begin():
|
|
242
|
+
await require_domain(session, domain_id)
|
|
243
|
+
rows = await WebhookRepository(session, domain_id).all()
|
|
244
|
+
return WebhookList(items=[WebhookOut(**_out(row)) for row in rows])
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
@router.patch(
|
|
248
|
+
f"{BASE}/{{webhook_id}}",
|
|
249
|
+
summary="Change the clearances of a webhook",
|
|
250
|
+
response_model=WebhookOut,
|
|
251
|
+
responses=_CLEARANCE_ERRORS,
|
|
252
|
+
)
|
|
253
|
+
async def update_webhook(
|
|
254
|
+
domain_id: str,
|
|
255
|
+
webhook_id: str,
|
|
256
|
+
body: UpdateWebhook,
|
|
257
|
+
access: Manage,
|
|
258
|
+
store: StoreDep,
|
|
259
|
+
request: Request,
|
|
260
|
+
auth: AuthDep,
|
|
261
|
+
) -> WebhookOut:
|
|
262
|
+
"""Replace the sensitivity clearances of a webhook; only added ones need their permission."""
|
|
263
|
+
# Read before the row lock: the authorizer may be slow, the lock is held only to compare.
|
|
264
|
+
held = await clearance_of(auth.authorizer, access.principal, domain_id, None)
|
|
265
|
+
async with store.sessions() as session, session.begin():
|
|
266
|
+
await require_domain(session, domain_id)
|
|
267
|
+
webhooks = WebhookRepository(session, domain_id)
|
|
268
|
+
current = await webhooks.get(webhook_id, for_update=True)
|
|
269
|
+
if current is None:
|
|
270
|
+
raise ProblemError(NOT_FOUND, f"Webhook {webhook_id!r} not found.")
|
|
271
|
+
_require_clearances(held, set(body.clearances) - set(current.clearances))
|
|
272
|
+
row = await webhooks.set_clearances(webhook_id, body.clearances)
|
|
273
|
+
if row is None:
|
|
274
|
+
raise ProblemError(NOT_FOUND, f"Webhook {webhook_id!r} not found.")
|
|
275
|
+
annotate(
|
|
276
|
+
request,
|
|
277
|
+
action="webhook.update",
|
|
278
|
+
target=row.id,
|
|
279
|
+
clearances_before=list(current.clearances),
|
|
280
|
+
clearances=list(row.clearances),
|
|
281
|
+
)
|
|
282
|
+
return WebhookOut(**_out(row))
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
@router.delete(
|
|
286
|
+
f"{BASE}/{{webhook_id}}", status_code=204, summary="Delete a webhook", responses=_ERRORS
|
|
287
|
+
)
|
|
288
|
+
async def delete_webhook(
|
|
289
|
+
domain_id: str, webhook_id: str, access: Manage, store: StoreDep, request: Request
|
|
290
|
+
) -> Response:
|
|
291
|
+
"""Remove a webhook and its pending deliveries."""
|
|
292
|
+
async with store.sessions() as session, session.begin():
|
|
293
|
+
await require_domain(session, domain_id)
|
|
294
|
+
deleted = await WebhookRepository(session, domain_id).delete(webhook_id)
|
|
295
|
+
if not deleted:
|
|
296
|
+
raise ProblemError(NOT_FOUND, f"Webhook {webhook_id!r} not found.")
|
|
297
|
+
annotate(request, action="webhook.delete", target=webhook_id)
|
|
298
|
+
return Response(status_code=204)
|