lorebase-store 0.2.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. lorebase_store/__init__.py +1 -0
  2. lorebase_store/_migrations/env.py +106 -0
  3. lorebase_store/_migrations/script.py.mako +27 -0
  4. lorebase_store/_migrations/versions/0001_initial_schema.py +1071 -0
  5. lorebase_store/_migrations/versions/0002_merge_attempt_details.py +74 -0
  6. lorebase_store/_migrations/versions/0003_webhooks.py +127 -0
  7. lorebase_store/_migrations/versions/0004_defer_revision_references.py +38 -0
  8. lorebase_store/_migrations/versions/0005_security_definer_reads.py +396 -0
  9. lorebase_store/_migrations/versions/0006_api_key_usage.py +48 -0
  10. lorebase_store/_migrations/versions/0007_document_extraction.py +154 -0
  11. lorebase_store/_migrations/versions/0008_definer_argument_checks.py +178 -0
  12. lorebase_store/_migrations/versions/0009_api_key_networks.py +33 -0
  13. lorebase_store/_migrations/versions/0010_overlay_branches.py +582 -0
  14. lorebase_store/_migrations/versions/0011_branch_scoped_search_index.py +229 -0
  15. lorebase_store/_migrations/versions/0012_overlay_neighborhood_lookups.py +185 -0
  16. lorebase_store/_migrations/versions/0013_web_sessions.py +72 -0
  17. lorebase_store/_migrations/versions/0014_embedding_jobs.py +70 -0
  18. lorebase_store/_migrations/versions/0015_append_only_audit_log.py +83 -0
  19. lorebase_store/_migrations/versions/0016_sealed_webhook_keys.py +55 -0
  20. lorebase_store/_migrations/versions/0017_background_branch_copies.py +77 -0
  21. lorebase_store/_migrations/versions/0018_search_lexicon.py +477 -0
  22. lorebase_store/_migrations/versions/0019_vault_bindings.py +56 -0
  23. lorebase_store/_migrations/versions/0020_outbox_consumers.py +150 -0
  24. lorebase_store/_migrations/versions/0021_schema_suggestions.py +114 -0
  25. lorebase_store/_migrations/versions/0022_read_audit_and_shared_rate_limits.py +133 -0
  26. lorebase_store/_migrations/versions/0023_agent_trace.py +96 -0
  27. lorebase_store/_migrations/versions/0024_embedding_template_keys.py +205 -0
  28. lorebase_store/_migrations/versions/0025_snapshot_refs.py +35 -0
  29. lorebase_store/_migrations/versions/0026_plain_reverse_link_index.py +112 -0
  30. lorebase_store/_migrations/versions/0027_vault_rotation_and_tombstones.py +145 -0
  31. lorebase_store/_migrations/versions/0028_operation_deferred_events.py +71 -0
  32. lorebase_store/_migrations/versions/0029_agent_trace_sessions.py +108 -0
  33. lorebase_store/_migrations/versions/0030_path_fanout.py +31 -0
  34. lorebase_store/_migrations/versions/0031_branch_builds.py +91 -0
  35. lorebase_store/_migrations/versions/0032_lineage_revisions.py +35 -0
  36. lorebase_store/_migrations/versions/0033_vault_moved_tokens.py +30 -0
  37. lorebase_store/_migrations/versions/0034_inference_endpoint_modes.py +73 -0
  38. lorebase_store/_migrations/versions/0035_search_truncation.py +104 -0
  39. lorebase_store/_migrations/versions/0036_lineage_maps.py +325 -0
  40. lorebase_store/_migrations/versions/0037_bm25_search.py +346 -0
  41. lorebase_store/auth/__init__.py +52 -0
  42. lorebase_store/auth/api_keys.py +161 -0
  43. lorebase_store/auth/audit.py +382 -0
  44. lorebase_store/auth/checkpoints.py +185 -0
  45. lorebase_store/auth/grants.py +161 -0
  46. lorebase_store/auth/principals.py +158 -0
  47. lorebase_store/auth/scope.py +70 -0
  48. lorebase_store/auth/sessions.py +153 -0
  49. lorebase_store/auth/unit.py +53 -0
  50. lorebase_store/auth/usage.py +70 -0
  51. lorebase_store/blobs.py +73 -0
  52. lorebase_store/bm25.py +346 -0
  53. lorebase_store/bulk.py +147 -0
  54. lorebase_store/bundle.py +1830 -0
  55. lorebase_store/changefeed.py +261 -0
  56. lorebase_store/commit/__init__.py +42 -0
  57. lorebase_store/commit/changes.py +67 -0
  58. lorebase_store/commit/errors.py +60 -0
  59. lorebase_store/commit/migrations.py +335 -0
  60. lorebase_store/commit/pipeline.py +754 -0
  61. lorebase_store/commit/schema.py +123 -0
  62. lorebase_store/db.py +58 -0
  63. lorebase_store/definer.py +50 -0
  64. lorebase_store/digests.py +99 -0
  65. lorebase_store/embeddings.py +444 -0
  66. lorebase_store/endpoint_modes.py +59 -0
  67. lorebase_store/extraction/__init__.py +61 -0
  68. lorebase_store/extraction/items.py +187 -0
  69. lorebase_store/extraction/materialize.py +364 -0
  70. lorebase_store/extraction/proposal.py +65 -0
  71. lorebase_store/extraction/review.py +168 -0
  72. lorebase_store/gc.py +970 -0
  73. lorebase_store/indexes.py +1033 -0
  74. lorebase_store/jobs.py +651 -0
  75. lorebase_store/keyring.py +72 -0
  76. lorebase_store/layout.py +335 -0
  77. lorebase_store/materialize/__init__.py +103 -0
  78. lorebase_store/materialize/apply.py +436 -0
  79. lorebase_store/materialize/background.py +1184 -0
  80. lorebase_store/materialize/branch.py +268 -0
  81. lorebase_store/materialize/build.py +536 -0
  82. lorebase_store/materialize/derived.py +540 -0
  83. lorebase_store/materialize/lineage_backfill.py +502 -0
  84. lorebase_store/materialize/links.py +266 -0
  85. lorebase_store/materialize/overlay.py +292 -0
  86. lorebase_store/materialize/paths.py +315 -0
  87. lorebase_store/materialize/rebuild.py +326 -0
  88. lorebase_store/materialize/rows.py +153 -0
  89. lorebase_store/materialize/sensitivity.py +66 -0
  90. lorebase_store/materialize/snapshot.py +205 -0
  91. lorebase_store/materialize/stale.py +444 -0
  92. lorebase_store/materialize/verify.py +321 -0
  93. lorebase_store/merging/__init__.py +39 -0
  94. lorebase_store/merging/attempts.py +334 -0
  95. lorebase_store/merging/lineage.py +123 -0
  96. lorebase_store/merging/service.py +652 -0
  97. lorebase_store/migrations.py +133 -0
  98. lorebase_store/models/__init__.py +136 -0
  99. lorebase_store/models/base.py +64 -0
  100. lorebase_store/models/enums.py +132 -0
  101. lorebase_store/models/governance.py +268 -0
  102. lorebase_store/models/ingestion.py +288 -0
  103. lorebase_store/models/materialized.py +277 -0
  104. lorebase_store/models/proposals.py +195 -0
  105. lorebase_store/models/queues.py +87 -0
  106. lorebase_store/models/tracing.py +116 -0
  107. lorebase_store/models/versioning.py +222 -0
  108. lorebase_store/models/webhooks.py +96 -0
  109. lorebase_store/nodestore.py +384 -0
  110. lorebase_store/objectstore.py +162 -0
  111. lorebase_store/outbox_events.py +166 -0
  112. lorebase_store/profiling/__init__.py +58 -0
  113. lorebase_store/profiling/delimited.py +109 -0
  114. lorebase_store/profiling/postgres.py +211 -0
  115. lorebase_store/profiling/record_files.py +127 -0
  116. lorebase_store/profiling/records.py +92 -0
  117. lorebase_store/profiling/sql_databases.py +298 -0
  118. lorebase_store/ratelimit.py +129 -0
  119. lorebase_store/reads.py +1491 -0
  120. lorebase_store/replicas.py +434 -0
  121. lorebase_store/repositories/__init__.py +152 -0
  122. lorebase_store/repositories/changesets.py +59 -0
  123. lorebase_store/repositories/deferred_events.py +142 -0
  124. lorebase_store/repositories/documents.py +305 -0
  125. lorebase_store/repositories/domains.py +97 -0
  126. lorebase_store/repositories/events.py +205 -0
  127. lorebase_store/repositories/index_view.py +489 -0
  128. lorebase_store/repositories/invocations.py +68 -0
  129. lorebase_store/repositories/materializations.py +276 -0
  130. lorebase_store/repositories/materialized.py +1129 -0
  131. lorebase_store/repositories/merges.py +417 -0
  132. lorebase_store/repositories/proposal_items.py +199 -0
  133. lorebase_store/repositories/proposals.py +184 -0
  134. lorebase_store/repositories/provenance.py +77 -0
  135. lorebase_store/repositories/refs.py +452 -0
  136. lorebase_store/repositories/revisions.py +273 -0
  137. lorebase_store/repositories/scope.py +33 -0
  138. lorebase_store/repositories/source_runs.py +123 -0
  139. lorebase_store/repositories/suggestions.py +638 -0
  140. lorebase_store/runtime.py +87 -0
  141. lorebase_store/schema_cache.py +50 -0
  142. lorebase_store/settings.py +879 -0
  143. lorebase_store/storage.py +1478 -0
  144. lorebase_store/telemetry.py +142 -0
  145. lorebase_store/testing.py +108 -0
  146. lorebase_store/tracing.py +505 -0
  147. lorebase_store/traverse.py +578 -0
  148. lorebase_store/vault/__init__.py +50 -0
  149. lorebase_store/vault/keys.py +216 -0
  150. lorebase_store/vault/quotes.py +133 -0
  151. lorebase_store/vault/rotation.py +209 -0
  152. lorebase_store/vault/settings.py +152 -0
  153. lorebase_store/vault/store.py +987 -0
  154. lorebase_store/vault/transfer.py +558 -0
  155. lorebase_store/webhooks/__init__.py +24 -0
  156. lorebase_store/webhooks/delivery.py +637 -0
  157. lorebase_store/webhooks/keys.py +126 -0
  158. lorebase_store/webhooks/repository.py +142 -0
  159. lorebase_store/webhooks/settings.py +138 -0
  160. lorebase_store/webhooks/signing.py +50 -0
  161. lorebase_store-0.2.3.dist-info/METADATA +13 -0
  162. lorebase_store-0.2.3.dist-info/RECORD +163 -0
  163. lorebase_store-0.2.3.dist-info/WHEEL +4 -0
@@ -0,0 +1,152 @@
1
+ """The `vault` settings section: keys of the personal data vault (design 13.5)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from enum import StrEnum
6
+ from typing import Self
7
+
8
+ from pydantic import Field, model_validator
9
+
10
+ from lorebase_platform.secrets import SecretValue
11
+ from lorebase_platform.settings import SectionModel, Settings, register_section
12
+
13
+ SECTION = "vault"
14
+ MIN_KEY_LENGTH = 32
15
+
16
+
17
+ class ErasedValues(StrEnum):
18
+ """What a write of a value erased before does (`vault.erased_values`)."""
19
+
20
+ ERASE = "erase"
21
+ """Store it erased: the Record holds an erased reference, as after the erasure."""
22
+ REFUSE = "refuse"
23
+ """Refuse the commit (`value.vault_erased`)."""
24
+ ALLOW = "allow"
25
+ """Store it again under a new token, readable until erased again."""
26
+
27
+
28
+ class VaultSettings(SectionModel):
29
+ """Keys of the vault. Keeper, Scribe and every process that commits need the same values.
30
+
31
+ Without keys, commits that store a value of a vault Attribute are refused
32
+ (`value.vault_unavailable`) and reads show vaulted values masked.
33
+ """
34
+
35
+ token_key: SecretValue | None = Field(
36
+ default=None,
37
+ description="Key (at least 32 characters) of the vault tokens that Record objects hold "
38
+ "and of the value fingerprints that erasure by value matches, e.g. "
39
+ "secret://env/LOREBASE_VAULT_TOKEN_KEY. To rotate it, give the new key a new "
40
+ "token_key_id and move the old one to previous_token_keys: tokens already stored keep "
41
+ "working, new values get tokens under the new key, and `lorebase admin vault rotate` "
42
+ "moves the fingerprints to it.",
43
+ )
44
+ token_key_id: str = Field(
45
+ default="1",
46
+ min_length=1,
47
+ max_length=64,
48
+ description="Id of token_key, stored with each fingerprint made under it.",
49
+ )
50
+ previous_token_keys: dict[str, SecretValue] = Field(
51
+ default_factory=dict,
52
+ description="Earlier token keys by id: a value written again finds the token it had "
53
+ "under them, and erasure by value matches fingerprints made under them. Remove one "
54
+ "when `lorebase admin vault status` shows no fingerprint and no erasure record under "
55
+ "its id.",
56
+ )
57
+ digest_key: SecretValue | None = Field(
58
+ default=None,
59
+ description="Key (at least 32 characters) of the keyed digests Lorebase stores of "
60
+ "inputs that may hold personal data (Operation parameters, the Record keys of erasure "
61
+ "audit entries). Default: derived from token_key when the vault is configured, else "
62
+ "the random installation secret revision 0027 stored in the database. Set it to keep "
63
+ "the digests unconfirmable for whoever reads the database.",
64
+ )
65
+ erased_values: ErasedValues = Field(
66
+ default=ErasedValues.ERASE,
67
+ description="A later write of an erased value (a Source sync, say): erase stores it "
68
+ "erased, refuse fails the commit, allow stores it again until erased again.",
69
+ )
70
+ tombstone_origins: tuple[str, ...] = Field(
71
+ default=("source_sync", "extraction", "import"),
72
+ description="Commit origins whose writes are checked against the fingerprints of erased "
73
+ "values (erased_values). Writes of people and agents are not: they choose the value, so "
74
+ "the outcome would tell them whether a guessed value was erased.",
75
+ )
76
+ lock_timeout_ms: int = Field(
77
+ default=10_000,
78
+ ge=0,
79
+ description="How long a commit storing vault values waits for an erasure in the same "
80
+ "Domain to finish (0: no limit).",
81
+ )
82
+ erase_lock_timeout_ms: int = Field(
83
+ default=30_000,
84
+ ge=0,
85
+ description="How long an erasure waits for the commits storing vault values in the same "
86
+ "Domain to finish (0: no limit); it then answers 503 and can be retried.",
87
+ )
88
+ rekey_batch: int = Field(
89
+ default=500,
90
+ ge=1,
91
+ le=10_000,
92
+ description="Vault rows re-encrypted per transaction by the rotation job.",
93
+ )
94
+ rekey_rows_per_job: int = Field(
95
+ default=50_000,
96
+ ge=1,
97
+ description="Rows one rotation job handles before it queues its continuation.",
98
+ )
99
+ encryption_key: SecretValue | None = Field(
100
+ default=None,
101
+ description="Key (at least 32 characters) that encrypts new vault values (AES-256-GCM), "
102
+ "e.g. secret://env/LOREBASE_VAULT_KEY.",
103
+ )
104
+ key_id: str = Field(
105
+ default="1",
106
+ min_length=1,
107
+ max_length=64,
108
+ description="Id stored with each value encrypted under encryption_key; give the key a "
109
+ "new id when it changes and move the old one to previous_keys.",
110
+ )
111
+ previous_keys: dict[str, SecretValue] = Field(
112
+ default_factory=dict,
113
+ description="Earlier encryption keys by id: values stored under them stay readable. "
114
+ "Remove one only when no vault row has its id.",
115
+ )
116
+
117
+ @model_validator(mode="after")
118
+ def _consistent(self) -> Self:
119
+ keys = [
120
+ self.token_key,
121
+ self.encryption_key,
122
+ self.digest_key,
123
+ *self.previous_keys.values(),
124
+ *self.previous_token_keys.values(),
125
+ ]
126
+ if any(key is not None and len(key.get_secret_value()) < MIN_KEY_LENGTH for key in keys):
127
+ raise ValueError(f"vault keys need at least {MIN_KEY_LENGTH} characters")
128
+ if (self.token_key is None) != (self.encryption_key is None):
129
+ raise ValueError("vault.token_key and vault.encryption_key are set together")
130
+ if self.previous_keys and self.encryption_key is None:
131
+ raise ValueError("vault.previous_keys need an encryption_key")
132
+ if self.key_id in self.previous_keys:
133
+ raise ValueError(f"vault.key_id {self.key_id!r} is also in previous_keys")
134
+ if self.previous_token_keys and self.token_key is None:
135
+ raise ValueError("vault.previous_token_keys need a token_key")
136
+ if self.token_key_id in self.previous_token_keys:
137
+ raise ValueError(
138
+ f"vault.token_key_id {self.token_key_id!r} is also in previous_token_keys"
139
+ )
140
+ return self
141
+
142
+ @property
143
+ def configured(self) -> bool:
144
+ return self.encryption_key is not None
145
+
146
+
147
+ register_section(SECTION, VaultSettings)
148
+
149
+
150
+ def vault_settings(settings: Settings) -> VaultSettings:
151
+ """The validated `vault` section."""
152
+ return settings.section(SECTION, VaultSettings)