lorebase-store 0.2.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lorebase_store/__init__.py +1 -0
- lorebase_store/_migrations/env.py +106 -0
- lorebase_store/_migrations/script.py.mako +27 -0
- lorebase_store/_migrations/versions/0001_initial_schema.py +1071 -0
- lorebase_store/_migrations/versions/0002_merge_attempt_details.py +74 -0
- lorebase_store/_migrations/versions/0003_webhooks.py +127 -0
- lorebase_store/_migrations/versions/0004_defer_revision_references.py +38 -0
- lorebase_store/_migrations/versions/0005_security_definer_reads.py +396 -0
- lorebase_store/_migrations/versions/0006_api_key_usage.py +48 -0
- lorebase_store/_migrations/versions/0007_document_extraction.py +154 -0
- lorebase_store/_migrations/versions/0008_definer_argument_checks.py +178 -0
- lorebase_store/_migrations/versions/0009_api_key_networks.py +33 -0
- lorebase_store/_migrations/versions/0010_overlay_branches.py +582 -0
- lorebase_store/_migrations/versions/0011_branch_scoped_search_index.py +229 -0
- lorebase_store/_migrations/versions/0012_overlay_neighborhood_lookups.py +185 -0
- lorebase_store/_migrations/versions/0013_web_sessions.py +72 -0
- lorebase_store/_migrations/versions/0014_embedding_jobs.py +70 -0
- lorebase_store/_migrations/versions/0015_append_only_audit_log.py +83 -0
- lorebase_store/_migrations/versions/0016_sealed_webhook_keys.py +55 -0
- lorebase_store/_migrations/versions/0017_background_branch_copies.py +77 -0
- lorebase_store/_migrations/versions/0018_search_lexicon.py +477 -0
- lorebase_store/_migrations/versions/0019_vault_bindings.py +56 -0
- lorebase_store/_migrations/versions/0020_outbox_consumers.py +150 -0
- lorebase_store/_migrations/versions/0021_schema_suggestions.py +114 -0
- lorebase_store/_migrations/versions/0022_read_audit_and_shared_rate_limits.py +133 -0
- lorebase_store/_migrations/versions/0023_agent_trace.py +96 -0
- lorebase_store/_migrations/versions/0024_embedding_template_keys.py +205 -0
- lorebase_store/_migrations/versions/0025_snapshot_refs.py +35 -0
- lorebase_store/_migrations/versions/0026_plain_reverse_link_index.py +112 -0
- lorebase_store/_migrations/versions/0027_vault_rotation_and_tombstones.py +145 -0
- lorebase_store/_migrations/versions/0028_operation_deferred_events.py +71 -0
- lorebase_store/_migrations/versions/0029_agent_trace_sessions.py +108 -0
- lorebase_store/_migrations/versions/0030_path_fanout.py +31 -0
- lorebase_store/_migrations/versions/0031_branch_builds.py +91 -0
- lorebase_store/_migrations/versions/0032_lineage_revisions.py +35 -0
- lorebase_store/_migrations/versions/0033_vault_moved_tokens.py +30 -0
- lorebase_store/_migrations/versions/0034_inference_endpoint_modes.py +73 -0
- lorebase_store/_migrations/versions/0035_search_truncation.py +104 -0
- lorebase_store/_migrations/versions/0036_lineage_maps.py +325 -0
- lorebase_store/_migrations/versions/0037_bm25_search.py +346 -0
- lorebase_store/auth/__init__.py +52 -0
- lorebase_store/auth/api_keys.py +161 -0
- lorebase_store/auth/audit.py +382 -0
- lorebase_store/auth/checkpoints.py +185 -0
- lorebase_store/auth/grants.py +161 -0
- lorebase_store/auth/principals.py +158 -0
- lorebase_store/auth/scope.py +70 -0
- lorebase_store/auth/sessions.py +153 -0
- lorebase_store/auth/unit.py +53 -0
- lorebase_store/auth/usage.py +70 -0
- lorebase_store/blobs.py +73 -0
- lorebase_store/bm25.py +346 -0
- lorebase_store/bulk.py +147 -0
- lorebase_store/bundle.py +1830 -0
- lorebase_store/changefeed.py +261 -0
- lorebase_store/commit/__init__.py +42 -0
- lorebase_store/commit/changes.py +67 -0
- lorebase_store/commit/errors.py +60 -0
- lorebase_store/commit/migrations.py +335 -0
- lorebase_store/commit/pipeline.py +754 -0
- lorebase_store/commit/schema.py +123 -0
- lorebase_store/db.py +58 -0
- lorebase_store/definer.py +50 -0
- lorebase_store/digests.py +99 -0
- lorebase_store/embeddings.py +444 -0
- lorebase_store/endpoint_modes.py +59 -0
- lorebase_store/extraction/__init__.py +61 -0
- lorebase_store/extraction/items.py +187 -0
- lorebase_store/extraction/materialize.py +364 -0
- lorebase_store/extraction/proposal.py +65 -0
- lorebase_store/extraction/review.py +168 -0
- lorebase_store/gc.py +970 -0
- lorebase_store/indexes.py +1033 -0
- lorebase_store/jobs.py +651 -0
- lorebase_store/keyring.py +72 -0
- lorebase_store/layout.py +335 -0
- lorebase_store/materialize/__init__.py +103 -0
- lorebase_store/materialize/apply.py +436 -0
- lorebase_store/materialize/background.py +1184 -0
- lorebase_store/materialize/branch.py +268 -0
- lorebase_store/materialize/build.py +536 -0
- lorebase_store/materialize/derived.py +540 -0
- lorebase_store/materialize/lineage_backfill.py +502 -0
- lorebase_store/materialize/links.py +266 -0
- lorebase_store/materialize/overlay.py +292 -0
- lorebase_store/materialize/paths.py +315 -0
- lorebase_store/materialize/rebuild.py +326 -0
- lorebase_store/materialize/rows.py +153 -0
- lorebase_store/materialize/sensitivity.py +66 -0
- lorebase_store/materialize/snapshot.py +205 -0
- lorebase_store/materialize/stale.py +444 -0
- lorebase_store/materialize/verify.py +321 -0
- lorebase_store/merging/__init__.py +39 -0
- lorebase_store/merging/attempts.py +334 -0
- lorebase_store/merging/lineage.py +123 -0
- lorebase_store/merging/service.py +652 -0
- lorebase_store/migrations.py +133 -0
- lorebase_store/models/__init__.py +136 -0
- lorebase_store/models/base.py +64 -0
- lorebase_store/models/enums.py +132 -0
- lorebase_store/models/governance.py +268 -0
- lorebase_store/models/ingestion.py +288 -0
- lorebase_store/models/materialized.py +277 -0
- lorebase_store/models/proposals.py +195 -0
- lorebase_store/models/queues.py +87 -0
- lorebase_store/models/tracing.py +116 -0
- lorebase_store/models/versioning.py +222 -0
- lorebase_store/models/webhooks.py +96 -0
- lorebase_store/nodestore.py +384 -0
- lorebase_store/objectstore.py +162 -0
- lorebase_store/outbox_events.py +166 -0
- lorebase_store/profiling/__init__.py +58 -0
- lorebase_store/profiling/delimited.py +109 -0
- lorebase_store/profiling/postgres.py +211 -0
- lorebase_store/profiling/record_files.py +127 -0
- lorebase_store/profiling/records.py +92 -0
- lorebase_store/profiling/sql_databases.py +298 -0
- lorebase_store/ratelimit.py +129 -0
- lorebase_store/reads.py +1491 -0
- lorebase_store/replicas.py +434 -0
- lorebase_store/repositories/__init__.py +152 -0
- lorebase_store/repositories/changesets.py +59 -0
- lorebase_store/repositories/deferred_events.py +142 -0
- lorebase_store/repositories/documents.py +305 -0
- lorebase_store/repositories/domains.py +97 -0
- lorebase_store/repositories/events.py +205 -0
- lorebase_store/repositories/index_view.py +489 -0
- lorebase_store/repositories/invocations.py +68 -0
- lorebase_store/repositories/materializations.py +276 -0
- lorebase_store/repositories/materialized.py +1129 -0
- lorebase_store/repositories/merges.py +417 -0
- lorebase_store/repositories/proposal_items.py +199 -0
- lorebase_store/repositories/proposals.py +184 -0
- lorebase_store/repositories/provenance.py +77 -0
- lorebase_store/repositories/refs.py +452 -0
- lorebase_store/repositories/revisions.py +273 -0
- lorebase_store/repositories/scope.py +33 -0
- lorebase_store/repositories/source_runs.py +123 -0
- lorebase_store/repositories/suggestions.py +638 -0
- lorebase_store/runtime.py +87 -0
- lorebase_store/schema_cache.py +50 -0
- lorebase_store/settings.py +879 -0
- lorebase_store/storage.py +1478 -0
- lorebase_store/telemetry.py +142 -0
- lorebase_store/testing.py +108 -0
- lorebase_store/tracing.py +505 -0
- lorebase_store/traverse.py +578 -0
- lorebase_store/vault/__init__.py +50 -0
- lorebase_store/vault/keys.py +216 -0
- lorebase_store/vault/quotes.py +133 -0
- lorebase_store/vault/rotation.py +209 -0
- lorebase_store/vault/settings.py +152 -0
- lorebase_store/vault/store.py +987 -0
- lorebase_store/vault/transfer.py +558 -0
- lorebase_store/webhooks/__init__.py +24 -0
- lorebase_store/webhooks/delivery.py +637 -0
- lorebase_store/webhooks/keys.py +126 -0
- lorebase_store/webhooks/repository.py +142 -0
- lorebase_store/webhooks/settings.py +138 -0
- lorebase_store/webhooks/signing.py +50 -0
- lorebase_store-0.2.3.dist-info/METADATA +13 -0
- lorebase_store-0.2.3.dist-info/RECORD +163 -0
- lorebase_store-0.2.3.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""Provenance of the sides of a merge, read from their materialized heads."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable, Mapping
|
|
6
|
+
|
|
7
|
+
from sqlalchemy import select
|
|
8
|
+
from sqlalchemy.orm import Session
|
|
9
|
+
|
|
10
|
+
from lorebase_core.merge import ChangeStamp
|
|
11
|
+
from lorebase_store.layout import BranchLayout, lineage_column, load_layout_sync
|
|
12
|
+
from lorebase_store.materialize.rows import PartitionType, parse_partition, split_link_key
|
|
13
|
+
from lorebase_store.repositories.provenance import PROVENANCE
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BranchLineage:
|
|
17
|
+
"""Core `ProvenanceLoader` over the materialized head of one branch.
|
|
18
|
+
|
|
19
|
+
The merge asks for the provenance of elements of the source head; the materialized rows of
|
|
20
|
+
the source branch hold their Lineage (Records and Links) and the last change of each
|
|
21
|
+
schema element (history). The Revision argument is not consulted: it is the branch head
|
|
22
|
+
the merge was started from, read in the same transaction. An overlay source is read
|
|
23
|
+
through its resolved rows.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, session: Session, domain_id: str, branch: str) -> None:
|
|
27
|
+
self._session = session
|
|
28
|
+
self._domain = domain_id
|
|
29
|
+
self._branch = branch
|
|
30
|
+
self._layout: BranchLayout | None = None
|
|
31
|
+
|
|
32
|
+
def _resolved(self) -> BranchLayout:
|
|
33
|
+
if self._layout is None:
|
|
34
|
+
self._layout = load_layout_sync(self._session, self._domain, self._branch)
|
|
35
|
+
return self._layout
|
|
36
|
+
|
|
37
|
+
def __call__(self, revision: bytes, partition: str, key: str) -> Mapping[str, str]:
|
|
38
|
+
parsed = parse_partition(partition)
|
|
39
|
+
layout = self._resolved()
|
|
40
|
+
if parsed.type is PartitionType.KIND:
|
|
41
|
+
records = layout.records()
|
|
42
|
+
query = select(lineage_column(records)).where(
|
|
43
|
+
records.c.domain_id == self._domain,
|
|
44
|
+
records.c.branch == self._branch,
|
|
45
|
+
records.c.kind_id == parsed.element_id,
|
|
46
|
+
records.c.key == key,
|
|
47
|
+
)
|
|
48
|
+
elif parsed.type is PartitionType.RELATION:
|
|
49
|
+
from_key, to_key = split_link_key(key)
|
|
50
|
+
links = layout.links()
|
|
51
|
+
query = select(lineage_column(links)).where(
|
|
52
|
+
links.c.domain_id == self._domain,
|
|
53
|
+
links.c.branch == self._branch,
|
|
54
|
+
links.c.relation_id == parsed.element_id,
|
|
55
|
+
links.c.from_key == from_key,
|
|
56
|
+
links.c.to_key == to_key,
|
|
57
|
+
)
|
|
58
|
+
else:
|
|
59
|
+
history = layout.history()
|
|
60
|
+
query = (
|
|
61
|
+
select(history.c.provenance)
|
|
62
|
+
.where(
|
|
63
|
+
history.c.domain_id == self._domain,
|
|
64
|
+
history.c.branch == self._branch,
|
|
65
|
+
history.c.partition == partition,
|
|
66
|
+
history.c.element_key == key,
|
|
67
|
+
)
|
|
68
|
+
.order_by(history.c.at.desc())
|
|
69
|
+
.limit(1)
|
|
70
|
+
)
|
|
71
|
+
found = self._session.execute(query).scalar_one_or_none()
|
|
72
|
+
return {str(k): str(v) for k, v in (found or {}).items()}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class MergeLineage:
|
|
76
|
+
"""Core `LineageLoader`: the change stamps of both sides of a merge.
|
|
77
|
+
|
|
78
|
+
`branches` maps the head Revision of each merged branch to its name; the Lineage of an
|
|
79
|
+
element is read from that branch's materialized head (see `BranchLineage`) and its
|
|
80
|
+
provenance ids are resolved to stamps (origin labels and time) from the provenance table.
|
|
81
|
+
Provenance rows are read once per merge.
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
def __init__(self, session: Session, domain_id: str, branches: Mapping[bytes, str]) -> None:
|
|
85
|
+
self._session = session
|
|
86
|
+
self._domain = domain_id
|
|
87
|
+
self._branches = {
|
|
88
|
+
revision: BranchLineage(session, domain_id, name) for revision, name in branches.items()
|
|
89
|
+
}
|
|
90
|
+
self._stamps: dict[str, ChangeStamp | None] = {}
|
|
91
|
+
|
|
92
|
+
def __call__(self, revision: bytes, partition: str, key: str) -> Mapping[str, ChangeStamp]:
|
|
93
|
+
branch = self._branches.get(revision)
|
|
94
|
+
if branch is None:
|
|
95
|
+
return {}
|
|
96
|
+
ids = branch(revision, partition, key)
|
|
97
|
+
self._load(ids.values())
|
|
98
|
+
found: dict[str, ChangeStamp] = {}
|
|
99
|
+
for attribute, provenance_id in ids.items():
|
|
100
|
+
stamp = self._stamps.get(provenance_id)
|
|
101
|
+
if stamp is not None:
|
|
102
|
+
found[attribute] = stamp
|
|
103
|
+
return found
|
|
104
|
+
|
|
105
|
+
def _load(self, ids: Iterable[str]) -> None:
|
|
106
|
+
missing = sorted({i for i in ids if i not in self._stamps})
|
|
107
|
+
if not missing:
|
|
108
|
+
return
|
|
109
|
+
rows = self._session.execute(
|
|
110
|
+
select(
|
|
111
|
+
PROVENANCE.c.id,
|
|
112
|
+
PROVENANCE.c.origin_type,
|
|
113
|
+
PROVENANCE.c.origin_ref,
|
|
114
|
+
PROVENANCE.c.actor,
|
|
115
|
+
PROVENANCE.c.at,
|
|
116
|
+
).where(PROVENANCE.c.domain_id == self._domain, PROVENANCE.c.id.in_(missing))
|
|
117
|
+
)
|
|
118
|
+
for row in rows:
|
|
119
|
+
self._stamps[row.id] = ChangeStamp.from_stored(
|
|
120
|
+
row.id, row.origin_type, row.origin_ref or {}, row.actor, row.at
|
|
121
|
+
)
|
|
122
|
+
for provenance_id in missing:
|
|
123
|
+
self._stamps.setdefault(provenance_id, None)
|