zikaron 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- zikaron/__init__.py +1 -0
- zikaron/cli/__init__.py +1 -0
- zikaron/cli/main.py +114 -0
- zikaron/core/__init__.py +1 -0
- zikaron/core/clock.py +78 -0
- zikaron/core/config/__init__.py +1 -0
- zikaron/core/config/keys.py +395 -0
- zikaron/core/config/resolution.py +267 -0
- zikaron/core/consolidation/__init__.py +1 -0
- zikaron/core/consolidation/authorization.py +316 -0
- zikaron/core/consolidation/candidates.py +147 -0
- zikaron/core/consolidation/context.py +166 -0
- zikaron/core/consolidation/grouping.py +383 -0
- zikaron/core/consolidation/groups.py +490 -0
- zikaron/core/consolidation/payload.py +246 -0
- zikaron/core/consolidation/planning.py +192 -0
- zikaron/core/consolidation/rowstate.py +68 -0
- zikaron/core/consolidation/runs.py +306 -0
- zikaron/core/consolidation/serving.py +462 -0
- zikaron/core/consolidation/verbs.py +500 -0
- zikaron/core/errors.py +355 -0
- zikaron/core/events.py +748 -0
- zikaron/core/indexing/__init__.py +1 -0
- zikaron/core/indexing/acquisition.py +255 -0
- zikaron/core/indexing/chunking.py +368 -0
- zikaron/core/indexing/encoder.py +537 -0
- zikaron/core/indexing/lexical.py +86 -0
- zikaron/core/indexing/model_cache.py +93 -0
- zikaron/core/indexing/model_pin.py +89 -0
- zikaron/core/indexing/vectors.py +223 -0
- zikaron/core/indexing/writes.py +461 -0
- zikaron/core/knowledge/__init__.py +5 -0
- zikaron/core/knowledge/arms.py +104 -0
- zikaron/core/knowledge/builds.py +204 -0
- zikaron/core/knowledge/candidates.py +130 -0
- zikaron/core/knowledge/changes.py +175 -0
- zikaron/core/knowledge/chunking.py +376 -0
- zikaron/core/knowledge/counters.py +228 -0
- zikaron/core/knowledge/database.py +380 -0
- zikaron/core/knowledge/ddl.py +196 -0
- zikaron/core/knowledge/disposal.py +213 -0
- zikaron/core/knowledge/errors.py +166 -0
- zikaron/core/knowledge/files.py +202 -0
- zikaron/core/knowledge/git.py +385 -0
- zikaron/core/knowledge/groups.py +450 -0
- zikaron/core/knowledge/lexical.py +64 -0
- zikaron/core/knowledge/lifecycle.py +418 -0
- zikaron/core/knowledge/lock.py +277 -0
- zikaron/core/knowledge/meta.py +393 -0
- zikaron/core/knowledge/paths.py +55 -0
- zikaron/core/knowledge/pending.py +59 -0
- zikaron/core/knowledge/registry.py +264 -0
- zikaron/core/knowledge/repair.py +152 -0
- zikaron/core/knowledge/reporting.py +436 -0
- zikaron/core/knowledge/roots.py +91 -0
- zikaron/core/knowledge/scan.py +429 -0
- zikaron/core/knowledge/search.py +346 -0
- zikaron/core/knowledge/state.py +174 -0
- zikaron/core/knowledge/text.py +166 -0
- zikaron/core/knowledge/vectors.py +102 -0
- zikaron/core/knowledge/walk.py +264 -0
- zikaron/core/knowledge/writes.py +127 -0
- zikaron/core/records/__init__.py +1 -0
- zikaron/core/records/memory.py +961 -0
- zikaron/core/records/receipts.py +161 -0
- zikaron/core/records/supersession.py +221 -0
- zikaron/core/retrieval/__init__.py +1 -0
- zikaron/core/retrieval/arms.py +318 -0
- zikaron/core/retrieval/block.py +107 -0
- zikaron/core/retrieval/eligibility.py +164 -0
- zikaron/core/retrieval/query.py +327 -0
- zikaron/core/retrieval/ranking.py +260 -0
- zikaron/core/retrieval/reads.py +294 -0
- zikaron/core/retrieval/retrieve.py +158 -0
- zikaron/core/retrieval/similarity.py +87 -0
- zikaron/core/signals/__init__.py +34 -0
- zikaron/core/signals/contention.py +106 -0
- zikaron/core/signals/dedup.py +201 -0
- zikaron/core/signals/horizon.py +47 -0
- zikaron/core/signals/repair.py +161 -0
- zikaron/core/signals/retirement.py +83 -0
- zikaron/core/signals/sessions.py +105 -0
- zikaron/core/signals/writes.py +200 -0
- zikaron/core/store/__init__.py +1 -0
- zikaron/core/store/connection.py +202 -0
- zikaron/core/store/ddl.py +215 -0
- zikaron/core/store/embedder.py +45 -0
- zikaron/core/store/meta.py +152 -0
- zikaron/core/store/permissions.py +160 -0
- zikaron/core/store/store.py +408 -0
- zikaron/core/store/transactions.py +181 -0
- zikaron/core/write/__init__.py +33 -0
- zikaron/core/write/dedup.py +145 -0
- zikaron/core/write/tools.py +290 -0
- zikaron/doctor/__init__.py +1 -0
- zikaron/doctor/checks.py +220 -0
- zikaron/doctor/main.py +64 -0
- zikaron/harness/__init__.py +1 -0
- zikaron/harness/detect.py +92 -0
- zikaron/harness/spec.py +320 -0
- zikaron/hook/__init__.py +1 -0
- zikaron/hook/connect.py +379 -0
- zikaron/hook/envelope.py +106 -0
- zikaron/hook/failure.py +104 -0
- zikaron/hook/limits.py +61 -0
- zikaron/hook/main.py +118 -0
- zikaron/hook/push.py +183 -0
- zikaron/hook/rpc.py +85 -0
- zikaron/hook/spawn_warm.py +81 -0
- zikaron/hook/subagent_policy.py +57 -0
- zikaron/hook/tripwire.py +54 -0
- zikaron/hook/warm_helper.py +137 -0
- zikaron/hook/write_policy.py +319 -0
- zikaron/install/__init__.py +4 -0
- zikaron/install/__main__.py +18 -0
- zikaron/install/assets.py +394 -0
- zikaron/install/entries.py +370 -0
- zikaron/install/harness.py +185 -0
- zikaron/install/main.py +375 -0
- zikaron/install/targets.py +789 -0
- zikaron/install/writer.py +973 -0
- zikaron/knowledge/__init__.py +1 -0
- zikaron/knowledge/__main__.py +17 -0
- zikaron/knowledge/indexer/__init__.py +1 -0
- zikaron/knowledge/indexer/__main__.py +17 -0
- zikaron/knowledge/indexer/detach.py +83 -0
- zikaron/knowledge/indexer/main.py +187 -0
- zikaron/knowledge/main.py +466 -0
- zikaron/knowledge/scope.py +133 -0
- zikaron/mcp/__init__.py +6 -0
- zikaron/mcp/connection.py +583 -0
- zikaron/mcp/consolidator.py +316 -0
- zikaron/mcp/errors.py +73 -0
- zikaron/mcp/main.py +66 -0
- zikaron/mcp/primary.py +420 -0
- zikaron/mcp/server.py +96 -0
- zikaron/mcp/spill.py +328 -0
- zikaron/mcp/tool_names.py +67 -0
- zikaron/py.typed +0 -0
- zikaron/service/__init__.py +1 -0
- zikaron/service/asyncio_compat.py +126 -0
- zikaron/service/context.py +251 -0
- zikaron/service/dispatch.py +332 -0
- zikaron/service/dispatch_consolidation.py +397 -0
- zikaron/service/dispatch_knowledge.py +469 -0
- zikaron/service/envelope.py +166 -0
- zikaron/service/lifecycle.py +467 -0
- zikaron/service/log.py +96 -0
- zikaron/service/main.py +531 -0
- zikaron/service/params.py +168 -0
- zikaron/service/paths.py +181 -0
- zikaron/service/rpc.py +176 -0
- zikaron/service/security.py +156 -0
- zikaron/service/serialize.py +204 -0
- zikaron/service/serialize_knowledge.py +238 -0
- zikaron/service/server.py +416 -0
- zikaron-0.1.0.dist-info/METADATA +770 -0
- zikaron-0.1.0.dist-info/RECORD +162 -0
- zikaron-0.1.0.dist-info/WHEEL +5 -0
- zikaron-0.1.0.dist-info/entry_points.txt +4 -0
- zikaron-0.1.0.dist-info/licenses/LICENSE +21 -0
- zikaron-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
"""Schema creation, open/close, and the checks each of those two paths is required to run.
|
|
2
|
+
|
|
3
|
+
`schema.md` §"Creating the dense index" fixes creation's order because `vec0` fixes a column's
|
|
4
|
+
width forever at `CREATE` time: validate the effective `embed_dim`, check the configured
|
|
5
|
+
embedder's actual output width against it, *then* create tables — in that order, so a
|
|
6
|
+
mismatched model name never produces a store whose every future write fails at insert time
|
|
7
|
+
pointing at the wrong culprit. Open re-validates `meta` on every call, per the same document's
|
|
8
|
+
"validation happens twice" rule, and refuses to proceed through a `reindexing` sentinel left by
|
|
9
|
+
a process that died mid-reindex, or through a `schema_version` this build does not support.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import re
|
|
14
|
+
import uuid
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from types import TracebackType
|
|
17
|
+
from typing import Final, Self
|
|
18
|
+
|
|
19
|
+
import aiosqlite
|
|
20
|
+
|
|
21
|
+
from zikaron.core.config.resolution import EffectiveConfig
|
|
22
|
+
from zikaron.core.errors import BadConfigSource, ErrorCode, ZikaronError
|
|
23
|
+
from zikaron.core.store import ddl, meta, permissions
|
|
24
|
+
from zikaron.core.store.connection import open_connection
|
|
25
|
+
from zikaron.core.store.embedder import Embedder
|
|
26
|
+
|
|
27
|
+
#: The schema version every table and invariant in this build implements. Sourced from nowhere
|
|
28
|
+
#: else: `zikaron.core.errors.ERROR_SPECS[ErrorCode.SCHEMA_INCOMPATIBLE]`'s payload fixes
|
|
29
|
+
#: `supported` to the same integer, and a test asserts the two agree rather than one restating
|
|
30
|
+
#: the other: one declaration, checked from two directions, rather than two that agree today.
|
|
31
|
+
SUPPORTED_SCHEMA_VERSION: Final = 1
|
|
32
|
+
|
|
33
|
+
_DB_FILENAME: Final = "memory.db"
|
|
34
|
+
|
|
35
|
+
#: `vec0` has no introspection function for a column's declared width; the only place it is
|
|
36
|
+
#: recorded is the literal `CREATE VIRTUAL TABLE` text `sqlite_master` stores verbatim. Matched
|
|
37
|
+
#: as the **whole** value, not a substring search — `sqlite_master.sql` holds exactly one
|
|
38
|
+
#: statement per row, so a genuine `vec0` table's row is entirely this shape and nothing else,
|
|
39
|
+
#: and searching for the shape *inside* an arbitrary string would accept a table that merely
|
|
40
|
+
#: quotes the phrase as data (a `CHECK` constraint's string literal, for instance) without ever
|
|
41
|
+
#: being `vec0` at all. `fullmatch` after collapsing whitespace is what makes "the entire
|
|
42
|
+
#: recorded statement is this shape" the actual test, rather than "this shape occurs somewhere
|
|
43
|
+
#: in the recorded statement."
|
|
44
|
+
_VEC0_WIDTH_PATTERN: Final = re.compile(
|
|
45
|
+
r"CREATE\s+VIRTUAL\s+TABLE\s+memory_vec\s+USING\s+vec0\s*\(\s*embedding\s+float\[(\d+)\]\s*\)",
|
|
46
|
+
re.IGNORECASE,
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
async def _physical_vec0_width(db: aiosqlite.Connection) -> int:
|
|
51
|
+
"""The width `memory_vec.embedding` was actually created at, read back from its own DDL.
|
|
52
|
+
|
|
53
|
+
`meta.embed_dim` records what the store's `meta` row *says* the width is; this reads what
|
|
54
|
+
the column *is*, so invariant 11's check on open cannot be satisfied by a `meta` row that
|
|
55
|
+
agrees with the effective config while the physical column disagrees with both — nor by an
|
|
56
|
+
object merely *named* `memory_vec` that is not a `vec0` virtual table at all. Matching the
|
|
57
|
+
*whole* recorded statement, not searching for the shape somewhere inside it, is what rules
|
|
58
|
+
that second case out: a table whose `CHECK` constraint merely quotes the expected phrase as
|
|
59
|
+
string data would satisfy a substring search without being `vec0` at all, since the phrase
|
|
60
|
+
would then simply be present somewhere in a longer string that is not, as a whole, that
|
|
61
|
+
statement.
|
|
62
|
+
"""
|
|
63
|
+
rows = await db.execute_fetchall("SELECT sql FROM sqlite_master WHERE name = 'memory_vec'")
|
|
64
|
+
matches = list(rows)
|
|
65
|
+
if len(matches) != 1:
|
|
66
|
+
raise ZikaronError(
|
|
67
|
+
ErrorCode.BAD_CONFIG,
|
|
68
|
+
source=BadConfigSource.META,
|
|
69
|
+
key="memory_vec",
|
|
70
|
+
value=f"{len(matches)} matching tables",
|
|
71
|
+
expected="exactly one memory_vec table",
|
|
72
|
+
)
|
|
73
|
+
(sql,) = matches[0]
|
|
74
|
+
collapsed = re.sub(r"\s+", " ", str(sql)).strip()
|
|
75
|
+
found = _VEC0_WIDTH_PATTERN.fullmatch(collapsed)
|
|
76
|
+
if found is None:
|
|
77
|
+
raise ZikaronError(
|
|
78
|
+
ErrorCode.BAD_CONFIG,
|
|
79
|
+
source=BadConfigSource.META,
|
|
80
|
+
key="memory_vec",
|
|
81
|
+
value=str(sql),
|
|
82
|
+
expected="CREATE VIRTUAL TABLE memory_vec USING vec0 (embedding float[<n>])",
|
|
83
|
+
)
|
|
84
|
+
return int(found.group(1))
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
async def _read_meta_table(db: aiosqlite.Connection) -> dict[str, str]:
|
|
88
|
+
"""Every `meta` row's text value, or an empty mapping if `meta` itself does not exist.
|
|
89
|
+
|
|
90
|
+
`existing_only=True` on `open_connection` stops `Store.open` from *creating* a missing
|
|
91
|
+
store, but it does nothing about an existing `memory.db` whose `meta` table is itself
|
|
92
|
+
absent — dropped by hand, or left behind by a process that failed between creating the
|
|
93
|
+
file and running its DDL. That case must still fail at the Zikaron boundary as
|
|
94
|
+
`bad_config`, not as a raw `sqlite3.OperationalError` bubbling straight out of a driver
|
|
95
|
+
call. Returning an empty mapping rather than raising here reuses `parse_and_validate`'s own
|
|
96
|
+
existing "first required key found missing" reporting — a store with no `meta` table is a
|
|
97
|
+
store missing all five required keys, which is exactly true, and reporting it that way
|
|
98
|
+
costs no second error path.
|
|
99
|
+
"""
|
|
100
|
+
try:
|
|
101
|
+
rows = await db.execute_fetchall("SELECT key, value FROM meta")
|
|
102
|
+
except aiosqlite.Error:
|
|
103
|
+
return {}
|
|
104
|
+
return {str(key): str(value) for key, value in rows}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _store_not_created(db_path: Path) -> ZikaronError:
|
|
108
|
+
"""What a failed connect to an existing-only `memory.db` means to the memory store.
|
|
109
|
+
|
|
110
|
+
Scoped to the connect rather than to the whole open, because a database that connected and
|
|
111
|
+
then refused a pragma has been created — saying it has not would send an operator looking for
|
|
112
|
+
a missing file that is right there.
|
|
113
|
+
"""
|
|
114
|
+
return ZikaronError(
|
|
115
|
+
ErrorCode.BAD_CONFIG,
|
|
116
|
+
source=BadConfigSource.DERIVED,
|
|
117
|
+
key="store_dir",
|
|
118
|
+
value=str(db_path),
|
|
119
|
+
expected="an existing, openable memory.db — this store has not been created",
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _dimension_mismatch(
|
|
124
|
+
config: EffectiveConfig, embed_dim: int, embedder: Embedder
|
|
125
|
+
) -> ZikaronError:
|
|
126
|
+
return ZikaronError(
|
|
127
|
+
ErrorCode.BAD_CONFIG,
|
|
128
|
+
source=BadConfigSource.FILE,
|
|
129
|
+
file=str(config.source_of("embed_dim") or "<default>"),
|
|
130
|
+
key="embedding.embed_dim",
|
|
131
|
+
value=str(embed_dim),
|
|
132
|
+
expected=f"the configured embedder ({embedder.model_name}) actually emits "
|
|
133
|
+
f"{embedder.dim}-dimensional vectors",
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _model_name_mismatch(config: EffectiveConfig, embedder: Embedder) -> ZikaronError:
|
|
138
|
+
configured = config.get_str("embed_model")
|
|
139
|
+
return ZikaronError(
|
|
140
|
+
ErrorCode.BAD_CONFIG,
|
|
141
|
+
source=BadConfigSource.FILE,
|
|
142
|
+
file=str(config.source_of("embed_model") or "<default>"),
|
|
143
|
+
key="embedding.embed_model",
|
|
144
|
+
value=configured,
|
|
145
|
+
expected=f"the embedder passed to create() reports model_name={embedder.model_name!r}, "
|
|
146
|
+
"which must match the configured embed_model — meta seeds from this call's embedder, "
|
|
147
|
+
"not from the config text alone",
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _reindexing_in_progress(since: str) -> ZikaronError:
|
|
152
|
+
return ZikaronError(ErrorCode.REINDEXING, since=since)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _schema_too_new(found: int) -> ZikaronError:
|
|
156
|
+
return ZikaronError(
|
|
157
|
+
ErrorCode.SCHEMA_INCOMPATIBLE, found=found, supported=SUPPORTED_SCHEMA_VERSION
|
|
158
|
+
)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _embed_settings_disagree(
|
|
162
|
+
current: meta.StoreMeta, configured_model: str, configured_dim: int
|
|
163
|
+
) -> ZikaronError:
|
|
164
|
+
return ZikaronError(
|
|
165
|
+
ErrorCode.BAD_CONFIG,
|
|
166
|
+
source=BadConfigSource.META,
|
|
167
|
+
key="embed_model/embed_dim",
|
|
168
|
+
value=f"{configured_model}/{configured_dim}",
|
|
169
|
+
expected=f"{current.embed_model}/{current.embed_dim} (the values this store was "
|
|
170
|
+
"created with — invariant 11 requires a forced reindex, never a silent continue, "
|
|
171
|
+
"on disagreement)",
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _physical_width_disagrees(recorded_dim: int, physical_width: int) -> ZikaronError:
|
|
176
|
+
return ZikaronError(
|
|
177
|
+
ErrorCode.BAD_CONFIG,
|
|
178
|
+
source=BadConfigSource.META,
|
|
179
|
+
key="embed_dim",
|
|
180
|
+
value=str(recorded_dim),
|
|
181
|
+
expected=f"{physical_width} (the width memory_vec's own column was actually created "
|
|
182
|
+
"at — meta.embed_dim must agree with the physical column, not only with the config, "
|
|
183
|
+
"or a corrupted meta row could silently outlive the vec0 table it describes)",
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
class Store:
|
|
188
|
+
"""An open connection to one Zikaron store, plus the `meta` it was opened with.
|
|
189
|
+
|
|
190
|
+
Construct only through `create` or `open`, never directly: both classmethods run the
|
|
191
|
+
validation their path requires before a `Store` exists to hand back, so holding one is
|
|
192
|
+
holding a store already known to satisfy invariants 1, 3 and 11 for this open.
|
|
193
|
+
|
|
194
|
+
`opened_inode` is `db_path`'s own inode, captured by `open_connection` immediately after
|
|
195
|
+
its own `aiosqlite.connect()` returned, with no `await` in between — the tightest available
|
|
196
|
+
capture short of a materially larger VFS-level integration a deliberate, human-authorized
|
|
197
|
+
decision declined to pursue (see `open_connection`'s own docstring for the full reasoning
|
|
198
|
+
and the narrow, accepted gap this leaves). It is *not* re-derivable later from
|
|
199
|
+
`self.path.stat()`: that would read whatever file currently sits at the path, which is
|
|
200
|
+
precisely the question this field exists to answer independently of.
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
def __init__(
|
|
204
|
+
self,
|
|
205
|
+
db: aiosqlite.Connection,
|
|
206
|
+
db_path: Path,
|
|
207
|
+
current_meta: meta.StoreMeta,
|
|
208
|
+
opened_inode: int,
|
|
209
|
+
) -> None:
|
|
210
|
+
self._db: Final = db
|
|
211
|
+
self.path: Final = db_path
|
|
212
|
+
self.meta: Final = current_meta
|
|
213
|
+
self.opened_inode: Final = opened_inode
|
|
214
|
+
|
|
215
|
+
@property
|
|
216
|
+
def connection(self) -> aiosqlite.Connection:
|
|
217
|
+
"""The underlying `aiosqlite` connection, for a caller that needs to run its own SQL.
|
|
218
|
+
|
|
219
|
+
Exposed rather than wrapped with per-statement methods: this module owns schema creation
|
|
220
|
+
and the open path, and no record behaviour lives here for a method to wrap.
|
|
221
|
+
"""
|
|
222
|
+
return self._db
|
|
223
|
+
|
|
224
|
+
async def close(self) -> None:
|
|
225
|
+
"""Close the underlying connection. Safe to call once; not idempotent."""
|
|
226
|
+
await self._db.close()
|
|
227
|
+
|
|
228
|
+
async def __aenter__(self) -> Self:
|
|
229
|
+
return self
|
|
230
|
+
|
|
231
|
+
async def __aexit__(
|
|
232
|
+
self,
|
|
233
|
+
exc_type: type[BaseException] | None,
|
|
234
|
+
exc: BaseException | None,
|
|
235
|
+
traceback: TracebackType | None,
|
|
236
|
+
) -> None:
|
|
237
|
+
await self.close()
|
|
238
|
+
|
|
239
|
+
@classmethod
|
|
240
|
+
async def create(cls, store_dir: Path, config: EffectiveConfig, embedder: Embedder) -> Self:
|
|
241
|
+
"""Create a fresh store at `store_dir/memory.db`, or raise before touching any table.
|
|
242
|
+
|
|
243
|
+
Order is the contract, not an implementation choice (`schema.md` §"Creating the dense
|
|
244
|
+
index"): the effective `embed_dim` is validated, then the embedder's *actual* reported
|
|
245
|
+
width is checked against it, and only once both hold does any `CREATE` statement run.
|
|
246
|
+
A mismatch is `bad_config` naming both values, with **no database file yet on disk** —
|
|
247
|
+
`store_dir` is created, but `memory.db` is opened only after the check passes, so a
|
|
248
|
+
typo'd model name never produces a store to clean up.
|
|
249
|
+
|
|
250
|
+
Args:
|
|
251
|
+
store_dir: the `.zikaron` directory this store lives in. Created at `0700` if
|
|
252
|
+
absent; tightened to it if it exists wider.
|
|
253
|
+
config: the effective configuration to seed `meta` from. Its `embed_dim` and
|
|
254
|
+
`embed_model` become this store's dual-homed `meta` values, authoritative from
|
|
255
|
+
this point on.
|
|
256
|
+
embedder: the configured embedder. Consulted only for `.dim` and `.model_name` —
|
|
257
|
+
creation embeds nothing.
|
|
258
|
+
|
|
259
|
+
Raises:
|
|
260
|
+
ZikaronError: `BAD_CONFIG`, naming `embed_dim` and the embedder's actual width, if
|
|
261
|
+
the effective config's `embed_dim` does not equal what `embedder.dim` reports;
|
|
262
|
+
or naming `embed_model`, if `embedder.model_name` disagrees with the configured
|
|
263
|
+
`embed_model` — the config states which model to seed `meta` with, and the
|
|
264
|
+
embedder that actually produced the width check must be that model, not merely
|
|
265
|
+
one of the right dimension.
|
|
266
|
+
"""
|
|
267
|
+
embed_dim = config.get_int("embed_dim")
|
|
268
|
+
embed_model = config.get_str("embed_model")
|
|
269
|
+
if embedder.dim != embed_dim:
|
|
270
|
+
raise _dimension_mismatch(config, embed_dim, embedder)
|
|
271
|
+
if embedder.model_name != embed_model:
|
|
272
|
+
raise _model_name_mismatch(config, embedder)
|
|
273
|
+
|
|
274
|
+
await asyncio.to_thread(permissions.ensure_store_dir, store_dir)
|
|
275
|
+
db_path = store_dir / _DB_FILENAME
|
|
276
|
+
chunk_max_tokens = config.get_int("chunk_max_tokens")
|
|
277
|
+
|
|
278
|
+
# A pre-existing `memory.db` — left behind by an earlier failed attempt, or widened by
|
|
279
|
+
# an operator's own `chmod` — must be tightened before anything fallible runs, not only
|
|
280
|
+
# after this call succeeds. `enforce_store_file_mode` after a successful create closes
|
|
281
|
+
# the gap for files *this* call creates; it says nothing about a file that was already
|
|
282
|
+
# here and wide before this call started, and a failure partway through must not leave
|
|
283
|
+
# that file exactly as wide as it found it.
|
|
284
|
+
for path in permissions.database_files(db_path):
|
|
285
|
+
await asyncio.to_thread(permissions.enforce_store_file_mode, path)
|
|
286
|
+
|
|
287
|
+
with permissions.restrictive_umask():
|
|
288
|
+
db, opened_inode = await open_connection(db_path, pragmas=ddl.PRAGMAS)
|
|
289
|
+
try:
|
|
290
|
+
defaults = await cls._create_tables_and_meta(
|
|
291
|
+
db, embed_dim, embed_model, chunk_max_tokens
|
|
292
|
+
)
|
|
293
|
+
except BaseException:
|
|
294
|
+
await db.close()
|
|
295
|
+
raise
|
|
296
|
+
|
|
297
|
+
for path in permissions.database_files(db_path):
|
|
298
|
+
await asyncio.to_thread(permissions.enforce_store_file_mode, path)
|
|
299
|
+
|
|
300
|
+
current_meta = meta.parse_and_validate(defaults)
|
|
301
|
+
return cls(db, db_path, current_meta, opened_inode)
|
|
302
|
+
|
|
303
|
+
@staticmethod
|
|
304
|
+
async def _create_tables_and_meta(
|
|
305
|
+
db: aiosqlite.Connection, embed_dim: int, embed_model: str, chunk_max_tokens: int
|
|
306
|
+
) -> dict[str, str]:
|
|
307
|
+
"""Run every `CREATE` statement plus the `meta` defaults insert, in one transaction.
|
|
308
|
+
|
|
309
|
+
Split out of `create` so that method's own control flow stays about ordering and error
|
|
310
|
+
handling, while this one is purely "what SQL runs" — the two concerns `TRY301` otherwise
|
|
311
|
+
conflates by wanting every raise abstracted out of a `try` block that has no raise of
|
|
312
|
+
its own to abstract.
|
|
313
|
+
"""
|
|
314
|
+
await db.execute("BEGIN")
|
|
315
|
+
for statement in ddl.FIXED_STATEMENTS:
|
|
316
|
+
await db.execute(statement)
|
|
317
|
+
await db.execute(ddl.memory_vec_statement(embed_dim))
|
|
318
|
+
store_id = str(uuid.uuid4())
|
|
319
|
+
defaults = dict(
|
|
320
|
+
meta.defaults_at_creation(
|
|
321
|
+
schema_version=SUPPORTED_SCHEMA_VERSION,
|
|
322
|
+
store_id=store_id,
|
|
323
|
+
embed_model=embed_model,
|
|
324
|
+
embed_dim=embed_dim,
|
|
325
|
+
chunk_max_tokens=chunk_max_tokens,
|
|
326
|
+
)
|
|
327
|
+
)
|
|
328
|
+
for key, value in defaults.items():
|
|
329
|
+
await db.execute("INSERT INTO meta (key, value) VALUES (?, ?)", (key, value))
|
|
330
|
+
await db.commit()
|
|
331
|
+
return defaults
|
|
332
|
+
|
|
333
|
+
@classmethod
|
|
334
|
+
async def open(cls, store_dir: Path, config: EffectiveConfig) -> Self:
|
|
335
|
+
"""Open an existing store at `store_dir/memory.db`, validating on every call.
|
|
336
|
+
|
|
337
|
+
Runs, in order: permission enforcement and the symlink refusal (tightening a store left
|
|
338
|
+
wider than `0700`/`0600` by an older build or an operator's own `chmod`), the
|
|
339
|
+
`reindexing` sentinel check (invariant 3 — a process that died mid-reindex must not be
|
|
340
|
+
read through), all five required `meta` keys (`schema.md`'s "validation happens
|
|
341
|
+
twice"), the `schema_version` gate, invariant 11's physical `memory_vec` width check
|
|
342
|
+
(against `meta`, not only against the config — `meta` can drift from the table it
|
|
343
|
+
describes even when it agrees with the file), and finally invariant 11's
|
|
344
|
+
`embed_model`/`embed_dim` comparison against the effective config.
|
|
345
|
+
|
|
346
|
+
Args:
|
|
347
|
+
store_dir: the `.zikaron` directory holding `memory.db`.
|
|
348
|
+
config: the effective configuration this open is running under, compared against
|
|
349
|
+
`meta`'s dual-homed values rather than trusted silently.
|
|
350
|
+
|
|
351
|
+
Raises:
|
|
352
|
+
ZikaronError: `BAD_CONFIG` if `store_dir` is a symlink; `REINDEXING` if the sentinel
|
|
353
|
+
is present; `BAD_CONFIG` (`source='meta'`) if a required key is missing,
|
|
354
|
+
unparseable, or out of range, if the physical `memory_vec` column's width
|
|
355
|
+
disagrees with `meta.embed_dim`, or if `embed_model`/`embed_dim` disagrees with
|
|
356
|
+
the effective config (invariant 11); `SCHEMA_INCOMPATIBLE` if
|
|
357
|
+
`meta.schema_version` exceeds `SUPPORTED_SCHEMA_VERSION`.
|
|
358
|
+
"""
|
|
359
|
+
# `enforce_existing_store_permissions` is itself synchronous filesystem I/O (symlink
|
|
360
|
+
# checks, `stat`, a possible `chmod` per ancestor and per store file) — run through
|
|
361
|
+
# `asyncio.to_thread` rather than called directly, so this coroutine's first blocking
|
|
362
|
+
# step does not hold the event loop of a caller that shares one across many concurrent
|
|
363
|
+
# requests, exactly as every other blocking call this `open` path makes already does
|
|
364
|
+
# (`open_connection`'s own `aiosqlite.connect`, `_validate_on_open`'s SQL). A caller with
|
|
365
|
+
# no such sharing to protect (a short-lived script, one request at a time) pays this
|
|
366
|
+
# cost identically either way; only a caller like `zikaron-mcp` — one long-running
|
|
367
|
+
# process holding many concurrent tool calls on one loop — depends on it.
|
|
368
|
+
await asyncio.to_thread(permissions.enforce_existing_store_permissions, store_dir)
|
|
369
|
+
db_path = store_dir / _DB_FILENAME
|
|
370
|
+
db, opened_inode = await open_connection(
|
|
371
|
+
db_path,
|
|
372
|
+
pragmas=ddl.PRAGMAS,
|
|
373
|
+
existing_only=True,
|
|
374
|
+
connect_failure=lambda _: _store_not_created(db_path),
|
|
375
|
+
)
|
|
376
|
+
try:
|
|
377
|
+
current_meta = await cls._validate_on_open(db, config)
|
|
378
|
+
except BaseException:
|
|
379
|
+
await db.close()
|
|
380
|
+
raise
|
|
381
|
+
return cls(db, db_path, current_meta, opened_inode)
|
|
382
|
+
|
|
383
|
+
@staticmethod
|
|
384
|
+
async def _validate_on_open(
|
|
385
|
+
db: aiosqlite.Connection, config: EffectiveConfig
|
|
386
|
+
) -> meta.StoreMeta:
|
|
387
|
+
"""Every check `open` must run before handing back a `Store`, in the required order."""
|
|
388
|
+
raw = await _read_meta_table(db)
|
|
389
|
+
|
|
390
|
+
since = raw.get(meta.REINDEXING_KEY)
|
|
391
|
+
if since is not None:
|
|
392
|
+
raise _reindexing_in_progress(since)
|
|
393
|
+
|
|
394
|
+
current_meta = meta.parse_and_validate(raw)
|
|
395
|
+
|
|
396
|
+
if current_meta.schema_version > SUPPORTED_SCHEMA_VERSION:
|
|
397
|
+
raise _schema_too_new(current_meta.schema_version)
|
|
398
|
+
|
|
399
|
+
physical_width = await _physical_vec0_width(db)
|
|
400
|
+
if physical_width != current_meta.embed_dim:
|
|
401
|
+
raise _physical_width_disagrees(current_meta.embed_dim, physical_width)
|
|
402
|
+
|
|
403
|
+
configured_model = config.get_str("embed_model")
|
|
404
|
+
configured_dim = config.get_int("embed_dim")
|
|
405
|
+
if current_meta.embed_model != configured_model or current_meta.embed_dim != configured_dim:
|
|
406
|
+
raise _embed_settings_disagree(current_meta, configured_model, configured_dim)
|
|
407
|
+
|
|
408
|
+
return current_meta
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""How a transaction ends, in one place, for every layer that opens one.
|
|
2
|
+
|
|
3
|
+
`schema.md` invariant 2 makes "exactly one SQLite transaction" a property of every logical
|
|
4
|
+
mutation, and `retrieval.md` makes it a property of a read too, because a probe and the
|
|
5
|
+
`chunk_count` it is compared against have to see one snapshot. Every layer therefore needs the
|
|
6
|
+
same three things — `BEGIN`, the work, and a finalization that leaves the connection out of a
|
|
7
|
+
transaction whatever happened — and the third is where the difficulty is: a `COMMIT` can fail, a
|
|
8
|
+
failed `COMMIT` can leave the transaction open, and a `ROLLBACK` can fail too. A second copy of
|
|
9
|
+
that ladder is precisely how a write already reported as failed comes to be committed by whatever
|
|
10
|
+
runs next, so there is one.
|
|
11
|
+
|
|
12
|
+
This sits in `store/` because everything here is a property of SQLite rather than of records,
|
|
13
|
+
indexing or retrieval: which result codes mean *locked*, and what has to be true of a connection
|
|
14
|
+
when a caller lets go of it.
|
|
15
|
+
|
|
16
|
+
**What varies between callers is only how a driver-level failure is named**, so that is the one
|
|
17
|
+
parameter: a function from the driver's exception to the wire error it should become, or to `None`
|
|
18
|
+
for a failure that has no Zikaron code and should propagate as itself. `architecture.md` §Errors
|
|
19
|
+
fixes both halves — a write names contention `store_busy` and everything else `index_failed`,
|
|
20
|
+
while a read names contention and propagates the rest, since `index_failed`'s contract speaks of
|
|
21
|
+
index maintenance and a rolled-back write.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
import contextlib
|
|
25
|
+
import sqlite3
|
|
26
|
+
from collections.abc import Awaitable, Callable
|
|
27
|
+
from typing import Final, NoReturn
|
|
28
|
+
|
|
29
|
+
import aiosqlite
|
|
30
|
+
|
|
31
|
+
from zikaron.core.errors import ErrorCode, ZikaronError
|
|
32
|
+
|
|
33
|
+
#: One caller's work inside an already-open transaction.
|
|
34
|
+
type Work[T] = Callable[[aiosqlite.Connection], Awaitable[T]]
|
|
35
|
+
|
|
36
|
+
#: How a caller names a driver-level failure: the wire error it becomes, or `None` to propagate
|
|
37
|
+
#: the driver's own exception because no Zikaron code describes it.
|
|
38
|
+
type FailureMap = Callable[[aiosqlite.Error], Exception | None]
|
|
39
|
+
|
|
40
|
+
#: The SQLite **primary** result codes that mean *the store was locked, try again* rather than
|
|
41
|
+
#: *the operation failed*. `store_busy` is the one error the design tells a caller it may retry,
|
|
42
|
+
#: so collapsing contention into anything terminal would take a retryable answer and make it look
|
|
43
|
+
#: final. Primary codes rather than names, because SQLite reports *extended* results — a WAL
|
|
44
|
+
#: reader whose snapshot went stale before it tried to write gets `SQLITE_BUSY_SNAPSHOT`, which is
|
|
45
|
+
#: contention by every meaning that matters and matches no exact-name test for `SQLITE_BUSY`.
|
|
46
|
+
_CONTENTION: Final = frozenset({sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED})
|
|
47
|
+
|
|
48
|
+
#: SQLite packs an extended result as the primary code in its low byte plus a subcode above it.
|
|
49
|
+
_PRIMARY_RESULT_MASK: Final = 0xFF
|
|
50
|
+
|
|
51
|
+
#: The two codes invariant 10's rejection carve-out applies to: a rejected call that still commits
|
|
52
|
+
#: its audit event and, for a version conflict, the receipt for the record it returned. Every other
|
|
53
|
+
#: `ZikaronError` a mutation can raise has written nothing at the point it raises, so it rolls back
|
|
54
|
+
#: the ordinary way.
|
|
55
|
+
_CARVE_OUT_CODES: Final = (ErrorCode.VERSION_CONFLICT, ErrorCode.NO_READ_RECEIPT)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def propagate(_error: aiosqlite.Error) -> None:
|
|
59
|
+
"""The `FailureMap` for work no wire error code describes: raise the driver's own exception.
|
|
60
|
+
|
|
61
|
+
Returning `None` has to be a deliberate choice rather than a default, because a raw driver
|
|
62
|
+
exception escaping unnamed leaves whoever serializes a response with no code to send. This is
|
|
63
|
+
that choice, made once, for every caller whose work is genuinely outside the wire contract —
|
|
64
|
+
the knowledge-base registry and each knowledge base's own tables, neither of which any RPC
|
|
65
|
+
returns a code about. A caller inside the contract names its failures instead.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def is_contention(error: aiosqlite.Error) -> bool:
|
|
70
|
+
"""Whether this driver failure means the store was locked rather than the work failed.
|
|
71
|
+
|
|
72
|
+
Read off the exception's own SQLite result code rather than its message text: the message is
|
|
73
|
+
prose that varies between builds, while the code is the contract SQLite itself publishes. An
|
|
74
|
+
exception carrying no result code — one constructed rather than raised by the driver — is not
|
|
75
|
+
contention by definition.
|
|
76
|
+
"""
|
|
77
|
+
code = getattr(error, "sqlite_errorcode", None)
|
|
78
|
+
return isinstance(code, int) and (code & _PRIMARY_RESULT_MASK) in _CONTENTION
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
async def commit_or_roll_back(db: aiosqlite.Connection, error: BaseException | None) -> None:
|
|
82
|
+
"""Commit on success or on a carve-out rejection; roll back on anything else.
|
|
83
|
+
|
|
84
|
+
This is the one place that decides which of invariant 10's two outcomes applies, and it belongs
|
|
85
|
+
to the transaction-*owning* caller — never to a neutral `<verb>_within_transaction` core that
|
|
86
|
+
something else may have composed into a wider transaction. Only the owner structurally knows
|
|
87
|
+
that authorization ran with nothing else staged ahead of it, which is what makes committing a
|
|
88
|
+
rejection safe; a neutral core cannot see what its caller staged before calling it, so it must
|
|
89
|
+
never touch the transaction on any path.
|
|
90
|
+
|
|
91
|
+
The decision is made from the raised error's own **code**, not from where it was raised: a
|
|
92
|
+
`version_conflict` or `no_read_receipt` has deliberately written an audit event, and for the
|
|
93
|
+
conflict a receipt for the record it returned, so those two commit while the domain mutation
|
|
94
|
+
they rejected does not exist to commit.
|
|
95
|
+
"""
|
|
96
|
+
if error is None or (isinstance(error, ZikaronError) and error.code in _CARVE_OUT_CODES):
|
|
97
|
+
await db.commit()
|
|
98
|
+
else:
|
|
99
|
+
await db.rollback()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _raise_mapped(error: aiosqlite.Error, failure: FailureMap) -> NoReturn:
|
|
103
|
+
"""Raise the wire error this driver failure maps to, or re-raise the driver's own.
|
|
104
|
+
|
|
105
|
+
Annotated `NoReturn` because it is what lets the callers below be written without a trailing
|
|
106
|
+
`raise` that can never run: the type checker knows the branch terminates here, so there is no
|
|
107
|
+
unreachable statement to keep honest.
|
|
108
|
+
"""
|
|
109
|
+
mapped = failure(error)
|
|
110
|
+
if mapped is None:
|
|
111
|
+
raise error
|
|
112
|
+
raise mapped from error
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
async def finalize(
|
|
116
|
+
db: aiosqlite.Connection, error: BaseException | None, *, failure: FailureMap
|
|
117
|
+
) -> None:
|
|
118
|
+
"""Commit or roll back per invariant 10, leaving the connection out of a transaction either way.
|
|
119
|
+
|
|
120
|
+
Two obligations, and the second is the one easy to miss. A `COMMIT` that fails means the write
|
|
121
|
+
did not happen, so it must be reported rather than merely logged — and it can leave the
|
|
122
|
+
transaction *open*, with every staged row still visible on this connection. Reporting the
|
|
123
|
+
failure without also rolling back would leave a write already announced as failed sitting there
|
|
124
|
+
to be committed by whatever runs next, and would make the following call's `BEGIN` fail as a
|
|
125
|
+
nested one. So a failed finalization always attempts the rollback before raising.
|
|
126
|
+
|
|
127
|
+
**If that rollback also fails, the connection is closed.** At that point the transaction state
|
|
128
|
+
could not be repaired, so every later use of this connection risks committing the very write
|
|
129
|
+
this call has just reported as failed — a durable, silent wrong answer. Closing it makes the
|
|
130
|
+
next use of it fail loudly instead, which is the trade this whole layer is built on: one noisy
|
|
131
|
+
failure in preference to one quiet corruption. It does mean disrupting another holder of a
|
|
132
|
+
connection this function only borrows, and that is accepted rather than overlooked — the holder
|
|
133
|
+
gets an error it can act on, where the alternative is a memory the store claims not to have.
|
|
134
|
+
|
|
135
|
+
When this fires while another error is already propagating, the raise replaces it and keeps it
|
|
136
|
+
as the new error's context: the transaction's own outcome is the more useful answer, and
|
|
137
|
+
preserving only the first fault would leave the caller told nothing about the work's fate.
|
|
138
|
+
"""
|
|
139
|
+
try:
|
|
140
|
+
await commit_or_roll_back(db, error)
|
|
141
|
+
except aiosqlite.Error as caught:
|
|
142
|
+
# Unconditional, because rolling back a connection that holds no transaction is a no-op and
|
|
143
|
+
# asking first would only add a branch that means nothing.
|
|
144
|
+
try:
|
|
145
|
+
await db.rollback()
|
|
146
|
+
except aiosqlite.Error:
|
|
147
|
+
with contextlib.suppress(Exception):
|
|
148
|
+
await db.close()
|
|
149
|
+
_raise_mapped(caught, failure)
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
async def in_one_transaction[T](
|
|
153
|
+
db: aiosqlite.Connection, work: Work[T], *, failure: FailureMap
|
|
154
|
+
) -> T:
|
|
155
|
+
"""Run `work` inside one transaction, then commit or roll back per invariant 10.
|
|
156
|
+
|
|
157
|
+
Every driver-level failure goes through `failure`, and all three places one can occur are
|
|
158
|
+
covered: opening the transaction, the work itself, and finalizing. A raw `sqlite3` exception
|
|
159
|
+
escaping *unnamed* would leave whoever serializes the response with no code to send — which is
|
|
160
|
+
why `failure` returning `None` has to be a deliberate choice by that caller rather than the
|
|
161
|
+
default.
|
|
162
|
+
|
|
163
|
+
A `ZikaronError` raised by `work` passes through untouched, so a rejection keeps its own code —
|
|
164
|
+
and, for invariant 10's two carve-out codes, its committed audit trail.
|
|
165
|
+
"""
|
|
166
|
+
try:
|
|
167
|
+
await db.execute("BEGIN")
|
|
168
|
+
except aiosqlite.Error as caught:
|
|
169
|
+
_raise_mapped(caught, failure)
|
|
170
|
+
error: BaseException | None = None
|
|
171
|
+
try:
|
|
172
|
+
result = await work(db)
|
|
173
|
+
except aiosqlite.Error as caught:
|
|
174
|
+
error = caught
|
|
175
|
+
_raise_mapped(caught, failure)
|
|
176
|
+
except BaseException as caught:
|
|
177
|
+
error = caught
|
|
178
|
+
raise
|
|
179
|
+
finally:
|
|
180
|
+
await finalize(db, error, failure=failure)
|
|
181
|
+
return result
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""The tool-facing write path: the remember, amend and retire verbs."""
|
|
2
|
+
|
|
3
|
+
from zikaron.core.write.dedup import DedupPolicy, NearDuplicate, offer
|
|
4
|
+
from zikaron.core.write.tools import (
|
|
5
|
+
Amended,
|
|
6
|
+
AmendOutcome,
|
|
7
|
+
Conflict,
|
|
8
|
+
Remembered,
|
|
9
|
+
RememberOutcome,
|
|
10
|
+
Retired,
|
|
11
|
+
RetireOutcome,
|
|
12
|
+
WriteCall,
|
|
13
|
+
amend,
|
|
14
|
+
remember,
|
|
15
|
+
retire,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
__all__ = [
|
|
19
|
+
"AmendOutcome",
|
|
20
|
+
"Amended",
|
|
21
|
+
"Conflict",
|
|
22
|
+
"DedupPolicy",
|
|
23
|
+
"NearDuplicate",
|
|
24
|
+
"RememberOutcome",
|
|
25
|
+
"Remembered",
|
|
26
|
+
"RetireOutcome",
|
|
27
|
+
"Retired",
|
|
28
|
+
"WriteCall",
|
|
29
|
+
"amend",
|
|
30
|
+
"offer",
|
|
31
|
+
"remember",
|
|
32
|
+
"retire",
|
|
33
|
+
]
|