scriptoria 0.6.4__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scriptoria-0.6.4 → scriptoria-0.8.0}/PKG-INFO +1 -1
- {scriptoria-0.6.4 → scriptoria-0.8.0}/pyproject.toml +1 -1
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/__init__.py +1 -1
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/anchors.py +21 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/facts.py +33 -10
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/lock.py +91 -38
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/conftest.py +10 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_fact_cmd.py +73 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_lock.py +87 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_verify.py +38 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/uv.lock +1 -1
- {scriptoria-0.6.4 → scriptoria-0.8.0}/.gitignore +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/README.md +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/pyrightconfig.json +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/blocks.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/cli.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/embeddings.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/errors.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/frontmatter.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/graph.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/hashing.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/ingest.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/manifest.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/pageindex_adapter.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/query.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/retrieval.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/src/scrip/similar.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_anchor_cmd.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_anchors.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_blocks.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_embeddings.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_facts.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_frontmatter.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_graph_status.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_hashing.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_index_cmd.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_ingest.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_json_shapes.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_manifest.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_new_cmd.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_pageindex_cmd.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_query.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_retrieval.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_similar_cmd.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_span_cmd.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_stamp.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_status_cmd.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_unlock_cmd.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_version.py +0 -0
- {scriptoria-0.6.4 → scriptoria-0.8.0}/tests/test_watch.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptoria
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Summary: Deterministic scriptorium-keeper (the `scrip` CLI): staleness, provenance integrity, and fact queries for an agent-compiled knowledge base
|
|
5
5
|
Project-URL: Homepage, https://github.com/coredipper/scriptorium
|
|
6
6
|
Project-URL: Changelog, https://github.com/coredipper/scriptorium/blob/main/CHANGELOG.md
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
# Distribution name on PyPI is `scriptoria` (scrip/scriptorium were taken); the
|
|
3
3
|
# CLI command and the import package both remain `scrip`.
|
|
4
4
|
name = "scriptoria"
|
|
5
|
-
version = "0.
|
|
5
|
+
version = "0.8.0"
|
|
6
6
|
description = "Deterministic scriptorium-keeper (the `scrip` CLI): staleness, provenance integrity, and fact queries for an agent-compiled knowledge base"
|
|
7
7
|
readme = "README.md"
|
|
8
8
|
requires-python = ">=3.10"
|
|
@@ -13,7 +13,7 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
from pathlib import Path
|
|
15
15
|
|
|
16
|
-
__version__ = "0.
|
|
16
|
+
__version__ = "0.8.0"
|
|
17
17
|
|
|
18
18
|
# --- canonical vault layout ------------------------------------------------
|
|
19
19
|
# ``root`` is the repo/instance root: the directory containing ``vault/``.
|
|
@@ -199,6 +199,27 @@ def verify_vault(root: Path) -> dict:
|
|
|
199
199
|
text = _source_text(root, rec["source_id"], src_cache)
|
|
200
200
|
record(f"claim:{cid}", rec, resolve(text, rec["anchor"]))
|
|
201
201
|
|
|
202
|
+
# cited edges (graph.ndjson): an edge that carries an anchor is verified like
|
|
203
|
+
# a claim; a bare {src,dst,kind} edge has no citation to resolve and is skipped.
|
|
204
|
+
edges_path = facts_dir(root) / "graph.ndjson"
|
|
205
|
+
if edges_path.exists():
|
|
206
|
+
with edges_path.open(encoding="utf-8") as f:
|
|
207
|
+
for lineno, raw_line in enumerate(f, start=1):
|
|
208
|
+
line = raw_line.strip()
|
|
209
|
+
if not line:
|
|
210
|
+
continue
|
|
211
|
+
try:
|
|
212
|
+
rec = json.loads(line)
|
|
213
|
+
except json.JSONDecodeError as e:
|
|
214
|
+
raise DataError(f"graph.ndjson:{lineno}: invalid JSON: {e}") from e
|
|
215
|
+
if "anchor" not in rec:
|
|
216
|
+
continue
|
|
217
|
+
if "source_id" not in rec:
|
|
218
|
+
raise DataError(f"graph.ndjson:{lineno}: cited edge missing 'source_id'")
|
|
219
|
+
text = _source_text(root, rec["source_id"], src_cache)
|
|
220
|
+
where = f"edge:{rec.get('src')}->{rec.get('dst')} ({rec.get('kind')})"
|
|
221
|
+
record(where, rec, resolve(text, rec["anchor"]))
|
|
222
|
+
|
|
202
223
|
wd = wiki_dir(root)
|
|
203
224
|
if wd.is_dir():
|
|
204
225
|
for path in sorted(wd.rglob("*.md")):
|
|
@@ -53,7 +53,10 @@ _CLAIM_ALLOWED = frozenset((*_CLAIM_REQUIRED, "claim_text", "tags"))
|
|
|
53
53
|
_ENTITY_REQUIRED = ("entity_id", "name", "kind")
|
|
54
54
|
_ENTITY_ALLOWED = frozenset((*_ENTITY_REQUIRED, "tags"))
|
|
55
55
|
_EDGE_REQUIRED = ("src", "dst", "kind")
|
|
56
|
-
|
|
56
|
+
# An edge may optionally be *cited*: a verbatim ``quote`` + ``source_id`` whose
|
|
57
|
+
# ``anchor`` scrip mints+verifies exactly as for a claim. Bare edges stay
|
|
58
|
+
# structural. ``anchor`` remains scrip-owned (proposing it is a schema error).
|
|
59
|
+
_EDGE_ALLOWED = frozenset((*_EDGE_REQUIRED, "quote", "source_id"))
|
|
57
60
|
_DECISIONS = ("supersede", "qualify", "keep-both")
|
|
58
61
|
_RECON_REQUIRED = ("decision", "claim_a", "claim_b")
|
|
59
62
|
_RECON_ALLOWED = frozenset((*_RECON_REQUIRED, "winner", "rationale"))
|
|
@@ -161,6 +164,17 @@ def _validate(table: str, rec: dict, index: int) -> None:
|
|
|
161
164
|
_check_shape(rec, index, _EDGE_REQUIRED, _EDGE_ALLOWED)
|
|
162
165
|
for key in _EDGE_REQUIRED:
|
|
163
166
|
_check_str(rec, key, index)
|
|
167
|
+
if "quote" in rec or "source_id" in rec:
|
|
168
|
+
# both or neither: a quote needs a source to anchor against
|
|
169
|
+
missing = [k for k in ("quote", "source_id") if k not in rec]
|
|
170
|
+
if missing:
|
|
171
|
+
raise DataError(
|
|
172
|
+
f"record {index}: a cited edge needs both 'quote' and 'source_id' "
|
|
173
|
+
f"(missing: {', '.join(missing)})"
|
|
174
|
+
)
|
|
175
|
+
_check_str(rec, "source_id", index)
|
|
176
|
+
# quote emptiness is a per-record finding (EMPTY_QUOTE), not a schema error
|
|
177
|
+
_check_str(rec, "quote", index, allow_blank=True)
|
|
164
178
|
else: # reconciliations
|
|
165
179
|
_check_shape(rec, index, _RECON_REQUIRED, _RECON_ALLOWED, owned=_RECON_OWNED)
|
|
166
180
|
for key in ("decision", "claim_a", "claim_b"):
|
|
@@ -390,13 +404,16 @@ def add(root: Path, table: str, proposals: list[dict]) -> dict:
|
|
|
390
404
|
resolved: list[dict | None] = [None] * len(proposals)
|
|
391
405
|
path = facts_dir(root) / _FILES[table]
|
|
392
406
|
with lock.write_lock(root):
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
407
|
+
# Resolve quotes INSIDE the lock: raw/ only changes via a *locked*
|
|
408
|
+
# `ingest --reingest`, so holding the lock from verification through
|
|
409
|
+
# append closes the window where a re-ingest could land between the
|
|
410
|
+
# two and silently break the just-minted anchors. Claims always carry a
|
|
411
|
+
# quote; an edge does only when it is *cited*.
|
|
412
|
+
if table == "claims" or table == "edges":
|
|
398
413
|
src_cache: dict[str, str | None] = {}
|
|
399
414
|
for i, rec in enumerate(proposals):
|
|
415
|
+
if table == "edges" and "quote" not in rec:
|
|
416
|
+
continue # bare edge: nothing to anchor
|
|
400
417
|
fail, res = _resolve_claim(root, rec, i, src_cache)
|
|
401
418
|
if fail:
|
|
402
419
|
failures.append(fail)
|
|
@@ -478,7 +495,12 @@ def add(root: Path, table: str, proposals: list[dict]) -> dict:
|
|
|
478
495
|
skipped.append({"index": i, "reason": "duplicate", "existing_id": None})
|
|
479
496
|
continue
|
|
480
497
|
seen_edges.add(key)
|
|
481
|
-
|
|
498
|
+
full = {"src": rec["src"], "dst": rec["dst"], "kind": rec["kind"]}
|
|
499
|
+
res = resolved[i]
|
|
500
|
+
if res is not None: # cited edge: carry the minted provenance
|
|
501
|
+
full["source_id"] = res["source_id"]
|
|
502
|
+
full["anchor"] = res["anchor"]
|
|
503
|
+
appended.append(full)
|
|
482
504
|
else: # reconciliations
|
|
483
505
|
claim_ids = {c.get("claim_id") for c in _read_table(facts_dir(root) / "claims.ndjson")[0]}
|
|
484
506
|
for i, rec in enumerate(proposals):
|
|
@@ -528,10 +550,11 @@ def add(root: Path, table: str, proposals: list[dict]) -> dict:
|
|
|
528
550
|
f.write("\n")
|
|
529
551
|
f.write(payload)
|
|
530
552
|
new_sources: list[str] = []
|
|
531
|
-
if table
|
|
553
|
+
if table in ("claims", "edges"):
|
|
532
554
|
for rec in appended:
|
|
533
|
-
|
|
534
|
-
|
|
555
|
+
sid = rec.get("source_id") # cited edges carry one; bare edges don't
|
|
556
|
+
if sid and sid not in new_sources:
|
|
557
|
+
new_sources.append(sid)
|
|
535
558
|
_write_meta(root, meta, new_sources)
|
|
536
559
|
|
|
537
560
|
return {"table": table, "appended": appended, "skipped": skipped, "failures": []}
|
|
@@ -8,14 +8,18 @@ writers don't interleave stamps and edits. Reads (``status``, ``verify``,
|
|
|
8
8
|
|
|
9
9
|
The lock is a small JSON file created atomically with ``O_CREAT|O_EXCL`` holding
|
|
10
10
|
``{pid, host, acquired_at}``. A lock whose holder is a dead process *on this
|
|
11
|
-
host* is **stale** and is reclaimed automatically on acquire
|
|
12
|
-
live
|
|
13
|
-
|
|
11
|
+
host* is **stale** and is reclaimed automatically on acquire. A lock that looks
|
|
12
|
+
live is **waited on**: ``acquire`` polls (with backoff) up to a timeout —
|
|
13
|
+
``SCRIP_LOCK_TIMEOUT`` seconds, default 10 — so two agents serialize cooperatively
|
|
14
|
+
rather than one failing immediately. If the lock is still held when the wait
|
|
15
|
+
elapses it fails (exit 2) and points the user at ``scrip unlock``; ``timeout=0``
|
|
16
|
+
keeps the old fail-fast behavior. A lock from another host is treated as live (we
|
|
17
|
+
can't prove the process dead), so it is waited on but never reclaimed.
|
|
14
18
|
|
|
15
19
|
It is advisory, not a kernel mutex: a tiny TOCTOU window remains when reclaiming
|
|
16
|
-
a stale lock
|
|
17
|
-
this guards; it is a guardrail against
|
|
18
|
-
distributed lock manager.
|
|
20
|
+
a stale lock, and waiting is best-effort polling, not a fair queue. That is
|
|
21
|
+
acceptable for the single-machine workflow this guards; it is a guardrail against
|
|
22
|
+
accidental concurrent writes, not a distributed lock manager.
|
|
19
23
|
"""
|
|
20
24
|
|
|
21
25
|
from __future__ import annotations
|
|
@@ -24,6 +28,7 @@ import json
|
|
|
24
28
|
import os
|
|
25
29
|
import socket
|
|
26
30
|
import tempfile
|
|
31
|
+
import time
|
|
27
32
|
from contextlib import contextmanager
|
|
28
33
|
from datetime import datetime, timezone
|
|
29
34
|
from pathlib import Path
|
|
@@ -31,6 +36,31 @@ from pathlib import Path
|
|
|
31
36
|
from . import lock_path
|
|
32
37
|
from .errors import LockError, UsageError
|
|
33
38
|
|
|
39
|
+
# How long ``acquire`` waits for a busy lock before giving up (seconds). The default
|
|
40
|
+
# is cooperative: concurrent writers serialize rather than failing fast. ``0`` keeps
|
|
41
|
+
# the old fail-fast behavior; ``SCRIP_LOCK_TIMEOUT`` overrides it. The poll interval
|
|
42
|
+
# backs off from _POLL_MIN to _POLL_MAX so a freed lock is taken promptly without
|
|
43
|
+
# busy-spinning.
|
|
44
|
+
_DEFAULT_LOCK_TIMEOUT = 10.0
|
|
45
|
+
_POLL_MIN = 0.025
|
|
46
|
+
_POLL_MAX = 0.25
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _lock_timeout(explicit: float | None) -> float:
|
|
50
|
+
"""Seconds ``acquire`` should wait for a busy lock. An explicit argument wins;
|
|
51
|
+
otherwise read ``SCRIP_LOCK_TIMEOUT`` (default :data:`_DEFAULT_LOCK_TIMEOUT`). A
|
|
52
|
+
missing or unparseable env value falls back to the default; negatives clamp to 0
|
|
53
|
+
(fail fast)."""
|
|
54
|
+
if explicit is not None:
|
|
55
|
+
return max(0.0, explicit)
|
|
56
|
+
raw = os.environ.get("SCRIP_LOCK_TIMEOUT")
|
|
57
|
+
if raw is None or not raw.strip():
|
|
58
|
+
return _DEFAULT_LOCK_TIMEOUT
|
|
59
|
+
try:
|
|
60
|
+
return max(0.0, float(raw))
|
|
61
|
+
except ValueError:
|
|
62
|
+
return _DEFAULT_LOCK_TIMEOUT
|
|
63
|
+
|
|
34
64
|
|
|
35
65
|
def _now() -> str:
|
|
36
66
|
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
@@ -129,45 +159,68 @@ def _create(path: Path, payload: bytes) -> None:
|
|
|
129
159
|
pass
|
|
130
160
|
|
|
131
161
|
|
|
132
|
-
def acquire(root: Path) -> dict:
|
|
162
|
+
def acquire(root: Path, *, timeout: float | None = None) -> dict:
|
|
133
163
|
"""Take the lock, returning the holder info. Reclaims a *provably-dead* lock
|
|
134
|
-
|
|
135
|
-
|
|
164
|
+
immediately. If the lock looks live (or is mid-creation by another writer),
|
|
165
|
+
*waits* — polling with backoff — up to ``timeout`` seconds for it to free, then
|
|
166
|
+
raises :class:`LockError` (exit 2) if it is still held. ``timeout`` defaults to
|
|
167
|
+
``SCRIP_LOCK_TIMEOUT`` (``10`` s); ``timeout=0`` fails fast (the old behavior).
|
|
168
|
+
A holder that dies *during* the wait is reclaimed on the next poll."""
|
|
136
169
|
p = lock_path(root)
|
|
137
170
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
138
171
|
info = _holder_info()
|
|
139
172
|
payload = (json.dumps(info, ensure_ascii=False) + "\n").encode("utf-8")
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
except FileExistsError:
|
|
144
|
-
pass
|
|
173
|
+
wait = _lock_timeout(timeout)
|
|
174
|
+
deadline = time.monotonic() + wait
|
|
175
|
+
poll = _POLL_MIN
|
|
145
176
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
177
|
+
def _try_take() -> bool:
|
|
178
|
+
"""One acquisition attempt: True if we now hold the lock. A provably-dead
|
|
179
|
+
lock on this host is reclaimed and retried once; otherwise we don't hold it."""
|
|
180
|
+
try:
|
|
181
|
+
_create(p, payload)
|
|
182
|
+
return True
|
|
183
|
+
except FileExistsError:
|
|
184
|
+
pass
|
|
185
|
+
if _reclaimable(_read(p)):
|
|
186
|
+
try:
|
|
187
|
+
p.unlink()
|
|
188
|
+
except FileNotFoundError:
|
|
189
|
+
pass
|
|
190
|
+
try:
|
|
191
|
+
_create(p, payload)
|
|
192
|
+
return True
|
|
193
|
+
except FileExistsError:
|
|
194
|
+
pass # raced with another writer; fall through to wait/retry
|
|
195
|
+
return False
|
|
157
196
|
|
|
158
|
-
#
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
197
|
+
# Always attempt once (so a free lock is taken even at timeout=0), then poll —
|
|
198
|
+
# re-checking the deadline both before sleeping and again after, so we never
|
|
199
|
+
# retry (and grab) a lock that freed only once the budget was already spent.
|
|
200
|
+
if _try_take():
|
|
201
|
+
return info
|
|
202
|
+
while True:
|
|
203
|
+
remaining = deadline - time.monotonic()
|
|
204
|
+
if remaining <= 0:
|
|
205
|
+
break
|
|
206
|
+
time.sleep(min(poll, remaining)) # remaining > 0 here, so never negative
|
|
207
|
+
poll = min(poll * 2, _POLL_MAX)
|
|
208
|
+
if time.monotonic() >= deadline:
|
|
209
|
+
break
|
|
210
|
+
if _try_take():
|
|
211
|
+
return info
|
|
212
|
+
|
|
213
|
+
existing = _read(p) # who holds it now, for the diagnostic
|
|
214
|
+
waited = f" after waiting {wait:g}s" if wait else ""
|
|
215
|
+
if existing is None:
|
|
166
216
|
raise LockError(
|
|
167
|
-
"
|
|
168
|
-
"`scrip unlock` if it is stuck."
|
|
217
|
+
".kb/lock is held but not yet readable (another writer may be "
|
|
218
|
+
f"acquiring it){waited}. Retry; run `scrip unlock --force` if it is stuck."
|
|
169
219
|
) from None
|
|
170
|
-
|
|
220
|
+
raise LockError(
|
|
221
|
+
f"vault is locked by another writer ({_describe(existing)}){waited}. "
|
|
222
|
+
f"Wait for it to finish, or run `scrip unlock --force` if it is stuck."
|
|
223
|
+
) from None
|
|
171
224
|
|
|
172
225
|
|
|
173
226
|
def release(root: Path, info: dict | None = None) -> None:
|
|
@@ -187,8 +240,8 @@ def release(root: Path, info: dict | None = None) -> None:
|
|
|
187
240
|
|
|
188
241
|
|
|
189
242
|
@contextmanager
|
|
190
|
-
def write_lock(root: Path):
|
|
191
|
-
info = acquire(root)
|
|
243
|
+
def write_lock(root: Path, *, timeout: float | None = None):
|
|
244
|
+
info = acquire(root, timeout=timeout)
|
|
192
245
|
try:
|
|
193
246
|
yield info
|
|
194
247
|
finally:
|
|
@@ -16,6 +16,16 @@ import pytest
|
|
|
16
16
|
from scrip import anchors, blocks, frontmatter, hashing
|
|
17
17
|
|
|
18
18
|
|
|
19
|
+
@pytest.fixture(autouse=True)
|
|
20
|
+
def _fail_fast_lock(monkeypatch):
|
|
21
|
+
"""Keep the hermetic suite fast: the write lock now *waits* for a busy lock
|
|
22
|
+
(``SCRIP_LOCK_TIMEOUT``, default 10s). Pin it to 0 (fail-fast) for tests so the
|
|
23
|
+
lock-blocked cases return immediately instead of sitting through the wait. Tests
|
|
24
|
+
of the waiting path pass an explicit ``timeout=`` to :func:`scrip.lock.acquire`,
|
|
25
|
+
which overrides this env default."""
|
|
26
|
+
monkeypatch.setenv("SCRIP_LOCK_TIMEOUT", "0")
|
|
27
|
+
|
|
28
|
+
|
|
19
29
|
class KB:
|
|
20
30
|
def __init__(self, root: Path):
|
|
21
31
|
self.root = root
|
|
@@ -31,6 +31,12 @@ def _claim(quote, source="raw/s", **kw):
|
|
|
31
31
|
return rec
|
|
32
32
|
|
|
33
33
|
|
|
34
|
+
def _edge(src="entity/a", dst="entity/b", kind="relates-to", **kw):
|
|
35
|
+
rec = {"src": src, "dst": dst, "kind": kind}
|
|
36
|
+
rec.update(kw)
|
|
37
|
+
return rec
|
|
38
|
+
|
|
39
|
+
|
|
34
40
|
def _ndjson(*recs):
|
|
35
41
|
return "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in recs)
|
|
36
42
|
|
|
@@ -55,6 +61,13 @@ def _recs_lines(kb):
|
|
|
55
61
|
return [json.loads(s) for s in p.read_text(encoding="utf-8").splitlines() if s.strip()]
|
|
56
62
|
|
|
57
63
|
|
|
64
|
+
def _graph_lines(kb):
|
|
65
|
+
p = kb.root / "vault" / "facts" / "graph.ndjson"
|
|
66
|
+
if not p.exists():
|
|
67
|
+
return []
|
|
68
|
+
return [json.loads(s) for s in p.read_text(encoding="utf-8").splitlines() if s.strip()]
|
|
69
|
+
|
|
70
|
+
|
|
58
71
|
def _two_claims(kb):
|
|
59
72
|
"""Seed a contradiction pair to reconcile."""
|
|
60
73
|
kb.add_raw("s", SRC)
|
|
@@ -476,3 +489,63 @@ def test_fact_add_edges_appends_and_skips_duplicates(kb):
|
|
|
476
489
|
assert _run_add(kb, _ndjson(edge), "--table", "edges") == 0
|
|
477
490
|
lines = (kb.root / "vault" / "facts" / "graph.ndjson").read_text(encoding="utf-8")
|
|
478
491
|
assert len(lines.splitlines()) == 1
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
# --------------------------------------------------------------------------- #
|
|
495
|
+
# Cited edges: an edge may carry a verbatim quote + source_id; scrip mints and
|
|
496
|
+
# verifies an anchor for it exactly as it does for a claim. Bare edges still work.
|
|
497
|
+
# --------------------------------------------------------------------------- #
|
|
498
|
+
def test_fact_add_bare_edge_carries_no_provenance(kb):
|
|
499
|
+
# additivity guard: an edge with no quote stays purely structural
|
|
500
|
+
assert _run_add(kb, _ndjson(_edge()), "--table", "edges") == 0
|
|
501
|
+
[edge] = _graph_lines(kb)
|
|
502
|
+
assert set(edge) == {"src", "dst", "kind"}
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def test_fact_add_cited_edge_mints_and_verifies_anchor(kb):
|
|
506
|
+
kb.add_raw("s", SRC)
|
|
507
|
+
rc = _run_add(
|
|
508
|
+
kb,
|
|
509
|
+
_ndjson(_edge(quote="Caching answers beats recomputing them.", source_id="raw/s")),
|
|
510
|
+
"--table", "edges",
|
|
511
|
+
)
|
|
512
|
+
assert rc == 0
|
|
513
|
+
[edge] = _graph_lines(kb)
|
|
514
|
+
assert (edge["src"], edge["dst"], edge["kind"]) == ("entity/a", "entity/b", "relates-to")
|
|
515
|
+
assert edge["source_id"] == "raw/s"
|
|
516
|
+
# the minted anchor resolves uniquely in the source, and scrip verify confirms it
|
|
517
|
+
assert anchors.resolve(SRC, edge["anchor"]) == "OK"
|
|
518
|
+
assert cli.main(["verify", "--root", str(kb.root)]) == 0
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
def test_fact_add_cited_edge_broken_quote_fails_batch(kb, capsys):
|
|
522
|
+
kb.add_raw("s", SRC)
|
|
523
|
+
rc = _run_add(
|
|
524
|
+
kb,
|
|
525
|
+
_ndjson(_edge(quote="this sentence is absent from the source", source_id="raw/s")),
|
|
526
|
+
"--table", "edges", "--json",
|
|
527
|
+
)
|
|
528
|
+
assert rc == 1
|
|
529
|
+
assert _graph_lines(kb) == [] # all-or-nothing: nothing written
|
|
530
|
+
[failure] = json.loads(capsys.readouterr().out)["failures"]
|
|
531
|
+
assert failure["status"] == "BROKEN"
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def test_fact_add_cited_edge_requires_both_quote_and_source_id(kb):
|
|
535
|
+
kb.add_raw("s", SRC)
|
|
536
|
+
# a quote needs a source to anchor against, and vice versa — both or neither
|
|
537
|
+
assert _run_add(kb, _ndjson(_edge(quote="Caching answers beats recomputing them.")),
|
|
538
|
+
"--table", "edges") == 3
|
|
539
|
+
assert _run_add(kb, _ndjson(_edge(source_id="raw/s")), "--table", "edges") == 3
|
|
540
|
+
assert _graph_lines(kb) == []
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def test_fact_add_cited_edge_dedups_on_triple(kb, capsys):
|
|
544
|
+
kb.add_raw("s", SRC)
|
|
545
|
+
cited = _ndjson(_edge(quote="Caching answers beats recomputing them.", source_id="raw/s"))
|
|
546
|
+
assert _run_add(kb, cited, "--table", "edges") == 0
|
|
547
|
+
capsys.readouterr()
|
|
548
|
+
# same (src,dst,kind) again — even bare — is the same relation, so it's skipped
|
|
549
|
+
assert _run_add(kb, _ndjson(_edge()), "--table", "edges", "--json") == 0
|
|
550
|
+
assert len(_graph_lines(kb)) == 1
|
|
551
|
+
assert json.loads(capsys.readouterr().out)["skipped"][0]["reason"] == "duplicate"
|
|
@@ -6,6 +6,8 @@ import os
|
|
|
6
6
|
import socket
|
|
7
7
|
import stat
|
|
8
8
|
import subprocess
|
|
9
|
+
import threading
|
|
10
|
+
import time
|
|
9
11
|
|
|
10
12
|
import pytest
|
|
11
13
|
|
|
@@ -85,6 +87,91 @@ def test_release_only_removes_our_own_lock(tmp_path):
|
|
|
85
87
|
lock_path(tmp_path).unlink()
|
|
86
88
|
|
|
87
89
|
|
|
90
|
+
# --- waiting (cooperative blocking acquire) ---------------------------------
|
|
91
|
+
def test_acquire_timeout_zero_fails_fast_on_live_lock(tmp_path):
|
|
92
|
+
info = lock.acquire(tmp_path, timeout=0)
|
|
93
|
+
try:
|
|
94
|
+
start = time.monotonic()
|
|
95
|
+
with pytest.raises(errors.LockError) as ei:
|
|
96
|
+
lock.acquire(tmp_path, timeout=0)
|
|
97
|
+
assert ei.value.exit_code == 2
|
|
98
|
+
assert time.monotonic() - start < 0.2 # immediate — no wait at timeout=0
|
|
99
|
+
finally:
|
|
100
|
+
lock.release(tmp_path, info)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_acquire_waits_for_a_busy_lock_then_succeeds(tmp_path):
|
|
104
|
+
# a writer holds the lock briefly; a waiting acquire grabs it once it frees,
|
|
105
|
+
# rather than failing fast.
|
|
106
|
+
info = lock.acquire(tmp_path, timeout=0)
|
|
107
|
+
|
|
108
|
+
def _release_after():
|
|
109
|
+
time.sleep(0.2)
|
|
110
|
+
lock.release(tmp_path, info)
|
|
111
|
+
|
|
112
|
+
t = threading.Thread(target=_release_after)
|
|
113
|
+
t.start()
|
|
114
|
+
try:
|
|
115
|
+
start = time.monotonic()
|
|
116
|
+
info2 = lock.acquire(tmp_path, timeout=5) # blocks ~0.2s, then acquires
|
|
117
|
+
waited = time.monotonic() - start
|
|
118
|
+
assert waited >= 0.15 # it actually waited for the release
|
|
119
|
+
assert json.loads(lock_path(tmp_path).read_text())["pid"] == os.getpid()
|
|
120
|
+
lock.release(tmp_path, info2)
|
|
121
|
+
finally:
|
|
122
|
+
t.join()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def test_acquire_times_out_while_lock_stays_held(tmp_path):
|
|
126
|
+
info = lock.acquire(tmp_path, timeout=0)
|
|
127
|
+
try:
|
|
128
|
+
start = time.monotonic()
|
|
129
|
+
with pytest.raises(errors.LockError) as ei:
|
|
130
|
+
lock.acquire(tmp_path, timeout=0.3)
|
|
131
|
+
waited = time.monotonic() - start
|
|
132
|
+
assert ei.value.exit_code == 2
|
|
133
|
+
assert 0.25 <= waited < 3.0 # waited ~the timeout, then gave up (not forever)
|
|
134
|
+
finally:
|
|
135
|
+
lock.release(tmp_path, info)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_acquire_does_not_grab_a_lock_freed_after_the_deadline(tmp_path, monkeypatch):
|
|
139
|
+
# A writer releases only once the timeout has elapsed. acquire must time out
|
|
140
|
+
# (exit 2), not wake from its last sleep, retry, and grab the just-freed lock —
|
|
141
|
+
# which would silently extend the budget past SCRIP_LOCK_TIMEOUT. Driven by a
|
|
142
|
+
# mocked clock so the boundary is deterministic (real-time races are not).
|
|
143
|
+
info = lock.acquire(tmp_path, timeout=0) # our live pid holds it
|
|
144
|
+
clock = {"t": 100.0}
|
|
145
|
+
sleeps: list[float] = []
|
|
146
|
+
monkeypatch.setattr(lock.time, "monotonic", lambda: clock["t"])
|
|
147
|
+
|
|
148
|
+
def fake_sleep(d):
|
|
149
|
+
sleeps.append(d)
|
|
150
|
+
clock["t"] += d
|
|
151
|
+
if clock["t"] >= 100.3 and lock_path(tmp_path).exists():
|
|
152
|
+
lock.release(tmp_path, info) # freed, but only at/after the deadline
|
|
153
|
+
|
|
154
|
+
monkeypatch.setattr(lock.time, "sleep", fake_sleep)
|
|
155
|
+
try:
|
|
156
|
+
with pytest.raises(errors.LockError) as ei:
|
|
157
|
+
lock.acquire(tmp_path, timeout=0.3) # deadline = 100.0 + 0.3
|
|
158
|
+
assert ei.value.exit_code == 2
|
|
159
|
+
assert all(d >= 0 for d in sleeps) # never a negative sleep duration
|
|
160
|
+
finally:
|
|
161
|
+
if lock_path(tmp_path).exists():
|
|
162
|
+
lock.release(tmp_path, info)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def test_lock_timeout_resolution(monkeypatch):
|
|
166
|
+
monkeypatch.delenv("SCRIP_LOCK_TIMEOUT", raising=False)
|
|
167
|
+
assert lock._lock_timeout(None) == 10.0 # default when unset
|
|
168
|
+
monkeypatch.setenv("SCRIP_LOCK_TIMEOUT", "3.5")
|
|
169
|
+
assert lock._lock_timeout(None) == 3.5 # env configures the default
|
|
170
|
+
assert lock._lock_timeout(0) == 0.0 # an explicit timeout wins over the env
|
|
171
|
+
monkeypatch.setenv("SCRIP_LOCK_TIMEOUT", "nonsense")
|
|
172
|
+
assert lock._lock_timeout(None) == 10.0 # a bad env value falls back to default
|
|
173
|
+
|
|
174
|
+
|
|
88
175
|
# --- staleness --------------------------------------------------------------
|
|
89
176
|
def test_leftover_temp_from_recycled_pid_does_not_block_acquire(tmp_path, monkeypatch):
|
|
90
177
|
"""A temp file left by a crashed process must not be mistaken for a held lock
|
|
@@ -1,9 +1,17 @@
|
|
|
1
|
+
import json
|
|
2
|
+
|
|
1
3
|
import pytest
|
|
2
4
|
from scrip.errors import DataError
|
|
3
5
|
|
|
4
6
|
from scrip import anchors, cli
|
|
5
7
|
|
|
6
8
|
|
|
9
|
+
def _write_edge(kb, rec):
|
|
10
|
+
p = kb.root / "vault" / "facts" / "graph.ndjson"
|
|
11
|
+
with open(p, "a", encoding="utf-8") as f:
|
|
12
|
+
f.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
|
13
|
+
|
|
14
|
+
|
|
7
15
|
def test_cli_verify_fails_on_ambiguous_by_default(kb):
|
|
8
16
|
# a quote that occurs twice in the source -> AMBIGUOUS
|
|
9
17
|
kb.add_raw("a", "# A\n\nalpha beta gamma. alpha beta gamma.\n")
|
|
@@ -62,3 +70,33 @@ def test_verify_wiki_footnote_anchor(kb):
|
|
|
62
70
|
res = anchors.verify_vault(kb.root)
|
|
63
71
|
assert res["ok"] == 1
|
|
64
72
|
assert res["broken"] == []
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_verify_cited_edge_anchor_resolves(kb):
|
|
76
|
+
src = "# A\n\nThe sky is blue today over the hills.\n"
|
|
77
|
+
kb.add_raw("a", src)
|
|
78
|
+
anchor = anchors.make_anchor(src, "the sky is blue today")
|
|
79
|
+
_write_edge(kb, {"src": "entity/sky", "dst": "entity/color", "kind": "is",
|
|
80
|
+
"source_id": "raw/a", "anchor": anchor})
|
|
81
|
+
res = anchors.verify_vault(kb.root)
|
|
82
|
+
assert res["ok"] == 1
|
|
83
|
+
assert res["broken"] == []
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def test_verify_broken_cited_edge_anchor_is_listed(kb):
|
|
87
|
+
kb.add_raw("a", "# A\n\nThe sky is blue.\n")
|
|
88
|
+
bogus = "qh:" + ("0" * 64) + "|loc:0.0|len:10"
|
|
89
|
+
_write_edge(kb, {"src": "entity/a", "dst": "entity/b", "kind": "rel",
|
|
90
|
+
"source_id": "raw/a", "anchor": bogus})
|
|
91
|
+
res = anchors.verify_vault(kb.root)
|
|
92
|
+
assert len(res["broken"]) == 1
|
|
93
|
+
assert res["broken"][0]["where"].startswith("edge:")
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_verify_bare_edge_is_not_checked(kb):
|
|
97
|
+
# an uncited edge has nothing to resolve, so verify ignores it (additive)
|
|
98
|
+
kb.add_raw("a", "# A\n\nThe sky is blue.\n")
|
|
99
|
+
_write_edge(kb, {"src": "entity/a", "dst": "entity/b", "kind": "rel"})
|
|
100
|
+
res = anchors.verify_vault(kb.root)
|
|
101
|
+
assert res["checked"] == 0
|
|
102
|
+
assert res["broken"] == []
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|