scriptoria 0.8.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scriptoria-0.8.0 → scriptoria-0.9.0}/.gitignore +4 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/PKG-INFO +2 -1
- {scriptoria-0.8.0 → scriptoria-0.9.0}/README.md +1 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/pyproject.toml +1 -1
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/__init__.py +1 -1
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/cli.py +26 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/facts.py +52 -5
- scriptoria-0.9.0/src/scrip/ontology.py +162 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_fact_cmd.py +130 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_facts.py +14 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/uv.lock +2 -2
- {scriptoria-0.8.0 → scriptoria-0.9.0}/pyrightconfig.json +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/anchors.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/blocks.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/embeddings.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/errors.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/frontmatter.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/graph.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/hashing.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/ingest.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/lock.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/manifest.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/pageindex_adapter.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/query.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/retrieval.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/src/scrip/similar.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/conftest.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_anchor_cmd.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_anchors.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_blocks.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_embeddings.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_frontmatter.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_graph_status.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_hashing.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_index_cmd.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_ingest.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_json_shapes.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_lock.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_manifest.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_new_cmd.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_pageindex_cmd.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_query.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_retrieval.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_similar_cmd.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_span_cmd.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_stamp.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_status_cmd.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_unlock_cmd.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_verify.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_version.py +0 -0
- {scriptoria-0.8.0 → scriptoria-0.9.0}/tests/test_watch.py +0 -0
|
@@ -15,6 +15,10 @@ build/
|
|
|
15
15
|
# OS
|
|
16
16
|
.DS_Store
|
|
17
17
|
|
|
18
|
+
# Secrets: harness wiki-publish credentials (gh_username/password/wikirepo).
|
|
19
|
+
# Never commit. The real file lives with the private KB, not in this repo.
|
|
20
|
+
.env
|
|
21
|
+
|
|
18
22
|
# Embeddings index: a large, regenerable binary cache (rebuild with `scrip index`)
|
|
19
23
|
/.kb/embeddings/
|
|
20
24
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: scriptoria
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Deterministic scriptorium-keeper (the `scrip` CLI): staleness, provenance integrity, and fact queries for an agent-compiled knowledge base
|
|
5
5
|
Project-URL: Homepage, https://github.com/coredipper/scriptorium
|
|
6
6
|
Project-URL: Changelog, https://github.com/coredipper/scriptorium/blob/main/CHANGELOG.md
|
|
@@ -54,6 +54,7 @@ uv run --project scrip scrip --help
|
|
|
54
54
|
| `scrip status` | Report `STALE` / `OK` / `UNCOMPILED` artifacts from the dependency graph. `--no-cache` recomputes from files; `--rebuild-manifest` regenerates the cache. |
|
|
55
55
|
| `scrip verify` | Check every provenance anchor still resolves to text in its source; check referenced sources exist and `claim_id`s are unique. Fails on `BROKEN` and `AMBIGUOUS` by default; `--allow-ambiguous` downgrades `AMBIGUOUS` to a warning. |
|
|
56
56
|
| `scrip query [claims\|entities\|edges\|contradictions\|reconciliations]` | Structured query over `vault/facts/*.ndjson` via DuckDB. `--sql "<duckdb>"`, `--where`, `--limit`. |
|
|
57
|
+
| `scrip ontology` | Validate and summarize optional `vault/ontology.yaml` vocabulary for fact predicates, entity kinds, and edge kinds. |
|
|
57
58
|
| `scrip ingest <url\|file>` | Fetch/read a source and write canonical `vault/raw/<slug>.md` plus sidecar metadata. HTML/PDF need the optional `[ingest]` extra. |
|
|
58
59
|
| `scrip new concept\|entity <slug> --from raw/...` | Scaffold a derived wiki page for the agent to fill. |
|
|
59
60
|
| `scrip anchor "<quote>" --source raw/<slug>` | Mint a content-anchored footnote for a verbatim source quote. |
|
|
@@ -23,6 +23,7 @@ uv run --project scrip scrip --help
|
|
|
23
23
|
| `scrip status` | Report `STALE` / `OK` / `UNCOMPILED` artifacts from the dependency graph. `--no-cache` recomputes from files; `--rebuild-manifest` regenerates the cache. |
|
|
24
24
|
| `scrip verify` | Check every provenance anchor still resolves to text in its source; check referenced sources exist and `claim_id`s are unique. Fails on `BROKEN` and `AMBIGUOUS` by default; `--allow-ambiguous` downgrades `AMBIGUOUS` to a warning. |
|
|
25
25
|
| `scrip query [claims\|entities\|edges\|contradictions\|reconciliations]` | Structured query over `vault/facts/*.ndjson` via DuckDB. `--sql "<duckdb>"`, `--where`, `--limit`. |
|
|
26
|
+
| `scrip ontology` | Validate and summarize optional `vault/ontology.yaml` vocabulary for fact predicates, entity kinds, and edge kinds. |
|
|
26
27
|
| `scrip ingest <url\|file>` | Fetch/read a source and write canonical `vault/raw/<slug>.md` plus sidecar metadata. HTML/PDF need the optional `[ingest]` extra. |
|
|
27
28
|
| `scrip new concept\|entity <slug> --from raw/...` | Scaffold a derived wiki page for the agent to fill. |
|
|
28
29
|
| `scrip anchor "<quote>" --source raw/<slug>` | Mint a content-anchored footnote for a verbatim source quote. |
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
# Distribution name on PyPI is `scriptoria` (scrip/scriptorium were taken); the
|
|
3
3
|
# CLI command and the import package both remain `scrip`.
|
|
4
4
|
name = "scriptoria"
|
|
5
|
-
version = "0.
|
|
5
|
+
version = "0.9.0"
|
|
6
6
|
description = "Deterministic scriptorium-keeper (the `scrip` CLI): staleness, provenance integrity, and fact queries for an agent-compiled knowledge base"
|
|
7
7
|
readme = "README.md"
|
|
8
8
|
requires-python = ">=3.10"
|
|
@@ -13,7 +13,7 @@ from __future__ import annotations
|
|
|
13
13
|
|
|
14
14
|
from pathlib import Path
|
|
15
15
|
|
|
16
|
-
__version__ = "0.
|
|
16
|
+
__version__ = "0.9.0"
|
|
17
17
|
|
|
18
18
|
# --- canonical vault layout ------------------------------------------------
|
|
19
19
|
# ``root`` is the repo/instance root: the directory containing ``vault/``.
|
|
@@ -493,6 +493,25 @@ def cmd_fact_add(args: argparse.Namespace) -> int:
|
|
|
493
493
|
return 1 if result["failures"] else 0
|
|
494
494
|
|
|
495
495
|
|
|
496
|
+
def cmd_ontology(args: argparse.Namespace) -> int:
|
|
497
|
+
from . import ontology
|
|
498
|
+
|
|
499
|
+
root = resolve_root(args.root)
|
|
500
|
+
ont = ontology.load(root)
|
|
501
|
+
result = {"path": str(ontology.ontology_path(root).relative_to(root)), **ont.summary()}
|
|
502
|
+
if args.json:
|
|
503
|
+
_emit(result)
|
|
504
|
+
else:
|
|
505
|
+
state = "active" if ont.active else "not present or empty"
|
|
506
|
+
print(f"ontology: {state}")
|
|
507
|
+
if ont.active:
|
|
508
|
+
print(f" entity kinds: {', '.join(result['entity_kinds']) or '(none)'}")
|
|
509
|
+
print(f" edge kinds: {', '.join(result['edge_kinds']) or '(none)'}")
|
|
510
|
+
print(f" claim predicates: {', '.join(result['claim_predicates']) or '(none)'}")
|
|
511
|
+
print(f" predicate aliases: {len(result['predicate_aliases'])}")
|
|
512
|
+
return 0
|
|
513
|
+
|
|
514
|
+
|
|
496
515
|
def cmd_ingest(args: argparse.Namespace) -> int:
|
|
497
516
|
from . import ingest, lock
|
|
498
517
|
|
|
@@ -764,6 +783,13 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
764
783
|
)
|
|
765
784
|
pfa.set_defaults(func=cmd_fact_add)
|
|
766
785
|
|
|
786
|
+
po = sub.add_parser(
|
|
787
|
+
"ontology",
|
|
788
|
+
parents=[common],
|
|
789
|
+
help="validate and summarize optional vault/ontology.yaml vocabulary",
|
|
790
|
+
)
|
|
791
|
+
po.set_defaults(func=cmd_ontology)
|
|
792
|
+
|
|
767
793
|
pin = sub.add_parser(
|
|
768
794
|
"ingest",
|
|
769
795
|
parents=[common],
|
|
@@ -21,6 +21,7 @@ from __future__ import annotations
|
|
|
21
21
|
|
|
22
22
|
import json
|
|
23
23
|
import re
|
|
24
|
+
from collections.abc import Iterator
|
|
24
25
|
from datetime import datetime, timezone
|
|
25
26
|
from pathlib import Path
|
|
26
27
|
|
|
@@ -33,6 +34,8 @@ except ImportError:
|
|
|
33
34
|
|
|
34
35
|
from . import anchors, facts_dir, lock
|
|
35
36
|
from .errors import DataError, UsageError
|
|
37
|
+
from .ontology import Ontology
|
|
38
|
+
from .ontology import load as load_ontology
|
|
36
39
|
|
|
37
40
|
_POLARITIES = ("asserts", "denies", "qualifies")
|
|
38
41
|
|
|
@@ -51,7 +54,7 @@ _RECON_OWNED = ("reconciliation_id", "at")
|
|
|
51
54
|
_CLAIM_REQUIRED = ("quote", "source_id", "subject", "predicate", "object", "polarity", "confidence")
|
|
52
55
|
_CLAIM_ALLOWED = frozenset((*_CLAIM_REQUIRED, "claim_text", "tags"))
|
|
53
56
|
_ENTITY_REQUIRED = ("entity_id", "name", "kind")
|
|
54
|
-
_ENTITY_ALLOWED = frozenset((*_ENTITY_REQUIRED, "tags"))
|
|
57
|
+
_ENTITY_ALLOWED = frozenset((*_ENTITY_REQUIRED, "tags", "uri", "same_as", "external_ids"))
|
|
55
58
|
_EDGE_REQUIRED = ("src", "dst", "kind")
|
|
56
59
|
# An edge may optionally be *cited*: a verbatim ``quote`` + ``source_id`` whose
|
|
57
60
|
# ``anchor`` scrip mints+verifies exactly as for a claim. Bare edges stay
|
|
@@ -76,6 +79,18 @@ def _now() -> str:
|
|
|
76
79
|
# --------------------------------------------------------------------------- #
|
|
77
80
|
# Input parsing & structural validation (DataError, exit 3)
|
|
78
81
|
# --------------------------------------------------------------------------- #
|
|
82
|
+
def _iter_lf_lines(text: str) -> Iterator[str]:
|
|
83
|
+
"""Yield ``text.split("\n")``-equivalent lines without allocating the list."""
|
|
84
|
+
start = 0
|
|
85
|
+
while True:
|
|
86
|
+
end = text.find("\n", start)
|
|
87
|
+
if end == -1:
|
|
88
|
+
yield text[start:]
|
|
89
|
+
return
|
|
90
|
+
yield text[start:end]
|
|
91
|
+
start = end + 1
|
|
92
|
+
|
|
93
|
+
|
|
79
94
|
def parse_ndjson(text: str) -> list[dict]:
|
|
80
95
|
"""Parse proposed records (one JSON object per line). Malformed input is a
|
|
81
96
|
:class:`DataError` with its line number; an empty input is a usage error."""
|
|
@@ -84,7 +99,7 @@ def parse_ndjson(text: str) -> list[dict]:
|
|
|
84
99
|
# delimited and may legally contain U+2028/U+2029/NEL inside a JSON string,
|
|
85
100
|
# which splitlines() would wrongly treat as record breaks. trailing \r (from
|
|
86
101
|
# \r\n) and the trailing empty element from a final newline are dropped below.
|
|
87
|
-
for lineno, raw_line in enumerate(text
|
|
102
|
+
for lineno, raw_line in enumerate(_iter_lf_lines(text), start=1):
|
|
88
103
|
line = raw_line.strip()
|
|
89
104
|
if not line:
|
|
90
105
|
continue
|
|
@@ -114,6 +129,26 @@ def _check_tags(rec: dict, index: int) -> None:
|
|
|
114
129
|
raise DataError(f"record {index}: 'tags' must be a list of strings")
|
|
115
130
|
|
|
116
131
|
|
|
132
|
+
def _check_string_list(rec: dict, key: str, index: int) -> None:
|
|
133
|
+
values = rec.get(key)
|
|
134
|
+
if values is None:
|
|
135
|
+
return
|
|
136
|
+
if not isinstance(values, list) or any(not isinstance(v, str) or not v.strip() for v in values):
|
|
137
|
+
raise DataError(f"record {index}: '{key}' must be a list of non-empty strings")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _check_string_map(rec: dict, key: str, index: int) -> None:
|
|
141
|
+
values = rec.get(key)
|
|
142
|
+
if values is None:
|
|
143
|
+
return
|
|
144
|
+
if (
|
|
145
|
+
not isinstance(values, dict)
|
|
146
|
+
or any(not isinstance(k, str) or not k.strip() for k in values)
|
|
147
|
+
or any(not isinstance(v, str) or not v.strip() for v in values.values())
|
|
148
|
+
):
|
|
149
|
+
raise DataError(f"record {index}: '{key}' must be a mapping of strings to strings")
|
|
150
|
+
|
|
151
|
+
|
|
117
152
|
def _check_shape(
|
|
118
153
|
rec: dict,
|
|
119
154
|
index: int,
|
|
@@ -135,13 +170,14 @@ def _check_shape(
|
|
|
135
170
|
raise DataError(f"record {index}: missing required field(s): {', '.join(missing)}")
|
|
136
171
|
|
|
137
172
|
|
|
138
|
-
def _validate(table: str, rec: dict, index: int) -> None:
|
|
173
|
+
def _validate(table: str, rec: dict, index: int, ont: Ontology) -> None:
|
|
139
174
|
if table == "claims":
|
|
140
175
|
_check_shape(rec, index, _CLAIM_REQUIRED, _CLAIM_ALLOWED)
|
|
141
176
|
# the quote's *emptiness* is a per-record finding, not a schema error
|
|
142
177
|
_check_str(rec, "quote", index, allow_blank=True)
|
|
143
178
|
for key in ("source_id", "subject", "predicate", "object"):
|
|
144
179
|
_check_str(rec, key, index)
|
|
180
|
+
rec["predicate"] = ont.canonical_claim_predicate(rec["predicate"], index)
|
|
145
181
|
if "claim_text" in rec:
|
|
146
182
|
_check_str(rec, "claim_text", index, allow_blank=True)
|
|
147
183
|
if rec["polarity"] not in _POLARITIES:
|
|
@@ -156,14 +192,20 @@ def _validate(table: str, rec: dict, index: int) -> None:
|
|
|
156
192
|
_check_shape(rec, index, _ENTITY_REQUIRED, _ENTITY_ALLOWED)
|
|
157
193
|
for key in _ENTITY_REQUIRED:
|
|
158
194
|
_check_str(rec, key, index)
|
|
195
|
+
ont.validate_entity_kind(rec["kind"], index)
|
|
159
196
|
eid = rec["entity_id"]
|
|
160
197
|
if not (eid.startswith("entity/") and _SLUG_RE.fullmatch(eid[len("entity/") :])):
|
|
161
198
|
raise DataError(f"record {index}: entity_id must look like entity/<slug>")
|
|
199
|
+
if "uri" in rec:
|
|
200
|
+
_check_str(rec, "uri", index)
|
|
201
|
+
_check_string_list(rec, "same_as", index)
|
|
202
|
+
_check_string_map(rec, "external_ids", index)
|
|
162
203
|
_check_tags(rec, index)
|
|
163
204
|
elif table == "edges":
|
|
164
205
|
_check_shape(rec, index, _EDGE_REQUIRED, _EDGE_ALLOWED)
|
|
165
206
|
for key in _EDGE_REQUIRED:
|
|
166
207
|
_check_str(rec, key, index)
|
|
208
|
+
ont.validate_edge_kind(rec["kind"], index)
|
|
167
209
|
if "quote" in rec or "source_id" in rec:
|
|
168
210
|
# both or neither: a quote needs a source to anchor against
|
|
169
211
|
missing = [k for k in ("quote", "source_id") if k not in rec]
|
|
@@ -397,8 +439,9 @@ def add(root: Path, table: str, proposals: list[dict]) -> dict:
|
|
|
397
439
|
"""
|
|
398
440
|
if table not in _FILES:
|
|
399
441
|
raise UsageError(f"unknown facts table: {table}")
|
|
442
|
+
ont = load_ontology(root)
|
|
400
443
|
for i, rec in enumerate(proposals):
|
|
401
|
-
_validate(table, rec, i)
|
|
444
|
+
_validate(table, rec, i, ont)
|
|
402
445
|
|
|
403
446
|
failures: list[dict] = []
|
|
404
447
|
resolved: list[dict | None] = [None] * len(proposals)
|
|
@@ -457,12 +500,16 @@ def add(root: Path, table: str, proposals: list[dict]) -> dict:
|
|
|
457
500
|
appended.append(full)
|
|
458
501
|
elif table == "entities":
|
|
459
502
|
def canon(rec: dict) -> dict:
|
|
460
|
-
|
|
503
|
+
out = {
|
|
461
504
|
"entity_id": rec["entity_id"],
|
|
462
505
|
"name": rec["name"],
|
|
463
506
|
"kind": rec["kind"],
|
|
464
507
|
"tags": rec.get("tags") or [],
|
|
465
508
|
}
|
|
509
|
+
for key in ("uri", "same_as", "external_ids"):
|
|
510
|
+
if key in rec:
|
|
511
|
+
out[key] = rec[key]
|
|
512
|
+
return out
|
|
466
513
|
|
|
467
514
|
byid = {rec.get("entity_id"): canon(rec) for rec in existing if "entity_id" in rec
|
|
468
515
|
and isinstance(rec.get("name"), str) and isinstance(rec.get("kind"), str)}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"""Optional semantic vocabulary for the facts layer.
|
|
2
|
+
|
|
3
|
+
The ontology is deliberately small and file-native. If ``vault/ontology.yaml`` is
|
|
4
|
+
absent, facts behave exactly as before. If present, it constrains selected free
|
|
5
|
+
text fields and can canonicalize claim predicates via aliases.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
|
+
from .errors import DataError
|
|
17
|
+
|
|
18
|
+
try:
|
|
19
|
+
from yaml import CSafeLoader as SafeLoader
|
|
20
|
+
except ImportError:
|
|
21
|
+
from yaml import SafeLoader
|
|
22
|
+
|
|
23
|
+
_ONTOLOGY_KEYS = frozenset(
|
|
24
|
+
("entity_kinds", "edge_kinds", "claim_predicates", "predicate_aliases")
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass(frozen=True)
|
|
29
|
+
class Ontology:
|
|
30
|
+
entity_kinds: frozenset[str] = frozenset()
|
|
31
|
+
edge_kinds: frozenset[str] = frozenset()
|
|
32
|
+
claim_predicates: frozenset[str] = frozenset()
|
|
33
|
+
predicate_aliases: dict[str, str] | None = None
|
|
34
|
+
|
|
35
|
+
@property
|
|
36
|
+
def active(self) -> bool:
|
|
37
|
+
return bool(
|
|
38
|
+
self.entity_kinds
|
|
39
|
+
or self.edge_kinds
|
|
40
|
+
or self.claim_predicates
|
|
41
|
+
or self.predicate_aliases
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
def canonical_claim_predicate(self, predicate: str, index: int) -> str:
|
|
45
|
+
aliases = self.predicate_aliases or {}
|
|
46
|
+
canonical = aliases.get(predicate, predicate)
|
|
47
|
+
if self.claim_predicates and canonical not in self.claim_predicates:
|
|
48
|
+
raise DataError(
|
|
49
|
+
f"record {index}: predicate {predicate!r} is not in "
|
|
50
|
+
f"vault/ontology.yaml claim_predicates"
|
|
51
|
+
)
|
|
52
|
+
return canonical
|
|
53
|
+
|
|
54
|
+
def validate_entity_kind(self, kind: str, index: int) -> None:
|
|
55
|
+
if self.entity_kinds and kind not in self.entity_kinds:
|
|
56
|
+
raise DataError(
|
|
57
|
+
f"record {index}: entity kind {kind!r} is not in "
|
|
58
|
+
f"vault/ontology.yaml entity_kinds"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
def validate_edge_kind(self, kind: str, index: int) -> None:
|
|
62
|
+
if self.edge_kinds and kind not in self.edge_kinds:
|
|
63
|
+
raise DataError(
|
|
64
|
+
f"record {index}: edge kind {kind!r} is not in "
|
|
65
|
+
f"vault/ontology.yaml edge_kinds"
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
def summary(self) -> dict[str, Any]:
|
|
69
|
+
return {
|
|
70
|
+
"active": self.active,
|
|
71
|
+
"entity_kinds": sorted(self.entity_kinds),
|
|
72
|
+
"edge_kinds": sorted(self.edge_kinds),
|
|
73
|
+
"claim_predicates": sorted(self.claim_predicates),
|
|
74
|
+
"predicate_aliases": dict(sorted((self.predicate_aliases or {}).items())),
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def ontology_path(root: Path) -> Path:
|
|
79
|
+
return root / "vault" / "ontology.yaml"
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def load(root: Path) -> Ontology:
|
|
83
|
+
path = ontology_path(root)
|
|
84
|
+
if not path.exists():
|
|
85
|
+
return Ontology()
|
|
86
|
+
try:
|
|
87
|
+
data = yaml.load(path.read_text(encoding="utf-8"), Loader=SafeLoader)
|
|
88
|
+
except yaml.YAMLError as e:
|
|
89
|
+
raise DataError(f"invalid vault/ontology.yaml: {e}") from e
|
|
90
|
+
if data is None:
|
|
91
|
+
return Ontology()
|
|
92
|
+
if not isinstance(data, dict):
|
|
93
|
+
raise DataError("invalid vault/ontology.yaml: expected a mapping")
|
|
94
|
+
_check_known_keys(data)
|
|
95
|
+
|
|
96
|
+
entity_kinds = _string_set(data, "entity_kinds")
|
|
97
|
+
edge_kinds = _string_set(data, "edge_kinds")
|
|
98
|
+
claim_predicates = _string_set(data, "claim_predicates")
|
|
99
|
+
predicate_aliases = _string_map(data, "predicate_aliases")
|
|
100
|
+
if claim_predicates:
|
|
101
|
+
for alias, target in predicate_aliases.items():
|
|
102
|
+
if target not in claim_predicates:
|
|
103
|
+
raise DataError(
|
|
104
|
+
"invalid vault/ontology.yaml: predicate_aliases target "
|
|
105
|
+
f"{target!r} for {alias!r} is not in claim_predicates"
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
return Ontology(
|
|
109
|
+
entity_kinds=frozenset(entity_kinds),
|
|
110
|
+
edge_kinds=frozenset(edge_kinds),
|
|
111
|
+
claim_predicates=frozenset(claim_predicates),
|
|
112
|
+
predicate_aliases=predicate_aliases,
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _check_known_keys(data: dict) -> None:
|
|
117
|
+
unknown = [key for key in data if not isinstance(key, str) or key not in _ONTOLOGY_KEYS]
|
|
118
|
+
if unknown:
|
|
119
|
+
rendered = ", ".join(repr(key) for key in unknown)
|
|
120
|
+
expected = ", ".join(sorted(_ONTOLOGY_KEYS))
|
|
121
|
+
raise DataError(
|
|
122
|
+
f"invalid vault/ontology.yaml: unknown key(s): {rendered}; expected one of: "
|
|
123
|
+
f"{expected}"
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _string_set(data: dict, key: str) -> set[str]:
|
|
128
|
+
value = data.get(key, [])
|
|
129
|
+
if value is None:
|
|
130
|
+
return set()
|
|
131
|
+
if not isinstance(value, list):
|
|
132
|
+
raise DataError(f"invalid vault/ontology.yaml: {key} must be a list of strings")
|
|
133
|
+
out: set[str] = set()
|
|
134
|
+
for i, item in enumerate(value):
|
|
135
|
+
if not isinstance(item, str) or not item.strip():
|
|
136
|
+
raise DataError(
|
|
137
|
+
f"invalid vault/ontology.yaml: {key}[{i}] must be a non-empty string"
|
|
138
|
+
)
|
|
139
|
+
out.add(item.strip())
|
|
140
|
+
return out
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def _string_map(data: dict, key: str) -> dict[str, str]:
|
|
144
|
+
value = data.get(key, {})
|
|
145
|
+
if value is None:
|
|
146
|
+
return {}
|
|
147
|
+
if not isinstance(value, dict):
|
|
148
|
+
raise DataError(
|
|
149
|
+
f"invalid vault/ontology.yaml: {key} must be a mapping of strings to strings"
|
|
150
|
+
)
|
|
151
|
+
out: dict[str, str] = {}
|
|
152
|
+
for raw_k, raw_v in value.items():
|
|
153
|
+
if not isinstance(raw_k, str) or not raw_k.strip():
|
|
154
|
+
raise DataError(
|
|
155
|
+
f"invalid vault/ontology.yaml: {key} keys must be non-empty strings"
|
|
156
|
+
)
|
|
157
|
+
if not isinstance(raw_v, str) or not raw_v.strip():
|
|
158
|
+
raise DataError(
|
|
159
|
+
f"invalid vault/ontology.yaml: {key}.{raw_k} must be a non-empty string"
|
|
160
|
+
)
|
|
161
|
+
out[raw_k.strip()] = raw_v.strip()
|
|
162
|
+
return out
|
|
@@ -68,6 +68,17 @@ def _graph_lines(kb):
|
|
|
68
68
|
return [json.loads(s) for s in p.read_text(encoding="utf-8").splitlines() if s.strip()]
|
|
69
69
|
|
|
70
70
|
|
|
71
|
+
def _entities_lines(kb):
|
|
72
|
+
p = kb.root / "vault" / "facts" / "entities.ndjson"
|
|
73
|
+
if not p.exists():
|
|
74
|
+
return []
|
|
75
|
+
return [json.loads(s) for s in p.read_text(encoding="utf-8").splitlines() if s.strip()]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _write_ontology(kb, text):
|
|
79
|
+
(kb.root / "vault" / "ontology.yaml").write_text(text, encoding="utf-8")
|
|
80
|
+
|
|
81
|
+
|
|
71
82
|
def _two_claims(kb):
|
|
72
83
|
"""Seed a contradiction pair to reconcile."""
|
|
73
84
|
kb.add_raw("s", SRC)
|
|
@@ -244,6 +255,38 @@ def test_fact_add_json_reports_appended_records(kb, capsys):
|
|
|
244
255
|
assert rec["anchor"].startswith("qh:")
|
|
245
256
|
|
|
246
257
|
|
|
258
|
+
def test_fact_add_ontology_canonicalizes_claim_predicate_aliases(kb):
|
|
259
|
+
kb.add_raw("s", SRC)
|
|
260
|
+
_write_ontology(
|
|
261
|
+
kb,
|
|
262
|
+
"""
|
|
263
|
+
claim_predicates:
|
|
264
|
+
- caches
|
|
265
|
+
predicate_aliases:
|
|
266
|
+
p: caches
|
|
267
|
+
""",
|
|
268
|
+
)
|
|
269
|
+
rc = _run_add(
|
|
270
|
+
kb,
|
|
271
|
+
_ndjson(_claim("Caching answers beats recomputing them.", predicate="p")),
|
|
272
|
+
)
|
|
273
|
+
assert rc == 0
|
|
274
|
+
assert _claims_lines(kb)[0]["predicate"] == "caches"
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def test_fact_add_ontology_rejects_unknown_claim_predicate(kb):
|
|
278
|
+
kb.add_raw("s", SRC)
|
|
279
|
+
_write_ontology(kb, "claim_predicates:\n - caches\n")
|
|
280
|
+
|
|
281
|
+
rc = _run_add(
|
|
282
|
+
kb,
|
|
283
|
+
_ndjson(_claim("Caching answers beats recomputing them.", predicate="p")),
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
assert rc == 3
|
|
287
|
+
assert _claims_lines(kb) == []
|
|
288
|
+
|
|
289
|
+
|
|
247
290
|
# --------------------------------------------------------------------------- #
|
|
248
291
|
# All-or-nothing batch + per-record failures (exit 1)
|
|
249
292
|
# --------------------------------------------------------------------------- #
|
|
@@ -483,6 +526,93 @@ def test_fact_add_entities_appends_and_conflicts(kb, capsys):
|
|
|
483
526
|
assert len(lines.splitlines()) == 1 # nothing else was appended
|
|
484
527
|
|
|
485
528
|
|
|
529
|
+
def test_fact_add_entities_accept_optional_uri_metadata(kb):
|
|
530
|
+
ent = {
|
|
531
|
+
"entity_id": "entity/duckdb",
|
|
532
|
+
"name": "DuckDB",
|
|
533
|
+
"kind": "tool",
|
|
534
|
+
"uri": "https://duckdb.org/",
|
|
535
|
+
"same_as": ["https://www.wikidata.org/wiki/Q118533698"],
|
|
536
|
+
"external_ids": {"wikidata": "Q118533698"},
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
assert _run_add(kb, _ndjson(ent), "--table", "entities") == 0
|
|
540
|
+
|
|
541
|
+
[stored] = _entities_lines(kb)
|
|
542
|
+
assert stored["uri"] == "https://duckdb.org/"
|
|
543
|
+
assert stored["same_as"] == ["https://www.wikidata.org/wiki/Q118533698"]
|
|
544
|
+
assert stored["external_ids"] == {"wikidata": "Q118533698"}
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def test_fact_add_ontology_rejects_unknown_entity_kind_and_edge_kind(kb):
|
|
548
|
+
_write_ontology(
|
|
549
|
+
kb,
|
|
550
|
+
"""
|
|
551
|
+
entity_kinds:
|
|
552
|
+
- tool
|
|
553
|
+
edge_kinds:
|
|
554
|
+
- depends-on
|
|
555
|
+
""",
|
|
556
|
+
)
|
|
557
|
+
|
|
558
|
+
assert (
|
|
559
|
+
_run_add(
|
|
560
|
+
kb,
|
|
561
|
+
_ndjson({"entity_id": "entity/a", "name": "A", "kind": "concept"}),
|
|
562
|
+
"--table",
|
|
563
|
+
"entities",
|
|
564
|
+
)
|
|
565
|
+
== 3
|
|
566
|
+
)
|
|
567
|
+
assert _run_add(kb, _ndjson(_edge(kind="relates-to")), "--table", "edges") == 3
|
|
568
|
+
assert _run_add(kb, _ndjson(_edge(kind="depends-on")), "--table", "edges") == 0
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def test_ontology_command_reports_optional_vocabulary(kb, capsys):
|
|
572
|
+
_write_ontology(
|
|
573
|
+
kb,
|
|
574
|
+
"""
|
|
575
|
+
entity_kinds:
|
|
576
|
+
- tool
|
|
577
|
+
edge_kinds:
|
|
578
|
+
- depends-on
|
|
579
|
+
claim_predicates:
|
|
580
|
+
- caches
|
|
581
|
+
predicate_aliases:
|
|
582
|
+
p: caches
|
|
583
|
+
""",
|
|
584
|
+
)
|
|
585
|
+
|
|
586
|
+
assert cli.main(["ontology", "--json", "--root", str(kb.root)]) == 0
|
|
587
|
+
|
|
588
|
+
data = json.loads(capsys.readouterr().out)
|
|
589
|
+
assert data["active"] is True
|
|
590
|
+
assert data["entity_kinds"] == ["tool"]
|
|
591
|
+
assert data["edge_kinds"] == ["depends-on"]
|
|
592
|
+
assert data["claim_predicates"] == ["caches"]
|
|
593
|
+
assert data["predicate_aliases"] == {"p": "caches"}
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def test_ontology_rejects_alias_to_unknown_predicate(kb):
|
|
597
|
+
_write_ontology(
|
|
598
|
+
kb,
|
|
599
|
+
"""
|
|
600
|
+
claim_predicates:
|
|
601
|
+
- caches
|
|
602
|
+
predicate_aliases:
|
|
603
|
+
p: stores
|
|
604
|
+
""",
|
|
605
|
+
)
|
|
606
|
+
|
|
607
|
+
assert cli.main(["ontology", "--root", str(kb.root)]) == 3
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def test_ontology_rejects_unknown_top_level_key(kb):
|
|
611
|
+
_write_ontology(kb, "claim_predicate:\n - caches\n")
|
|
612
|
+
|
|
613
|
+
assert cli.main(["ontology", "--root", str(kb.root)]) == 3
|
|
614
|
+
|
|
615
|
+
|
|
486
616
|
def test_fact_add_edges_appends_and_skips_duplicates(kb):
|
|
487
617
|
edge = {"src": "entity/duckdb", "dst": "entity/motherduck", "kind": "made-by"}
|
|
488
618
|
assert _run_add(kb, _ndjson(edge), "--table", "edges") == 0
|
|
@@ -4,6 +4,20 @@ from scrip.errors import DataError, UsageError
|
|
|
4
4
|
from scrip import facts
|
|
5
5
|
|
|
6
6
|
|
|
7
|
+
def test_iter_lf_lines_matches_split_newline_semantics():
|
|
8
|
+
samples = [
|
|
9
|
+
"",
|
|
10
|
+
"one",
|
|
11
|
+
"one\n",
|
|
12
|
+
"one\n\n",
|
|
13
|
+
"one\r\ntwo\r\n",
|
|
14
|
+
"one\rtwo",
|
|
15
|
+
"one\u2028two\ntwo\u2029three",
|
|
16
|
+
]
|
|
17
|
+
for text in samples:
|
|
18
|
+
assert list(facts._iter_lf_lines(text)) == text.split("\n")
|
|
19
|
+
|
|
20
|
+
|
|
7
21
|
def test_parse_ndjson_basic():
|
|
8
22
|
text = '{"a": 1}\n{"b": 2}\n'
|
|
9
23
|
assert facts.parse_ndjson(text) == [{"a": 1}, {"b": 2}]
|
|
@@ -249,7 +249,7 @@ name = "exceptiongroup"
|
|
|
249
249
|
version = "1.3.1"
|
|
250
250
|
source = { registry = "https://pypi.org/simple" }
|
|
251
251
|
dependencies = [
|
|
252
|
-
{ name = "typing-extensions"
|
|
252
|
+
{ name = "typing-extensions" },
|
|
253
253
|
]
|
|
254
254
|
sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" }
|
|
255
255
|
wheels = [
|
|
@@ -1141,7 +1141,7 @@ wheels = [
|
|
|
1141
1141
|
|
|
1142
1142
|
[[package]]
|
|
1143
1143
|
name = "scriptoria"
|
|
1144
|
-
version = "0.
|
|
1144
|
+
version = "0.9.0"
|
|
1145
1145
|
source = { editable = "." }
|
|
1146
1146
|
dependencies = [
|
|
1147
1147
|
{ name = "duckdb" },
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|