kbforge-okfquery 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kbforge_okfquery-0.1.0/.gitignore +19 -0
- kbforge_okfquery-0.1.0/PKG-INFO +244 -0
- kbforge_okfquery-0.1.0/README.md +231 -0
- kbforge_okfquery-0.1.0/pyproject.toml +37 -0
- kbforge_okfquery-0.1.0/src/okfquery/__init__.py +14 -0
- kbforge_okfquery-0.1.0/src/okfquery/cli.py +144 -0
- kbforge_okfquery-0.1.0/src/okfquery/load.py +132 -0
- kbforge_okfquery-0.1.0/src/okfquery/parse.py +252 -0
- kbforge_okfquery-0.1.0/src/okfquery/schema.py +51 -0
- kbforge_okfquery-0.1.0/tests/fixtures/clean/concepts/api/overview.md +18 -0
- kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/badyaml/overview.md +6 -0
- kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/claims/index.md +16 -0
- kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/incomplete/overview.md +12 -0
- kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/index.md +3 -0
- kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/log.md +3 -0
- kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/nofence/overview.md +3 -0
- kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/notmapping/overview.md +6 -0
- kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/ok/overview.md +22 -0
- kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/unterminated/overview.md +3 -0
- kbforge_okfquery-0.1.0/tests/test_load.py +241 -0
- kbforge_okfquery-0.1.0/tests/test_okfquery_cli.py +97 -0
- kbforge_okfquery-0.1.0/tests/test_okfquery_live.py +208 -0
- kbforge_okfquery-0.1.0/tests/test_parse.py +158 -0
- kbforge_okfquery-0.1.0/tests/test_roundtrip.py +127 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
*.egg-info/
|
|
4
|
+
dist/
|
|
5
|
+
build/
|
|
6
|
+
.eggs/
|
|
7
|
+
*.egg
|
|
8
|
+
.venv/
|
|
9
|
+
.pytest_cache/
|
|
10
|
+
.ruff_cache/
|
|
11
|
+
.coverage
|
|
12
|
+
|
|
13
|
+
# Local environment / secrets — never commit real keys
|
|
14
|
+
.env
|
|
15
|
+
.env.*
|
|
16
|
+
!.env.example
|
|
17
|
+
|
|
18
|
+
# subagent-driven-development scratch (ledger, briefs, reports)
|
|
19
|
+
.superpowers/
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: kbforge-okfquery
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: SQL over an OKF v0.2 bundle, via DuckDB
|
|
5
|
+
Author-email: Qing <qingye779@gmail.com>
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: duckdb,knowledge-base,okf,open-knowledge-format,sql
|
|
8
|
+
Requires-Python: >=3.12
|
|
9
|
+
Requires-Dist: duckdb>=1.1
|
|
10
|
+
Requires-Dist: pytz>=2024.1
|
|
11
|
+
Requires-Dist: pyyaml>=6
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# okfquery
|
|
15
|
+
|
|
16
|
+
SQL over an [OKF v0.2](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md)
|
|
17
|
+
bundle, via DuckDB. `okfquery` loads a published bundle into an in-memory DuckDB
|
|
18
|
+
connection and hands the connection back — no daemon, no cache, no committed
|
|
19
|
+
artifact. It depends on no kbforge code at runtime, so it reads any OKF v0.2
|
|
20
|
+
bundle, kbforge-built or not.
|
|
21
|
+
|
|
22
|
+
## Install and use
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
uv pip install kbforge-okfquery # or: pip install kbforge-okfquery
|
|
26
|
+
okfquery query "select count(*) from concepts" --bundle path/to/bundle
|
|
27
|
+
okfquery shell --bundle path/to/bundle
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
`query` runs one SQL statement and prints the result (`--format table|json|csv`).
|
|
31
|
+
`shell` writes a temporary `.duckdb` file, opens the `duckdb` CLI on it, and
|
|
32
|
+
deletes the file when the shell exits.
|
|
33
|
+
|
|
34
|
+
## Schema
|
|
35
|
+
|
|
36
|
+
```
|
|
37
|
+
$ okfquery schema
|
|
38
|
+
CREATE TABLE concepts (
|
|
39
|
+
path VARCHAR NOT NULL,
|
|
40
|
+
type VARCHAR,
|
|
41
|
+
title VARCHAR,
|
|
42
|
+
description VARCHAR,
|
|
43
|
+
generated_by VARCHAR,
|
|
44
|
+
-- TIMESTAMPTZ, never TIMESTAMP. Not because a naive column would corrupt
|
|
45
|
+
-- the instant on this package's load path -- `load` binds an aware Python
|
|
46
|
+
-- datetime through executemany, so the instant survives a naive column
|
|
47
|
+
-- fine. What a naive column loses is the type: values come back with no
|
|
48
|
+
-- tzinfo, so every comparison against now() (itself TIMESTAMPTZ) needs a
|
|
49
|
+
-- cast, and the column asserts UTC by convention with nothing recording
|
|
50
|
+
-- that it does. §4.4 law 4 exists to force an *aware* stamp; a column that
|
|
51
|
+
-- cannot hold one throws away exactly the property the law buys.
|
|
52
|
+
generated_at TIMESTAMPTZ,
|
|
53
|
+
facets JSON,
|
|
54
|
+
body VARCHAR
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
CREATE TABLE sources (
|
|
58
|
+
path VARCHAR NOT NULL,
|
|
59
|
+
-- A real column. kbforge's synthesize.assemble puts the OWNING anchor first
|
|
60
|
+
-- and grounding anchors after it, and that ordering is the only thing
|
|
61
|
+
-- telling owner from ground -- OKF has no field for it. Discarding ordinal
|
|
62
|
+
-- during unnest would destroy the signal.
|
|
63
|
+
ordinal INTEGER NOT NULL,
|
|
64
|
+
id VARCHAR,
|
|
65
|
+
resource VARCHAR,
|
|
66
|
+
content_hash VARCHAR
|
|
67
|
+
);
|
|
68
|
+
|
|
69
|
+
CREATE TABLE links (
|
|
70
|
+
path VARCHAR NOT NULL,
|
|
71
|
+
target VARCHAR NOT NULL
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
CREATE TABLE problems (
|
|
75
|
+
path VARCHAR NOT NULL,
|
|
76
|
+
kind VARCHAR NOT NULL,
|
|
77
|
+
detail VARCHAR NOT NULL
|
|
78
|
+
);
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
`path` is the bundle-relative concept path and the join key throughout.
|
|
82
|
+
`sources.ordinal = 0` is the owning source for a kbforge-produced bundle (the
|
|
83
|
+
convention `synthesize.assemble` writes) — on a foreign bundle it is merely
|
|
84
|
+
source order, with no error to tell you that. `facets` is one JSON column, open
|
|
85
|
+
by construction, so `facets->>'owner'` reaches whatever a connector attached.
|
|
86
|
+
`problems` is additive: a file that failed to parse still gets a `concepts` row
|
|
87
|
+
with NULLs for what could not be read, plus one `problems` row per distinct
|
|
88
|
+
failure — nothing a broken concept says gets silently dropped from an audit.
|
|
89
|
+
|
|
90
|
+
## Queries, run against the messy test fixture
|
|
91
|
+
|
|
92
|
+
All four queries below were run as shown against
|
|
93
|
+
`packages/okfquery/tests/fixtures/messy` — 9 `.md` files under `concepts/`, 7 of
|
|
94
|
+
them concepts (`concepts/index.md` and `concepts/log.md` are reserved and
|
|
95
|
+
fenceless, so they are skipped; `concepts/claims/index.md` is reserved-*named*
|
|
96
|
+
but carries frontmatter, so it counts).
|
|
97
|
+
|
|
98
|
+
**What is built on a given upstream document:**
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
$ okfquery query "
|
|
102
|
+
SELECT c.path, s.ordinal
|
|
103
|
+
FROM concepts c JOIN sources s USING (path)
|
|
104
|
+
WHERE s.resource = 'https://wiki/ok';
|
|
105
|
+
" --bundle packages/okfquery/tests/fixtures/messy
|
|
106
|
+
┌─────────────────────────┬─────────┐
|
|
107
|
+
│ path │ ordinal │
|
|
108
|
+
│ varchar │ int32 │
|
|
109
|
+
├─────────────────────────┼─────────┤
|
|
110
|
+
│ concepts/ok/overview.md │ 0 │
|
|
111
|
+
└─────────────────────────┴─────────┘
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
**Staleness by owning system** (`ordinal = 0` picks the owning source, not a
|
|
115
|
+
grounding one — see the ordinal note above):
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
$ okfquery query "
|
|
119
|
+
SELECT split_part(s.id, ':', 1) AS system,
|
|
120
|
+
count(*) AS concepts,
|
|
121
|
+
min(c.generated_at) AS oldest
|
|
122
|
+
FROM concepts c JOIN sources s USING (path)
|
|
123
|
+
WHERE s.ordinal = 0
|
|
124
|
+
GROUP BY 1 ORDER BY oldest;
|
|
125
|
+
" --bundle packages/okfquery/tests/fixtures/messy
|
|
126
|
+
┌─────────┬──────────┬──────────────────────────┐
|
|
127
|
+
│ system │ concepts │ oldest │
|
|
128
|
+
│ varchar │ int64 │ timestamp with time zone │
|
|
129
|
+
├─────────┼──────────┼──────────────────────────┤
|
|
130
|
+
│ wiki │ 3 │ 2026-08-23 00:00:00+00 │
|
|
131
|
+
└─────────┴──────────┴──────────────────────────┘
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
Only one system shows up on this fixture: most of the messy bundle's files
|
|
135
|
+
exist to exercise a `problems` kind and never got far enough to have a
|
|
136
|
+
`sources` entry at all, so they don't join into this query — which is correct,
|
|
137
|
+
not a bug in the query.
|
|
138
|
+
|
|
139
|
+
**Orphans — nothing links to them:**
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
$ okfquery query "
|
|
143
|
+
SELECT c.path FROM concepts c
|
|
144
|
+
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.target = c.path);
|
|
145
|
+
" --bundle packages/okfquery/tests/fixtures/messy
|
|
146
|
+
┌───────────────────────────────────┐
|
|
147
|
+
│ path │
|
|
148
|
+
│ varchar │
|
|
149
|
+
├───────────────────────────────────┤
|
|
150
|
+
│ concepts/badyaml/overview.md │
|
|
151
|
+
│ concepts/claims/index.md │
|
|
152
|
+
│ concepts/incomplete/overview.md │
|
|
153
|
+
│ concepts/nofence/overview.md │
|
|
154
|
+
│ concepts/notmapping/overview.md │
|
|
155
|
+
│ concepts/unterminated/overview.md │
|
|
156
|
+
└───────────────────────────────────┘
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
**Everything that failed to parse** (`--format csv` here, since the default
|
|
160
|
+
table renderer truncates long `detail` strings):
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
$ okfquery query "SELECT path, kind, detail FROM problems ORDER BY path, kind;" \
|
|
164
|
+
--bundle packages/okfquery/tests/fixtures/messy --format csv
|
|
165
|
+
path,kind,detail
|
|
166
|
+
concepts/badyaml/overview.md,invalid-yaml,frontmatter is not valid YAML: ParserError
|
|
167
|
+
concepts/incomplete/overview.md,bad-links,'links' entry 3 is not a string
|
|
168
|
+
concepts/incomplete/overview.md,bad-sources,'sources' entry 0 has no 'resource' (§5.1 requires one)
|
|
169
|
+
concepts/incomplete/overview.md,bad-timestamp,'generated.at' 'not-a-date' is not an ISO-8601 datetime
|
|
170
|
+
concepts/incomplete/overview.md,missing-required,"missing required OKF keys: title, description"
|
|
171
|
+
concepts/nofence/overview.md,no-frontmatter,file does not open a '---' frontmatter fence
|
|
172
|
+
concepts/notmapping/overview.md,frontmatter-not-mapping,"frontmatter parses to list, not a mapping"
|
|
173
|
+
concepts/unterminated/overview.md,unterminated-frontmatter,frontmatter fence is opened but never closed by a closing '---'
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
## The mirror, and why both obvious joins are wrong
|
|
177
|
+
|
|
178
|
+
`--mirror <path>` attaches a `mirror` view over a kbforge mirror directory
|
|
179
|
+
(`read_json_auto('<mirror>/*.json')`) — the same `CanonicalDocument` slots
|
|
180
|
+
`mirror.slot_key` writes, `anchor` struct included. It answers a different
|
|
181
|
+
question than the bundle alone can: not "what does this concept say" but "does
|
|
182
|
+
it still say what its source currently does."
|
|
183
|
+
|
|
184
|
+
**Both of the joins you'd reach for first are lossy, in different ways:**
|
|
185
|
+
|
|
186
|
+
- `sources.id = mirror.doc_id` holds for every shipped connector, but
|
|
187
|
+
`synthesize._source_entry` builds `id` from `anchor.system`/`anchor.native_id`
|
|
188
|
+
while `doc_id` is the mirror's own field — nothing enforces the two agree for
|
|
189
|
+
a third-party connector. Where they diverge, this join produces a silent
|
|
190
|
+
false miss, not an error.
|
|
191
|
+
- `sources.content_hash = mirror.anchor.content_hash` as an **equi-join** is not
|
|
192
|
+
the sound alternative either. `kbforge.pipeline.run` calls
|
|
193
|
+
`commit(mirror_path, docs)` right after `kbforge_publish`, and publishing only
|
|
194
|
+
opens a review request — kbforge never merges. Between publish and merge the
|
|
195
|
+
mirror already holds the new hash while the bundle still carries the previous
|
|
196
|
+
run's, so an equi-join on hash returns **zero rows for every concept with a
|
|
197
|
+
review request open** — exactly the ones an audit most wants to see.
|
|
198
|
+
|
|
199
|
+
The correct form: join on `id` with a **`LEFT JOIN`**, not an inner join, then
|
|
200
|
+
**compare** hashes instead of joining on them — an inner join silently drops
|
|
201
|
+
every source with no mirror row at all, which is exactly the "never
|
|
202
|
+
published" case the query needs to report. A match means current; a mismatch
|
|
203
|
+
means an update is sitting in review; a source with no mirror row was never
|
|
204
|
+
published. Run for real against the messy fixture (`concepts/ok/overview.md`'s
|
|
205
|
+
two sources — an owning `wiki:ok` and a grounding `notes:ok` — plus two other
|
|
206
|
+
concepts whose sources have no mirror row at all) with a synthetic
|
|
207
|
+
two-document mirror covering only `wiki:ok` and `notes:ok`, one hash left
|
|
208
|
+
matching and one changed to simulate an update stuck in review:
|
|
209
|
+
|
|
210
|
+
```bash
|
|
211
|
+
$ okfquery query "
|
|
212
|
+
SELECT s.path,
|
|
213
|
+
s.id,
|
|
214
|
+
s.content_hash AS bundle_hash,
|
|
215
|
+
m.anchor.content_hash AS mirror_hash,
|
|
216
|
+
CASE WHEN m.doc_id IS NULL THEN 'never published'
|
|
217
|
+
WHEN s.content_hash = m.anchor.content_hash THEN 'current'
|
|
218
|
+
ELSE 'update in review' END AS status
|
|
219
|
+
FROM sources s LEFT JOIN mirror m ON s.id = m.doc_id
|
|
220
|
+
ORDER BY s.path, s.id;
|
|
221
|
+
" --bundle packages/okfquery/tests/fixtures/messy --mirror /path/to/mirror
|
|
222
|
+
┌─────────────────────────────────┬─────────────────┬─────────────┬─────────────┬──────────────────┐
|
|
223
|
+
│ path │ id │ bundle_hash │ mirror_hash │ status │
|
|
224
|
+
│ varchar │ varchar │ varchar │ varchar │ varchar │
|
|
225
|
+
├─────────────────────────────────┼─────────────────┼─────────────┼─────────────┼──────────────────┤
|
|
226
|
+
│ concepts/claims/index.md │ wiki:claims │ h-claims │ NULL │ never published │
|
|
227
|
+
│ concepts/incomplete/overview.md │ wiki:incomplete │ NULL │ NULL │ never published │
|
|
228
|
+
│ concepts/ok/overview.md │ notes:ok │ h-ground │ h-ground-v2 │ update in review │
|
|
229
|
+
│ concepts/ok/overview.md │ wiki:ok │ h-own │ h-own │ current │
|
|
230
|
+
└─────────────────────────────────┴─────────────────┴─────────────┴─────────────┴──────────────────┘
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
That does not fix the `id` gap above: a connector whose anchor disagrees with
|
|
234
|
+
its `doc_id` still misses this join silently. There is no workaround for that
|
|
235
|
+
here — only the honest statement of it.
|
|
236
|
+
|
|
237
|
+
## Not built
|
|
238
|
+
|
|
239
|
+
- **No full-text search out of the box.** `body` is a plain column; `INSTALL
|
|
240
|
+
fts` and an index over it is a documented recipe, not a feature.
|
|
241
|
+
- **No remote bundles.** `--bundle` is a local checkout path; reading one over
|
|
242
|
+
HTTP wants a caching story this package deliberately avoids.
|
|
243
|
+
- **No MCP server.** `load()` is already small enough to wrap in one `query`
|
|
244
|
+
tool and a schema resource; nothing here does that yet.
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
# okfquery
|
|
2
|
+
|
|
3
|
+
SQL over an [OKF v0.2](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md)
|
|
4
|
+
bundle, via DuckDB. `okfquery` loads a published bundle into an in-memory DuckDB
|
|
5
|
+
connection and hands the connection back — no daemon, no cache, no committed
|
|
6
|
+
artifact. It depends on no kbforge code at runtime, so it reads any OKF v0.2
|
|
7
|
+
bundle, kbforge-built or not.
|
|
8
|
+
|
|
9
|
+
## Install and use
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
uv pip install kbforge-okfquery # or: pip install kbforge-okfquery
|
|
13
|
+
okfquery query "select count(*) from concepts" --bundle path/to/bundle
|
|
14
|
+
okfquery shell --bundle path/to/bundle
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
`query` runs one SQL statement and prints the result (`--format table|json|csv`).
|
|
18
|
+
`shell` writes a temporary `.duckdb` file, opens the `duckdb` CLI on it, and
|
|
19
|
+
deletes the file when the shell exits.
|
|
20
|
+
|
|
21
|
+
## Schema
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
$ okfquery schema
|
|
25
|
+
CREATE TABLE concepts (
|
|
26
|
+
path VARCHAR NOT NULL,
|
|
27
|
+
type VARCHAR,
|
|
28
|
+
title VARCHAR,
|
|
29
|
+
description VARCHAR,
|
|
30
|
+
generated_by VARCHAR,
|
|
31
|
+
-- TIMESTAMPTZ, never TIMESTAMP. Not because a naive column would corrupt
|
|
32
|
+
-- the instant on this package's load path -- `load` binds an aware Python
|
|
33
|
+
-- datetime through executemany, so the instant survives a naive column
|
|
34
|
+
-- fine. What a naive column loses is the type: values come back with no
|
|
35
|
+
-- tzinfo, so every comparison against now() (itself TIMESTAMPTZ) needs a
|
|
36
|
+
-- cast, and the column asserts UTC by convention with nothing recording
|
|
37
|
+
-- that it does. §4.4 law 4 exists to force an *aware* stamp; a column that
|
|
38
|
+
-- cannot hold one throws away exactly the property the law buys.
|
|
39
|
+
generated_at TIMESTAMPTZ,
|
|
40
|
+
facets JSON,
|
|
41
|
+
body VARCHAR
|
|
42
|
+
);
|
|
43
|
+
|
|
44
|
+
CREATE TABLE sources (
|
|
45
|
+
path VARCHAR NOT NULL,
|
|
46
|
+
-- A real column. kbforge's synthesize.assemble puts the OWNING anchor first
|
|
47
|
+
-- and grounding anchors after it, and that ordering is the only thing
|
|
48
|
+
-- telling owner from ground -- OKF has no field for it. Discarding ordinal
|
|
49
|
+
-- during unnest would destroy the signal.
|
|
50
|
+
ordinal INTEGER NOT NULL,
|
|
51
|
+
id VARCHAR,
|
|
52
|
+
resource VARCHAR,
|
|
53
|
+
content_hash VARCHAR
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
CREATE TABLE links (
|
|
57
|
+
path VARCHAR NOT NULL,
|
|
58
|
+
target VARCHAR NOT NULL
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
CREATE TABLE problems (
|
|
62
|
+
path VARCHAR NOT NULL,
|
|
63
|
+
kind VARCHAR NOT NULL,
|
|
64
|
+
detail VARCHAR NOT NULL
|
|
65
|
+
);
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`path` is the bundle-relative concept path and the join key throughout.
|
|
69
|
+
`sources.ordinal = 0` is the owning source for a kbforge-produced bundle (the
|
|
70
|
+
convention `synthesize.assemble` writes) — on a foreign bundle it is merely
|
|
71
|
+
source order, with no error to tell you that. `facets` is one JSON column, open
|
|
72
|
+
by construction, so `facets->>'owner'` reaches whatever a connector attached.
|
|
73
|
+
`problems` is additive: a file that failed to parse still gets a `concepts` row
|
|
74
|
+
with NULLs for what could not be read, plus one `problems` row per distinct
|
|
75
|
+
failure — nothing a broken concept says gets silently dropped from an audit.
|
|
76
|
+
|
|
77
|
+
## Queries, run against the messy test fixture
|
|
78
|
+
|
|
79
|
+
All four queries below were run as shown against
|
|
80
|
+
`packages/okfquery/tests/fixtures/messy` — 9 `.md` files under `concepts/`, 7 of
|
|
81
|
+
them concepts (`concepts/index.md` and `concepts/log.md` are reserved and
|
|
82
|
+
fenceless, so they are skipped; `concepts/claims/index.md` is reserved-*named*
|
|
83
|
+
but carries frontmatter, so it counts).
|
|
84
|
+
|
|
85
|
+
**What is built on a given upstream document:**
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
$ okfquery query "
|
|
89
|
+
SELECT c.path, s.ordinal
|
|
90
|
+
FROM concepts c JOIN sources s USING (path)
|
|
91
|
+
WHERE s.resource = 'https://wiki/ok';
|
|
92
|
+
" --bundle packages/okfquery/tests/fixtures/messy
|
|
93
|
+
┌─────────────────────────┬─────────┐
|
|
94
|
+
│ path │ ordinal │
|
|
95
|
+
│ varchar │ int32 │
|
|
96
|
+
├─────────────────────────┼─────────┤
|
|
97
|
+
│ concepts/ok/overview.md │ 0 │
|
|
98
|
+
└─────────────────────────┴─────────┘
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
**Staleness by owning system** (`ordinal = 0` picks the owning source, not a
|
|
102
|
+
grounding one — see the ordinal note above):
|
|
103
|
+
|
|
104
|
+
```bash
|
|
105
|
+
$ okfquery query "
|
|
106
|
+
SELECT split_part(s.id, ':', 1) AS system,
|
|
107
|
+
count(*) AS concepts,
|
|
108
|
+
min(c.generated_at) AS oldest
|
|
109
|
+
FROM concepts c JOIN sources s USING (path)
|
|
110
|
+
WHERE s.ordinal = 0
|
|
111
|
+
GROUP BY 1 ORDER BY oldest;
|
|
112
|
+
" --bundle packages/okfquery/tests/fixtures/messy
|
|
113
|
+
┌─────────┬──────────┬──────────────────────────┐
|
|
114
|
+
│ system │ concepts │ oldest │
|
|
115
|
+
│ varchar │ int64 │ timestamp with time zone │
|
|
116
|
+
├─────────┼──────────┼──────────────────────────┤
|
|
117
|
+
│ wiki │ 3 │ 2026-08-23 00:00:00+00 │
|
|
118
|
+
└─────────┴──────────┴──────────────────────────┘
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Only one system shows up on this fixture: most of the messy bundle's files
|
|
122
|
+
exist to exercise a `problems` kind and never got far enough to have a
|
|
123
|
+
`sources` entry at all, so they don't join into this query — which is correct,
|
|
124
|
+
not a bug in the query.
|
|
125
|
+
|
|
126
|
+
**Orphans — nothing links to them:**
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
$ okfquery query "
|
|
130
|
+
SELECT c.path FROM concepts c
|
|
131
|
+
WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.target = c.path);
|
|
132
|
+
" --bundle packages/okfquery/tests/fixtures/messy
|
|
133
|
+
┌───────────────────────────────────┐
|
|
134
|
+
│ path │
|
|
135
|
+
│ varchar │
|
|
136
|
+
├───────────────────────────────────┤
|
|
137
|
+
│ concepts/badyaml/overview.md │
|
|
138
|
+
│ concepts/claims/index.md │
|
|
139
|
+
│ concepts/incomplete/overview.md │
|
|
140
|
+
│ concepts/nofence/overview.md │
|
|
141
|
+
│ concepts/notmapping/overview.md │
|
|
142
|
+
│ concepts/unterminated/overview.md │
|
|
143
|
+
└───────────────────────────────────┘
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
**Everything that failed to parse** (`--format csv` here, since the default
|
|
147
|
+
table renderer truncates long `detail` strings):
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
$ okfquery query "SELECT path, kind, detail FROM problems ORDER BY path, kind;" \
|
|
151
|
+
--bundle packages/okfquery/tests/fixtures/messy --format csv
|
|
152
|
+
path,kind,detail
|
|
153
|
+
concepts/badyaml/overview.md,invalid-yaml,frontmatter is not valid YAML: ParserError
|
|
154
|
+
concepts/incomplete/overview.md,bad-links,'links' entry 3 is not a string
|
|
155
|
+
concepts/incomplete/overview.md,bad-sources,'sources' entry 0 has no 'resource' (§5.1 requires one)
|
|
156
|
+
concepts/incomplete/overview.md,bad-timestamp,'generated.at' 'not-a-date' is not an ISO-8601 datetime
|
|
157
|
+
concepts/incomplete/overview.md,missing-required,"missing required OKF keys: title, description"
|
|
158
|
+
concepts/nofence/overview.md,no-frontmatter,file does not open a '---' frontmatter fence
|
|
159
|
+
concepts/notmapping/overview.md,frontmatter-not-mapping,"frontmatter parses to list, not a mapping"
|
|
160
|
+
concepts/unterminated/overview.md,unterminated-frontmatter,frontmatter fence is opened but never closed by a closing '---'
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## The mirror, and why both obvious joins are wrong
|
|
164
|
+
|
|
165
|
+
`--mirror <path>` attaches a `mirror` view over a kbforge mirror directory
|
|
166
|
+
(`read_json_auto('<mirror>/*.json')`) — the same `CanonicalDocument` slots
|
|
167
|
+
`mirror.slot_key` writes, `anchor` struct included. It answers a different
|
|
168
|
+
question than the bundle alone can: not "what does this concept say" but "does
|
|
169
|
+
it still say what its source currently does."
|
|
170
|
+
|
|
171
|
+
**Both of the joins you'd reach for first are lossy, in different ways:**
|
|
172
|
+
|
|
173
|
+
- `sources.id = mirror.doc_id` holds for every shipped connector, but
|
|
174
|
+
`synthesize._source_entry` builds `id` from `anchor.system`/`anchor.native_id`
|
|
175
|
+
while `doc_id` is the mirror's own field — nothing enforces the two agree for
|
|
176
|
+
a third-party connector. Where they diverge, this join produces a silent
|
|
177
|
+
false miss, not an error.
|
|
178
|
+
- `sources.content_hash = mirror.anchor.content_hash` as an **equi-join** is not
|
|
179
|
+
the sound alternative either. `kbforge.pipeline.run` calls
|
|
180
|
+
`commit(mirror_path, docs)` right after `kbforge_publish`, and publishing only
|
|
181
|
+
opens a review request — kbforge never merges. Between publish and merge the
|
|
182
|
+
mirror already holds the new hash while the bundle still carries the previous
|
|
183
|
+
run's, so an equi-join on hash returns **zero rows for every concept with a
|
|
184
|
+
review request open** — exactly the ones an audit most wants to see.
|
|
185
|
+
|
|
186
|
+
The correct form: join on `id` with a **`LEFT JOIN`**, not an inner join, then
|
|
187
|
+
**compare** hashes instead of joining on them — an inner join silently drops
|
|
188
|
+
every source with no mirror row at all, which is exactly the "never
|
|
189
|
+
published" case the query needs to report. A match means current; a mismatch
|
|
190
|
+
means an update is sitting in review; a source with no mirror row was never
|
|
191
|
+
published. Run for real against the messy fixture (`concepts/ok/overview.md`'s
|
|
192
|
+
two sources — an owning `wiki:ok` and a grounding `notes:ok` — plus two other
|
|
193
|
+
concepts whose sources have no mirror row at all) with a synthetic
|
|
194
|
+
two-document mirror covering only `wiki:ok` and `notes:ok`, one hash left
|
|
195
|
+
matching and one changed to simulate an update stuck in review:
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
$ okfquery query "
|
|
199
|
+
SELECT s.path,
|
|
200
|
+
s.id,
|
|
201
|
+
s.content_hash AS bundle_hash,
|
|
202
|
+
m.anchor.content_hash AS mirror_hash,
|
|
203
|
+
CASE WHEN m.doc_id IS NULL THEN 'never published'
|
|
204
|
+
WHEN s.content_hash = m.anchor.content_hash THEN 'current'
|
|
205
|
+
ELSE 'update in review' END AS status
|
|
206
|
+
FROM sources s LEFT JOIN mirror m ON s.id = m.doc_id
|
|
207
|
+
ORDER BY s.path, s.id;
|
|
208
|
+
" --bundle packages/okfquery/tests/fixtures/messy --mirror /path/to/mirror
|
|
209
|
+
┌─────────────────────────────────┬─────────────────┬─────────────┬─────────────┬──────────────────┐
|
|
210
|
+
│ path │ id │ bundle_hash │ mirror_hash │ status │
|
|
211
|
+
│ varchar │ varchar │ varchar │ varchar │ varchar │
|
|
212
|
+
├─────────────────────────────────┼─────────────────┼─────────────┼─────────────┼──────────────────┤
|
|
213
|
+
│ concepts/claims/index.md │ wiki:claims │ h-claims │ NULL │ never published │
|
|
214
|
+
│ concepts/incomplete/overview.md │ wiki:incomplete │ NULL │ NULL │ never published │
|
|
215
|
+
│ concepts/ok/overview.md │ notes:ok │ h-ground │ h-ground-v2 │ update in review │
|
|
216
|
+
│ concepts/ok/overview.md │ wiki:ok │ h-own │ h-own │ current │
|
|
217
|
+
└─────────────────────────────────┴─────────────────┴─────────────┴─────────────┴──────────────────┘
|
|
218
|
+
```
|
|
219
|
+
|
|
220
|
+
That does not fix the `id` gap above: a connector whose anchor disagrees with
|
|
221
|
+
its `doc_id` still misses this join silently. There is no workaround for that
|
|
222
|
+
here — only the honest statement of it.
|
|
223
|
+
|
|
224
|
+
## Not built
|
|
225
|
+
|
|
226
|
+
- **No full-text search out of the box.** `body` is a plain column; `INSTALL
|
|
227
|
+
fts` and an index over it is a documented recipe, not a feature.
|
|
228
|
+
- **No remote bundles.** `--bundle` is a local checkout path; reading one over
|
|
229
|
+
HTTP wants a caching story this package deliberately avoids.
|
|
230
|
+
- **No MCP server.** `load()` is already small enough to wrap in one `query`
|
|
231
|
+
tool and a schema resource; nothing here does that yet.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "kbforge-okfquery"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "SQL over an OKF v0.2 bundle, via DuckDB"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.12"
|
|
7
|
+
authors = [{ name = "Qing", email = "qingye779@gmail.com" }]
|
|
8
|
+
license = { text = "MIT" }
|
|
9
|
+
keywords = ["okf", "open-knowledge-format", "duckdb", "sql", "knowledge-base"]
|
|
10
|
+
dependencies = [
|
|
11
|
+
"duckdb>=1.1",
|
|
12
|
+
"pyyaml>=6",
|
|
13
|
+
# DuckDB's Python client needs this to materialize ANY TIMESTAMPTZ column as
|
|
14
|
+
# a Python datetime, with or without a session timezone set -- without it,
|
|
15
|
+
# fetchone() on a TIMESTAMPTZ column raises InvalidInputException. `load`
|
|
16
|
+
# also does `SET timezone = 'UTC'`, but that's about rendering the same
|
|
17
|
+
# value the same way on every machine, not about pytz being needed at all.
|
|
18
|
+
"pytz>=2024.1",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.scripts]
|
|
22
|
+
okfquery = "okfquery.cli:main"
|
|
23
|
+
|
|
24
|
+
[build-system]
|
|
25
|
+
requires = ["hatchling"]
|
|
26
|
+
build-backend = "hatchling.build"
|
|
27
|
+
|
|
28
|
+
[tool.hatch.build.targets.wheel]
|
|
29
|
+
packages = ["src/okfquery"]
|
|
30
|
+
|
|
31
|
+
# kbforge is needed ONLY by tests/test_roundtrip.py, which builds a real bundle
|
|
32
|
+
# to check this package's replicated reserved rule has not drifted from
|
|
33
|
+
# validate._check_strict_okf. A PEP 735 group is never written into wheel
|
|
34
|
+
# metadata, so the published okfquery still depends on no kbforge code -- which
|
|
35
|
+
# is the claim spec §1 rests on.
|
|
36
|
+
[dependency-groups]
|
|
37
|
+
dev = ["kbforge"]
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"""SQL over an OKF v0.2 bundle.
|
|
2
|
+
|
|
3
|
+
`load()` returns a DuckDB connection, deliberately unwrapped: an OKF bundle is
|
|
4
|
+
just a DuckDB database, and anything this package wrapped would be a worse
|
|
5
|
+
version of what DuckDB already offers."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
__version__ = "0.1.0"
|
|
10
|
+
|
|
11
|
+
from okfquery.load import EmptyMirrorError, load
|
|
12
|
+
from okfquery.schema import SCHEMA_SQL
|
|
13
|
+
|
|
14
|
+
__all__ = ["EmptyMirrorError", "SCHEMA_SQL", "__version__", "load"]
|