kbforge-okfquery 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. kbforge_okfquery-0.1.0/.gitignore +19 -0
  2. kbforge_okfquery-0.1.0/PKG-INFO +244 -0
  3. kbforge_okfquery-0.1.0/README.md +231 -0
  4. kbforge_okfquery-0.1.0/pyproject.toml +37 -0
  5. kbforge_okfquery-0.1.0/src/okfquery/__init__.py +14 -0
  6. kbforge_okfquery-0.1.0/src/okfquery/cli.py +144 -0
  7. kbforge_okfquery-0.1.0/src/okfquery/load.py +132 -0
  8. kbforge_okfquery-0.1.0/src/okfquery/parse.py +252 -0
  9. kbforge_okfquery-0.1.0/src/okfquery/schema.py +51 -0
  10. kbforge_okfquery-0.1.0/tests/fixtures/clean/concepts/api/overview.md +18 -0
  11. kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/badyaml/overview.md +6 -0
  12. kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/claims/index.md +16 -0
  13. kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/incomplete/overview.md +12 -0
  14. kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/index.md +3 -0
  15. kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/log.md +3 -0
  16. kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/nofence/overview.md +3 -0
  17. kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/notmapping/overview.md +6 -0
  18. kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/ok/overview.md +22 -0
  19. kbforge_okfquery-0.1.0/tests/fixtures/messy/concepts/unterminated/overview.md +3 -0
  20. kbforge_okfquery-0.1.0/tests/test_load.py +241 -0
  21. kbforge_okfquery-0.1.0/tests/test_okfquery_cli.py +97 -0
  22. kbforge_okfquery-0.1.0/tests/test_okfquery_live.py +208 -0
  23. kbforge_okfquery-0.1.0/tests/test_parse.py +158 -0
  24. kbforge_okfquery-0.1.0/tests/test_roundtrip.py +127 -0
@@ -0,0 +1,19 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ *.egg-info/
4
+ dist/
5
+ build/
6
+ .eggs/
7
+ *.egg
8
+ .venv/
9
+ .pytest_cache/
10
+ .ruff_cache/
11
+ .coverage
12
+
13
+ # Local environment / secrets — never commit real keys
14
+ .env
15
+ .env.*
16
+ !.env.example
17
+
18
+ # subagent-driven-development scratch (ledger, briefs, reports)
19
+ .superpowers/
@@ -0,0 +1,244 @@
1
+ Metadata-Version: 2.5
2
+ Name: kbforge-okfquery
3
+ Version: 0.1.0
4
+ Summary: SQL over an OKF v0.2 bundle, via DuckDB
5
+ Author-email: Qing <qingye779@gmail.com>
6
+ License: MIT
7
+ Keywords: duckdb,knowledge-base,okf,open-knowledge-format,sql
8
+ Requires-Python: >=3.12
9
+ Requires-Dist: duckdb>=1.1
10
+ Requires-Dist: pytz>=2024.1
11
+ Requires-Dist: pyyaml>=6
12
+ Description-Content-Type: text/markdown
13
+
14
+ # okfquery
15
+
16
+ SQL over an [OKF v0.2](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md)
17
+ bundle, via DuckDB. `okfquery` loads a published bundle into an in-memory DuckDB
18
+ connection and hands the connection back — no daemon, no cache, no committed
19
+ artifact. It depends on no kbforge code at runtime, so it reads any OKF v0.2
20
+ bundle, kbforge-built or not.
21
+
22
+ ## Install and use
23
+
24
+ ```bash
25
+ uv pip install kbforge-okfquery # or: pip install kbforge-okfquery
26
+ okfquery query "select count(*) from concepts" --bundle path/to/bundle
27
+ okfquery shell --bundle path/to/bundle
28
+ ```
29
+
30
+ `query` runs one SQL statement and prints the result (`--format table|json|csv`).
31
+ `shell` writes a temporary `.duckdb` file, opens the `duckdb` CLI on it, and
32
+ deletes the file when the shell exits.
33
+
34
+ ## Schema
35
+
36
+ ```
37
+ $ okfquery schema
38
+ CREATE TABLE concepts (
39
+ path VARCHAR NOT NULL,
40
+ type VARCHAR,
41
+ title VARCHAR,
42
+ description VARCHAR,
43
+ generated_by VARCHAR,
44
+ -- TIMESTAMPTZ, never TIMESTAMP. Not because a naive column would corrupt
45
+ -- the instant on this package's load path -- `load` binds an aware Python
46
+ -- datetime through executemany, so the instant survives a naive column
47
+ -- fine. What a naive column loses is the type: values come back with no
48
+ -- tzinfo, so every comparison against now() (itself TIMESTAMPTZ) needs a
49
+ -- cast, and the column asserts UTC by convention with nothing recording
50
+ -- that it does. §4.4 law 4 exists to force an *aware* stamp; a column that
51
+ -- cannot hold one throws away exactly the property the law buys.
52
+ generated_at TIMESTAMPTZ,
53
+ facets JSON,
54
+ body VARCHAR
55
+ );
56
+
57
+ CREATE TABLE sources (
58
+ path VARCHAR NOT NULL,
59
+ -- A real column. kbforge's synthesize.assemble puts the OWNING anchor first
60
+ -- and grounding anchors after it, and that ordering is the only thing
61
+ -- telling owner from ground -- OKF has no field for it. Discarding ordinal
62
+ -- during unnest would destroy the signal.
63
+ ordinal INTEGER NOT NULL,
64
+ id VARCHAR,
65
+ resource VARCHAR,
66
+ content_hash VARCHAR
67
+ );
68
+
69
+ CREATE TABLE links (
70
+ path VARCHAR NOT NULL,
71
+ target VARCHAR NOT NULL
72
+ );
73
+
74
+ CREATE TABLE problems (
75
+ path VARCHAR NOT NULL,
76
+ kind VARCHAR NOT NULL,
77
+ detail VARCHAR NOT NULL
78
+ );
79
+ ```
80
+
81
+ `path` is the bundle-relative concept path and the join key throughout.
82
+ `sources.ordinal = 0` is the owning source for a kbforge-produced bundle (the
83
+ convention `synthesize.assemble` writes) — on a foreign bundle it is merely
84
+ source order, with no error to tell you that. `facets` is one JSON column, open
85
+ by construction, so `facets->>'owner'` reaches whatever a connector attached.
86
+ `problems` is additive: a file that failed to parse still gets a `concepts` row
87
+ with NULLs for what could not be read, plus one `problems` row per distinct
88
+ failure — nothing a broken concept says gets silently dropped from an audit.
89
+
90
+ ## Queries, run against the messy test fixture
91
+
92
+ All four queries below were run as shown against
93
+ `packages/okfquery/tests/fixtures/messy` — 9 `.md` files under `concepts/`, 7 of
94
+ them concepts (`concepts/index.md` and `concepts/log.md` are reserved and
95
+ fenceless, so they are skipped; `concepts/claims/index.md` is reserved-*named*
96
+ but carries frontmatter, so it counts).
97
+
98
+ **What is built on a given upstream document:**
99
+
100
+ ```bash
101
+ $ okfquery query "
102
+ SELECT c.path, s.ordinal
103
+ FROM concepts c JOIN sources s USING (path)
104
+ WHERE s.resource = 'https://wiki/ok';
105
+ " --bundle packages/okfquery/tests/fixtures/messy
106
+ ┌─────────────────────────┬─────────┐
107
+ │ path │ ordinal │
108
+ │ varchar │ int32 │
109
+ ├─────────────────────────┼─────────┤
110
+ │ concepts/ok/overview.md │ 0 │
111
+ └─────────────────────────┴─────────┘
112
+ ```
113
+
114
+ **Staleness by owning system** (`ordinal = 0` picks the owning source, not a
115
+ grounding one — see the ordinal note above):
116
+
117
+ ```bash
118
+ $ okfquery query "
119
+ SELECT split_part(s.id, ':', 1) AS system,
120
+ count(*) AS concepts,
121
+ min(c.generated_at) AS oldest
122
+ FROM concepts c JOIN sources s USING (path)
123
+ WHERE s.ordinal = 0
124
+ GROUP BY 1 ORDER BY oldest;
125
+ " --bundle packages/okfquery/tests/fixtures/messy
126
+ ┌─────────┬──────────┬──────────────────────────┐
127
+ │ system │ concepts │ oldest │
128
+ │ varchar │ int64 │ timestamp with time zone │
129
+ ├─────────┼──────────┼──────────────────────────┤
130
+ │ wiki │ 3 │ 2026-08-23 00:00:00+00 │
131
+ └─────────┴──────────┴──────────────────────────┘
132
+ ```
133
+
134
+ Only one system shows up on this fixture: most of the messy bundle's files
135
+ exist to exercise a `problems` kind and never got far enough to have a
136
+ `sources` entry at all, so they don't join into this query — which is correct,
137
+ not a bug in the query.
138
+
139
+ **Orphans — nothing links to them:**
140
+
141
+ ```bash
142
+ $ okfquery query "
143
+ SELECT c.path FROM concepts c
144
+ WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.target = c.path);
145
+ " --bundle packages/okfquery/tests/fixtures/messy
146
+ ┌───────────────────────────────────┐
147
+ │ path │
148
+ │ varchar │
149
+ ├───────────────────────────────────┤
150
+ │ concepts/badyaml/overview.md │
151
+ │ concepts/claims/index.md │
152
+ │ concepts/incomplete/overview.md │
153
+ │ concepts/nofence/overview.md │
154
+ │ concepts/notmapping/overview.md │
155
+ │ concepts/unterminated/overview.md │
156
+ └───────────────────────────────────┘
157
+ ```
158
+
159
+ **Everything that failed to parse** (`--format csv` here, since the default
160
+ table renderer truncates long `detail` strings):
161
+
162
+ ```bash
163
+ $ okfquery query "SELECT path, kind, detail FROM problems ORDER BY path, kind;" \
164
+ --bundle packages/okfquery/tests/fixtures/messy --format csv
165
+ path,kind,detail
166
+ concepts/badyaml/overview.md,invalid-yaml,frontmatter is not valid YAML: ParserError
167
+ concepts/incomplete/overview.md,bad-links,'links' entry 3 is not a string
168
+ concepts/incomplete/overview.md,bad-sources,'sources' entry 0 has no 'resource' (§5.1 requires one)
169
+ concepts/incomplete/overview.md,bad-timestamp,'generated.at' 'not-a-date' is not an ISO-8601 datetime
170
+ concepts/incomplete/overview.md,missing-required,"missing required OKF keys: title, description"
171
+ concepts/nofence/overview.md,no-frontmatter,file does not open a '---' frontmatter fence
172
+ concepts/notmapping/overview.md,frontmatter-not-mapping,"frontmatter parses to list, not a mapping"
173
+ concepts/unterminated/overview.md,unterminated-frontmatter,frontmatter fence is opened but never closed by a closing '---'
174
+ ```
175
+
176
+ ## The mirror, and why both obvious joins are wrong
177
+
178
+ `--mirror <path>` attaches a `mirror` view over a kbforge mirror directory
179
+ (`read_json_auto('<mirror>/*.json')`) — the same `CanonicalDocument` slots
180
+ `mirror.slot_key` writes, `anchor` struct included. It answers a different
181
+ question than the bundle alone can: not "what does this concept say" but "does
182
+ it still say what its source currently does."
183
+
184
+ **Both of the joins you'd reach for first are lossy, in different ways:**
185
+
186
+ - `sources.id = mirror.doc_id` holds for every shipped connector, but
187
+ `synthesize._source_entry` builds `id` from `anchor.system`/`anchor.native_id`
188
+ while `doc_id` is the mirror's own field — nothing enforces the two agree for
189
+ a third-party connector. Where they diverge, this join produces a silent
190
+ false miss, not an error.
191
+ - `sources.content_hash = mirror.anchor.content_hash` as an **equi-join** is not
192
+ the sound alternative either. `kbforge.pipeline.run` calls
193
+ `commit(mirror_path, docs)` right after `kbforge_publish`, and publishing only
194
+ opens a review request — kbforge never merges. Between publish and merge the
195
+ mirror already holds the new hash while the bundle still carries the previous
196
+ run's, so an equi-join on hash returns **zero rows for every concept with a
197
+ review request open** — exactly the ones an audit most wants to see.
198
+
199
+ The correct form: join on `id` with a **`LEFT JOIN`**, not an inner join, then
200
+ **compare** hashes instead of joining on them — an inner join silently drops
201
+ every source with no mirror row at all, which is exactly the "never
202
+ published" case the query needs to report. A match means current; a mismatch
203
+ means an update is sitting in review; a source with no mirror row was never
204
+ published. Run for real against the messy fixture (`concepts/ok/overview.md`'s
205
+ two sources — an owning `wiki:ok` and a grounding `notes:ok` — plus two other
206
+ concepts whose sources have no mirror row at all) with a synthetic
207
+ two-document mirror covering only `wiki:ok` and `notes:ok`, one hash left
208
+ matching and one changed to simulate an update stuck in review:
209
+
210
+ ```bash
211
+ $ okfquery query "
212
+ SELECT s.path,
213
+ s.id,
214
+ s.content_hash AS bundle_hash,
215
+ m.anchor.content_hash AS mirror_hash,
216
+ CASE WHEN m.doc_id IS NULL THEN 'never published'
217
+ WHEN s.content_hash = m.anchor.content_hash THEN 'current'
218
+ ELSE 'update in review' END AS status
219
+ FROM sources s LEFT JOIN mirror m ON s.id = m.doc_id
220
+ ORDER BY s.path, s.id;
221
+ " --bundle packages/okfquery/tests/fixtures/messy --mirror /path/to/mirror
222
+ ┌─────────────────────────────────┬─────────────────┬─────────────┬─────────────┬──────────────────┐
223
+ │ path │ id │ bundle_hash │ mirror_hash │ status │
224
+ │ varchar │ varchar │ varchar │ varchar │ varchar │
225
+ ├─────────────────────────────────┼─────────────────┼─────────────┼─────────────┼──────────────────┤
226
+ │ concepts/claims/index.md │ wiki:claims │ h-claims │ NULL │ never published │
227
+ │ concepts/incomplete/overview.md │ wiki:incomplete │ NULL │ NULL │ never published │
228
+ │ concepts/ok/overview.md │ notes:ok │ h-ground │ h-ground-v2 │ update in review │
229
+ │ concepts/ok/overview.md │ wiki:ok │ h-own │ h-own │ current │
230
+ └─────────────────────────────────┴─────────────────┴─────────────┴─────────────┴──────────────────┘
231
+ ```
232
+
233
+ That does not fix the `id` gap above: a connector whose anchor disagrees with
234
+ its `doc_id` still misses this join silently. There is no workaround for that
235
+ here — only the honest statement of it.
236
+
237
+ ## Not built
238
+
239
+ - **No full-text search out of the box.** `body` is a plain column; `INSTALL
240
+ fts` and an index over it is a documented recipe, not a feature.
241
+ - **No remote bundles.** `--bundle` is a local checkout path; reading one over
242
+ HTTP wants a caching story this package deliberately avoids.
243
+ - **No MCP server.** `load()` is already small enough to wrap in one `query`
244
+ tool and a schema resource; nothing here does that yet.
@@ -0,0 +1,231 @@
1
+ # okfquery
2
+
3
+ SQL over an [OKF v0.2](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md)
4
+ bundle, via DuckDB. `okfquery` loads a published bundle into an in-memory DuckDB
5
+ connection and hands the connection back — no daemon, no cache, no committed
6
+ artifact. It depends on no kbforge code at runtime, so it reads any OKF v0.2
7
+ bundle, kbforge-built or not.
8
+
9
+ ## Install and use
10
+
11
+ ```bash
12
+ uv pip install kbforge-okfquery # or: pip install kbforge-okfquery
13
+ okfquery query "select count(*) from concepts" --bundle path/to/bundle
14
+ okfquery shell --bundle path/to/bundle
15
+ ```
16
+
17
+ `query` runs one SQL statement and prints the result (`--format table|json|csv`).
18
+ `shell` writes a temporary `.duckdb` file, opens the `duckdb` CLI on it, and
19
+ deletes the file when the shell exits.
20
+
21
+ ## Schema
22
+
23
+ ```
24
+ $ okfquery schema
25
+ CREATE TABLE concepts (
26
+ path VARCHAR NOT NULL,
27
+ type VARCHAR,
28
+ title VARCHAR,
29
+ description VARCHAR,
30
+ generated_by VARCHAR,
31
+ -- TIMESTAMPTZ, never TIMESTAMP. Not because a naive column would corrupt
32
+ -- the instant on this package's load path -- `load` binds an aware Python
33
+ -- datetime through executemany, so the instant survives a naive column
34
+ -- fine. What a naive column loses is the type: values come back with no
35
+ -- tzinfo, so every comparison against now() (itself TIMESTAMPTZ) needs a
36
+ -- cast, and the column asserts UTC by convention with nothing recording
37
+ -- that it does. §4.4 law 4 exists to force an *aware* stamp; a column that
38
+ -- cannot hold one throws away exactly the property the law buys.
39
+ generated_at TIMESTAMPTZ,
40
+ facets JSON,
41
+ body VARCHAR
42
+ );
43
+
44
+ CREATE TABLE sources (
45
+ path VARCHAR NOT NULL,
46
+ -- A real column. kbforge's synthesize.assemble puts the OWNING anchor first
47
+ -- and grounding anchors after it, and that ordering is the only thing
48
+ -- telling owner from ground -- OKF has no field for it. Discarding ordinal
49
+ -- during unnest would destroy the signal.
50
+ ordinal INTEGER NOT NULL,
51
+ id VARCHAR,
52
+ resource VARCHAR,
53
+ content_hash VARCHAR
54
+ );
55
+
56
+ CREATE TABLE links (
57
+ path VARCHAR NOT NULL,
58
+ target VARCHAR NOT NULL
59
+ );
60
+
61
+ CREATE TABLE problems (
62
+ path VARCHAR NOT NULL,
63
+ kind VARCHAR NOT NULL,
64
+ detail VARCHAR NOT NULL
65
+ );
66
+ ```
67
+
68
+ `path` is the bundle-relative concept path and the join key throughout.
69
+ `sources.ordinal = 0` is the owning source for a kbforge-produced bundle (the
70
+ convention `synthesize.assemble` writes) — on a foreign bundle it is merely
71
+ source order, with no error to tell you that. `facets` is one JSON column, open
72
+ by construction, so `facets->>'owner'` reaches whatever a connector attached.
73
+ `problems` is additive: a file that failed to parse still gets a `concepts` row
74
+ with NULLs for what could not be read, plus one `problems` row per distinct
75
+ failure — nothing a broken concept says gets silently dropped from an audit.
76
+
77
+ ## Queries, run against the messy test fixture
78
+
79
+ All four queries below were run as shown against
80
+ `packages/okfquery/tests/fixtures/messy` — 9 `.md` files under `concepts/`, 7 of
81
+ them concepts (`concepts/index.md` and `concepts/log.md` are reserved and
82
+ fenceless, so they are skipped; `concepts/claims/index.md` is reserved-*named*
83
+ but carries frontmatter, so it counts).
84
+
85
+ **What is built on a given upstream document:**
86
+
87
+ ```bash
88
+ $ okfquery query "
89
+ SELECT c.path, s.ordinal
90
+ FROM concepts c JOIN sources s USING (path)
91
+ WHERE s.resource = 'https://wiki/ok';
92
+ " --bundle packages/okfquery/tests/fixtures/messy
93
+ ┌─────────────────────────┬─────────┐
94
+ │ path │ ordinal │
95
+ │ varchar │ int32 │
96
+ ├─────────────────────────┼─────────┤
97
+ │ concepts/ok/overview.md │ 0 │
98
+ └─────────────────────────┴─────────┘
99
+ ```
100
+
101
+ **Staleness by owning system** (`ordinal = 0` picks the owning source, not a
102
+ grounding one — see the ordinal note above):
103
+
104
+ ```bash
105
+ $ okfquery query "
106
+ SELECT split_part(s.id, ':', 1) AS system,
107
+ count(*) AS concepts,
108
+ min(c.generated_at) AS oldest
109
+ FROM concepts c JOIN sources s USING (path)
110
+ WHERE s.ordinal = 0
111
+ GROUP BY 1 ORDER BY oldest;
112
+ " --bundle packages/okfquery/tests/fixtures/messy
113
+ ┌─────────┬──────────┬──────────────────────────┐
114
+ │ system │ concepts │ oldest │
115
+ │ varchar │ int64 │ timestamp with time zone │
116
+ ├─────────┼──────────┼──────────────────────────┤
117
+ │ wiki │ 3 │ 2026-08-23 00:00:00+00 │
118
+ └─────────┴──────────┴──────────────────────────┘
119
+ ```
120
+
121
+ Only one system shows up on this fixture: most of the messy bundle's files
122
+ exist to exercise a `problems` kind and never got far enough to have a
123
+ `sources` entry at all, so they don't join into this query — which is correct,
124
+ not a bug in the query.
125
+
126
+ **Orphans — nothing links to them:**
127
+
128
+ ```bash
129
+ $ okfquery query "
130
+ SELECT c.path FROM concepts c
131
+ WHERE NOT EXISTS (SELECT 1 FROM links l WHERE l.target = c.path);
132
+ " --bundle packages/okfquery/tests/fixtures/messy
133
+ ┌───────────────────────────────────┐
134
+ │ path │
135
+ │ varchar │
136
+ ├───────────────────────────────────┤
137
+ │ concepts/badyaml/overview.md │
138
+ │ concepts/claims/index.md │
139
+ │ concepts/incomplete/overview.md │
140
+ │ concepts/nofence/overview.md │
141
+ │ concepts/notmapping/overview.md │
142
+ │ concepts/unterminated/overview.md │
143
+ └───────────────────────────────────┘
144
+ ```
145
+
146
+ **Everything that failed to parse** (`--format csv` here, since the default
147
+ table renderer truncates long `detail` strings):
148
+
149
+ ```bash
150
+ $ okfquery query "SELECT path, kind, detail FROM problems ORDER BY path, kind;" \
151
+ --bundle packages/okfquery/tests/fixtures/messy --format csv
152
+ path,kind,detail
153
+ concepts/badyaml/overview.md,invalid-yaml,frontmatter is not valid YAML: ParserError
154
+ concepts/incomplete/overview.md,bad-links,'links' entry 3 is not a string
155
+ concepts/incomplete/overview.md,bad-sources,'sources' entry 0 has no 'resource' (§5.1 requires one)
156
+ concepts/incomplete/overview.md,bad-timestamp,'generated.at' 'not-a-date' is not an ISO-8601 datetime
157
+ concepts/incomplete/overview.md,missing-required,"missing required OKF keys: title, description"
158
+ concepts/nofence/overview.md,no-frontmatter,file does not open a '---' frontmatter fence
159
+ concepts/notmapping/overview.md,frontmatter-not-mapping,"frontmatter parses to list, not a mapping"
160
+ concepts/unterminated/overview.md,unterminated-frontmatter,frontmatter fence is opened but never closed by a closing '---'
161
+ ```
162
+
163
+ ## The mirror, and why both obvious joins are wrong
164
+
165
+ `--mirror <path>` attaches a `mirror` view over a kbforge mirror directory
166
+ (`read_json_auto('<mirror>/*.json')`) — the same `CanonicalDocument` slots
167
+ `mirror.slot_key` writes, `anchor` struct included. It answers a different
168
+ question than the bundle alone can: not "what does this concept say" but "does
169
+ it still say what its source currently does."
170
+
171
+ **Both of the joins you'd reach for first are lossy, in different ways:**
172
+
173
+ - `sources.id = mirror.doc_id` holds for every shipped connector, but
174
+ `synthesize._source_entry` builds `id` from `anchor.system`/`anchor.native_id`
175
+ while `doc_id` is the mirror's own field — nothing enforces the two agree for
176
+ a third-party connector. Where they diverge, this join produces a silent
177
+ false miss, not an error.
178
+ - `sources.content_hash = mirror.anchor.content_hash` as an **equi-join** is not
179
+ the sound alternative either. `kbforge.pipeline.run` calls
180
+ `commit(mirror_path, docs)` right after `kbforge_publish`, and publishing only
181
+ opens a review request — kbforge never merges. Between publish and merge the
182
+ mirror already holds the new hash while the bundle still carries the previous
183
+ run's, so an equi-join on hash returns **zero rows for every concept with a
184
+ review request open** — exactly the ones an audit most wants to see.
185
+
186
+ The correct form: join on `id` with a **`LEFT JOIN`**, not an inner join, then
187
+ **compare** hashes instead of joining on them — an inner join silently drops
188
+ every source with no mirror row at all, which is exactly the "never
189
+ published" case the query needs to report. A match means current; a mismatch
190
+ means an update is sitting in review; a source with no mirror row was never
191
+ published. Run for real against the messy fixture (`concepts/ok/overview.md`'s
192
+ two sources — an owning `wiki:ok` and a grounding `notes:ok` — plus two other
193
+ concepts whose sources have no mirror row at all) with a synthetic
194
+ two-document mirror covering only `wiki:ok` and `notes:ok`, one hash left
195
+ matching and one changed to simulate an update stuck in review:
196
+
197
+ ```bash
198
+ $ okfquery query "
199
+ SELECT s.path,
200
+ s.id,
201
+ s.content_hash AS bundle_hash,
202
+ m.anchor.content_hash AS mirror_hash,
203
+ CASE WHEN m.doc_id IS NULL THEN 'never published'
204
+ WHEN s.content_hash = m.anchor.content_hash THEN 'current'
205
+ ELSE 'update in review' END AS status
206
+ FROM sources s LEFT JOIN mirror m ON s.id = m.doc_id
207
+ ORDER BY s.path, s.id;
208
+ " --bundle packages/okfquery/tests/fixtures/messy --mirror /path/to/mirror
209
+ ┌─────────────────────────────────┬─────────────────┬─────────────┬─────────────┬──────────────────┐
210
+ │ path │ id │ bundle_hash │ mirror_hash │ status │
211
+ │ varchar │ varchar │ varchar │ varchar │ varchar │
212
+ ├─────────────────────────────────┼─────────────────┼─────────────┼─────────────┼──────────────────┤
213
+ │ concepts/claims/index.md │ wiki:claims │ h-claims │ NULL │ never published │
214
+ │ concepts/incomplete/overview.md │ wiki:incomplete │ NULL │ NULL │ never published │
215
+ │ concepts/ok/overview.md │ notes:ok │ h-ground │ h-ground-v2 │ update in review │
216
+ │ concepts/ok/overview.md │ wiki:ok │ h-own │ h-own │ current │
217
+ └─────────────────────────────────┴─────────────────┴─────────────┴─────────────┴──────────────────┘
218
+ ```
219
+
220
+ That does not fix the `id` gap above: a connector whose anchor disagrees with
221
+ its `doc_id` still misses this join silently. There is no workaround for that
222
+ here — only the honest statement of it.
223
+
224
+ ## Not built
225
+
226
+ - **No full-text search out of the box.** `body` is a plain column; `INSTALL
227
+ fts` and an index over it is a documented recipe, not a feature.
228
+ - **No remote bundles.** `--bundle` is a local checkout path; reading one over
229
+ HTTP wants a caching story this package deliberately avoids.
230
+ - **No MCP server.** `load()` is already small enough to wrap in one `query`
231
+ tool and a schema resource; nothing here does that yet.
@@ -0,0 +1,37 @@
1
+ [project]
2
+ name = "kbforge-okfquery"
3
+ version = "0.1.0"
4
+ description = "SQL over an OKF v0.2 bundle, via DuckDB"
5
+ readme = "README.md"
6
+ requires-python = ">=3.12"
7
+ authors = [{ name = "Qing", email = "qingye779@gmail.com" }]
8
+ license = { text = "MIT" }
9
+ keywords = ["okf", "open-knowledge-format", "duckdb", "sql", "knowledge-base"]
10
+ dependencies = [
11
+ "duckdb>=1.1",
12
+ "pyyaml>=6",
13
+ # DuckDB's Python client needs this to materialize ANY TIMESTAMPTZ column as
14
+ # a Python datetime, with or without a session timezone set -- without it,
15
+ # fetchone() on a TIMESTAMPTZ column raises InvalidInputException. `load`
16
+ # also does `SET timezone = 'UTC'`, but that's about rendering the same
17
+ # value the same way on every machine, not about pytz being needed at all.
18
+ "pytz>=2024.1",
19
+ ]
20
+
21
+ [project.scripts]
22
+ okfquery = "okfquery.cli:main"
23
+
24
+ [build-system]
25
+ requires = ["hatchling"]
26
+ build-backend = "hatchling.build"
27
+
28
+ [tool.hatch.build.targets.wheel]
29
+ packages = ["src/okfquery"]
30
+
31
+ # kbforge is needed ONLY by tests/test_roundtrip.py, which builds a real bundle
32
+ # to check this package's replicated reserved rule has not drifted from
33
+ # validate._check_strict_okf. A PEP 735 group is never written into wheel
34
+ # metadata, so the published okfquery still depends on no kbforge code -- which
35
+ # is the claim spec §1 rests on.
36
+ [dependency-groups]
37
+ dev = ["kbforge"]
@@ -0,0 +1,14 @@
1
+ """SQL over an OKF v0.2 bundle.
2
+
3
+ `load()` returns a DuckDB connection, deliberately unwrapped: an OKF bundle is
4
+ just a DuckDB database, and anything this package wrapped would be a worse
5
+ version of what DuckDB already offers."""
6
+
7
+ from __future__ import annotations
8
+
9
+ __version__ = "0.1.0"
10
+
11
+ from okfquery.load import EmptyMirrorError, load
12
+ from okfquery.schema import SCHEMA_SQL
13
+
14
+ __all__ = ["EmptyMirrorError", "SCHEMA_SQL", "__version__", "load"]