kgmodule-utils 0.27.0__tar.gz → 0.28.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/PKG-INFO +32 -7
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/README.md +29 -4
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/pyproject.toml +11 -8
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/__init__.py +1 -1
- kgmodule_utils-0.28.0/src/kg_utils/devicepack.py +717 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/vector_backend.py +25 -1
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/LICENSE +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/analysis/__init__.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/analysis/scores.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/corpus_embedder.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/embed.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/embedder.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/extractor.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/__init__.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/converters.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/derived.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/manifest.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/pipeline.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/module.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/pipeline.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/py.typed +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/retrieval/__init__.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/retrieval/hits.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/semantic.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/snapshots/__init__.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/snapshots/manager.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/snapshots/models.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/specs.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/store.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/__init__.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/_config.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/_image.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/_text.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/factory.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/temporal.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/validation.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz/__init__.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz/graph_html.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz/theme.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz/tooltip.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/__init__.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/layout.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/organic.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/qt.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/species.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/worker/__init__.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/worker/client.py +0 -0
- {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/worker/ops.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kgmodule-utils
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.28.0
|
|
4
4
|
Summary: Shared types, graph store, semantic index, and pipeline base for the KGModule SDK
|
|
5
5
|
License-Expression: Elastic-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -38,8 +38,8 @@ Requires-Dist: openai (>=1.30.0) ; extra == "synthesis"
|
|
|
38
38
|
Requires-Dist: openai (>=1.30.0) ; extra == "synthesis-mflux"
|
|
39
39
|
Requires-Dist: pillow (>=10.0.0) ; extra == "synthesis"
|
|
40
40
|
Requires-Dist: pillow (>=10.0.0) ; extra == "synthesis-mflux"
|
|
41
|
-
Requires-Dist:
|
|
42
|
-
Requires-Dist:
|
|
41
|
+
Requires-Dist: pymupdf (>=2.0) ; extra == "ingest"
|
|
42
|
+
Requires-Dist: pymupdf (>=2.0) ; extra == "pdf"
|
|
43
43
|
Requires-Dist: pyvis (>=0.3.2) ; extra == "viz"
|
|
44
44
|
Requires-Dist: pyvista (>=0.44.0) ; extra == "viz3d-qt"
|
|
45
45
|
Requires-Dist: pyvista (>=0.44.0) ; extra == "viz3d-render"
|
|
@@ -56,7 +56,7 @@ Description-Content-Type: text/markdown
|
|
|
56
56
|
|
|
57
57
|
[](https://www.python.org/)
|
|
58
58
|
[](https://www.elastic.co/licensing/elastic-license)
|
|
59
|
-
[](https://github.com/Flux-Frontiers/KG_utils/releases)
|
|
60
60
|
[](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml)
|
|
61
61
|
[](https://python-poetry.org/)
|
|
62
62
|
[](https://doi.org/10.5281/zenodo.21284866)
|
|
@@ -81,6 +81,15 @@ Fleet modules use it in two ways. [PyCodeKG](https://github.com/Flux-Frontiers/p
|
|
|
81
81
|
|
|
82
82
|
## Latest News
|
|
83
83
|
|
|
84
|
+
- **0.28.0 (2026-10-23)** -- New `kg_utils.devicepack`: the on-device
|
|
85
|
+
search-pack format the Knowledge Press apps read (passage pack, vector
|
|
86
|
+
sidecar, manifest, golden rankings), lifted out of gutenberg_kg's
|
|
87
|
+
`export-swift` so any KG can write it. genealogy_kg's `export-swift` is its
|
|
88
|
+
first user. `SqliteVecBackend.iter_vectors()` reads a store's vectors back.
|
|
89
|
+
- **0.27.1 (2026-10-19)** -- `[pdf]` and `[ingest]` work again on a fresh
|
|
90
|
+
install. pymupdf4llm 2.0 is an empty placeholder (its code now ships inside
|
|
91
|
+
pymupdf 2.0), so 0.27.0's `pymupdf4llm>=1.28.2` resolved to no PDF support.
|
|
92
|
+
Both extras now declare `pymupdf>=2.0`. No API change.
|
|
84
93
|
- **0.27.0 (2026-10-14)** -- One converter per format, and incremental
|
|
85
94
|
vectors. `PdfConverter` is the fleet's single PDF converter (the same
|
|
86
95
|
`pymupdf4llm` call `doc_kg` makes), `PandocConverter` reads `.tex` and
|
|
@@ -219,7 +228,8 @@ The SDK covers the full life of a knowledge graph: core contracts and SQLite
|
|
|
219
228
|
storage (`specs`, `extractor`, `store`, `pipeline`), semantic indexing over
|
|
220
229
|
pluggable vector backends (`semantic`, `vector_backend`, `embed`, `embedder`),
|
|
221
230
|
document ingestion (`ingest`), a shared temporal contract for dated nodes
|
|
222
|
-
(`temporal`),
|
|
231
|
+
(`temporal`), on-device search packs for the Knowledge Press apps
|
|
232
|
+
(`devicepack`), temporal snapshots (`snapshots`), text + image
|
|
223
233
|
synthesis (`synthesis`), centrality analysis (`analysis`), and a visualization
|
|
224
234
|
stack that runs from interactive HTML (`viz`) through 3-D layouts and organic
|
|
225
235
|
trees grown as species (`viz3d`, `viz3d.species`) to Looking Glass light-field
|
|
@@ -575,6 +585,21 @@ about the host window.
|
|
|
575
585
|
| `CastResult` | Outcome of one cast: `path`, `error`, `elapsed`, and the `message` a status bar shows |
|
|
576
586
|
| `DEFAULT_QUILT_PRESET` / `DEFAULT_CAST_SCALE` | The preset and scale a cast uses when no `spec` is given (`"16-landscape"` at half size) |
|
|
577
587
|
|
|
588
|
+
### `kg_utils.devicepack`
|
|
589
|
+
|
|
590
|
+
The on-device search-pack format: what the Knowledge Press apps open with no
|
|
591
|
+
server. Needs NumPy for vectors (`semantic` or `viz3d` extra).
|
|
592
|
+
|
|
593
|
+
| Class / function | Description |
|
|
594
|
+
|---|---|
|
|
595
|
+
| `Passage` | One searchable row: id, KG, kind, content, and the optional title, author, genre and location columns |
|
|
596
|
+
| `write_passage_pack()` | Write `<name>.pack` (passages + FTS5) and its `<name>.vectors` sidecar from a vector map; returns `PackStats` |
|
|
597
|
+
| `search_pack()` | The reference hybrid search: dense over the sidecar, BM25 over FTS5 (phrase, then any term), RRF-fused; filters on `genre` only |
|
|
598
|
+
| `build_manifest()` | `manifest.json`: pack version, the embedder, every pack with sizes and checksums |
|
|
599
|
+
| `build_golden()` | `golden.json`: each query's top-k as the packs answer it, the app's parity gate |
|
|
600
|
+
| `read_vector_sidecar()` | Memory-map a sidecar; rejects a truncated or foreign file |
|
|
601
|
+
| `encode_vector()`, `rrf_fuse()`, `fts_phrase_expression()` | The pieces the search is built from |
|
|
602
|
+
|
|
578
603
|
### `kg_utils.analysis`
|
|
579
604
|
|
|
580
605
|
| Class / function | Description |
|
|
@@ -784,7 +809,7 @@ If you use kgmodule-utils in research or a project, please cite it:
|
|
|
784
809
|
|
|
785
810
|
**APA**
|
|
786
811
|
|
|
787
|
-
> Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.
|
|
812
|
+
> Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.28.0) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
|
|
788
813
|
|
|
789
814
|
**BibTeX**
|
|
790
815
|
|
|
@@ -792,7 +817,7 @@ If you use kgmodule-utils in research or a project, please cite it:
|
|
|
792
817
|
@software{suchanek_kgmodule_utils,
|
|
793
818
|
author = {Suchanek, Eric G.},
|
|
794
819
|
title = {{kgmodule-utils}: Shared SDK for the KGModule Knowledge-Graph Ecosystem},
|
|
795
|
-
version = {0.
|
|
820
|
+
version = {0.28.0},
|
|
796
821
|
year = {2026},
|
|
797
822
|
publisher = {Flux-Frontiers},
|
|
798
823
|
url = {https://github.com/Flux-Frontiers/KG_utils},
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
[](https://www.python.org/)
|
|
3
3
|
[](https://www.elastic.co/licensing/elastic-license)
|
|
4
|
-
[](https://github.com/Flux-Frontiers/KG_utils/releases)
|
|
5
5
|
[](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml)
|
|
6
6
|
[](https://python-poetry.org/)
|
|
7
7
|
[](https://doi.org/10.5281/zenodo.21284866)
|
|
@@ -26,6 +26,15 @@ Fleet modules use it in two ways. [PyCodeKG](https://github.com/Flux-Frontiers/p
|
|
|
26
26
|
|
|
27
27
|
## Latest News
|
|
28
28
|
|
|
29
|
+
- **0.28.0 (2026-10-23)** -- New `kg_utils.devicepack`: the on-device
|
|
30
|
+
search-pack format the Knowledge Press apps read (passage pack, vector
|
|
31
|
+
sidecar, manifest, golden rankings), lifted out of gutenberg_kg's
|
|
32
|
+
`export-swift` so any KG can write it. genealogy_kg's `export-swift` is its
|
|
33
|
+
first user. `SqliteVecBackend.iter_vectors()` reads a store's vectors back.
|
|
34
|
+
- **0.27.1 (2026-10-19)** -- `[pdf]` and `[ingest]` work again on a fresh
|
|
35
|
+
install. pymupdf4llm 2.0 is an empty placeholder (its code now ships inside
|
|
36
|
+
pymupdf 2.0), so 0.27.0's `pymupdf4llm>=1.28.2` resolved to no PDF support.
|
|
37
|
+
Both extras now declare `pymupdf>=2.0`. No API change.
|
|
29
38
|
- **0.27.0 (2026-10-14)** -- One converter per format, and incremental
|
|
30
39
|
vectors. `PdfConverter` is the fleet's single PDF converter (the same
|
|
31
40
|
`pymupdf4llm` call `doc_kg` makes), `PandocConverter` reads `.tex` and
|
|
@@ -164,7 +173,8 @@ The SDK covers the full life of a knowledge graph: core contracts and SQLite
|
|
|
164
173
|
storage (`specs`, `extractor`, `store`, `pipeline`), semantic indexing over
|
|
165
174
|
pluggable vector backends (`semantic`, `vector_backend`, `embed`, `embedder`),
|
|
166
175
|
document ingestion (`ingest`), a shared temporal contract for dated nodes
|
|
167
|
-
(`temporal`),
|
|
176
|
+
(`temporal`), on-device search packs for the Knowledge Press apps
|
|
177
|
+
(`devicepack`), temporal snapshots (`snapshots`), text + image
|
|
168
178
|
synthesis (`synthesis`), centrality analysis (`analysis`), and a visualization
|
|
169
179
|
stack that runs from interactive HTML (`viz`) through 3-D layouts and organic
|
|
170
180
|
trees grown as species (`viz3d`, `viz3d.species`) to Looking Glass light-field
|
|
@@ -520,6 +530,21 @@ about the host window.
|
|
|
520
530
|
| `CastResult` | Outcome of one cast: `path`, `error`, `elapsed`, and the `message` a status bar shows |
|
|
521
531
|
| `DEFAULT_QUILT_PRESET` / `DEFAULT_CAST_SCALE` | The preset and scale a cast uses when no `spec` is given (`"16-landscape"` at half size) |
|
|
522
532
|
|
|
533
|
+
### `kg_utils.devicepack`
|
|
534
|
+
|
|
535
|
+
The on-device search-pack format: what the Knowledge Press apps open with no
|
|
536
|
+
server. Needs NumPy for vectors (`semantic` or `viz3d` extra).
|
|
537
|
+
|
|
538
|
+
| Class / function | Description |
|
|
539
|
+
|---|---|
|
|
540
|
+
| `Passage` | One searchable row: id, KG, kind, content, and the optional title, author, genre and location columns |
|
|
541
|
+
| `write_passage_pack()` | Write `<name>.pack` (passages + FTS5) and its `<name>.vectors` sidecar from a vector map; returns `PackStats` |
|
|
542
|
+
| `search_pack()` | The reference hybrid search: dense over the sidecar, BM25 over FTS5 (phrase, then any term), RRF-fused; filters on `genre` only |
|
|
543
|
+
| `build_manifest()` | `manifest.json`: pack version, the embedder, every pack with sizes and checksums |
|
|
544
|
+
| `build_golden()` | `golden.json`: each query's top-k as the packs answer it, the app's parity gate |
|
|
545
|
+
| `read_vector_sidecar()` | Memory-map a sidecar; rejects a truncated or foreign file |
|
|
546
|
+
| `encode_vector()`, `rrf_fuse()`, `fts_phrase_expression()` | The pieces the search is built from |
|
|
547
|
+
|
|
523
548
|
### `kg_utils.analysis`
|
|
524
549
|
|
|
525
550
|
| Class / function | Description |
|
|
@@ -729,7 +754,7 @@ If you use kgmodule-utils in research or a project, please cite it:
|
|
|
729
754
|
|
|
730
755
|
**APA**
|
|
731
756
|
|
|
732
|
-
> Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.
|
|
757
|
+
> Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.28.0) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
|
|
733
758
|
|
|
734
759
|
**BibTeX**
|
|
735
760
|
|
|
@@ -737,7 +762,7 @@ If you use kgmodule-utils in research or a project, please cite it:
|
|
|
737
762
|
@software{suchanek_kgmodule_utils,
|
|
738
763
|
author = {Suchanek, Eric G.},
|
|
739
764
|
title = {{kgmodule-utils}: Shared SDK for the KGModule Knowledge-Graph Ecosystem},
|
|
740
|
-
version = {0.
|
|
765
|
+
version = {0.28.0},
|
|
741
766
|
year = {2026},
|
|
742
767
|
publisher = {Flux-Frontiers},
|
|
743
768
|
url = {https://github.com/Flux-Frontiers/KG_utils},
|
|
@@ -9,7 +9,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
9
9
|
|
|
10
10
|
[project]
|
|
11
11
|
name = "kgmodule-utils"
|
|
12
|
-
version = "0.
|
|
12
|
+
version = "0.28.0"
|
|
13
13
|
description = "Shared types, graph store, semantic index, and pipeline base for the KGModule SDK"
|
|
14
14
|
readme = "README.md"
|
|
15
15
|
license = "Elastic-2.0"
|
|
@@ -83,18 +83,21 @@ lancedb = [
|
|
|
83
83
|
# Document ingestion (kg_utils.ingest): pymupdf4llm converts PDF (the same
|
|
84
84
|
# call doc_kg makes, so both paths agree); anydoc converts Word, PowerPoint,
|
|
85
85
|
# Excel, OpenDocument, RTF, EPUB and CSV. Kept optional because .md/.txt/.rst
|
|
86
|
-
# ingest with no dependency at all.
|
|
87
|
-
#
|
|
88
|
-
#
|
|
89
|
-
#
|
|
86
|
+
# ingest with no dependency at all. pymupdf (AGPL-3.0 or Artifex commercial)
|
|
87
|
+
# is declared, not pymupdf4llm: since 2.0 the pymupdf wheel ships the
|
|
88
|
+
# pymupdf4llm module, and the pymupdf4llm 2.0 distribution is an empty
|
|
89
|
+
# placeholder that installs no code, so declaring it alone resolves to no PDF
|
|
90
|
+
# support at all. .tex/.html need the pandoc binary, which is not a Python
|
|
91
|
+
# dependency. No OCR: scanned PDFs are recorded as failed with a reason,
|
|
92
|
+
# never silently dropped.
|
|
90
93
|
ingest = [
|
|
91
94
|
"firecrawl-anydoc>=0.1.9",
|
|
92
|
-
"
|
|
95
|
+
"pymupdf>=2.0",
|
|
93
96
|
]
|
|
94
97
|
# PDF only: what doc_kg needs to read PDFs through PdfConverter without
|
|
95
98
|
# pulling in anydoc. [ingest] stays the "every format" extra.
|
|
96
99
|
pdf = [
|
|
97
|
-
"
|
|
100
|
+
"pymupdf>=2.0",
|
|
98
101
|
]
|
|
99
102
|
synthesis = [
|
|
100
103
|
"httpx>=0.27.0",
|
|
@@ -126,7 +129,7 @@ pytest = ">=9.0.3"
|
|
|
126
129
|
pytest-cov = "^5.0.0"
|
|
127
130
|
# Exercises the real converters in the ingest tests rather than stubs.
|
|
128
131
|
firecrawl-anydoc = ">=0.1.9"
|
|
129
|
-
|
|
132
|
+
pymupdf = ">=2.0"
|
|
130
133
|
# Capped below 0.16 deliberately: 0.16 expands the default rules and reformats
|
|
131
134
|
# Python in Markdown blocks. Adopt on purpose; lift the cap in its own change.
|
|
132
135
|
ruff = ">=0.15,<0.16"
|
|
@@ -0,0 +1,717 @@
|
|
|
1
|
+
"""kg_utils.devicepack -- on-device search packs, the format the Knowledge Press apps read.
|
|
2
|
+
|
|
3
|
+
A device pack is a corpus cut down to what an app needs to search it with no
|
|
4
|
+
server: a SQLite file of passages with an FTS5 index, a sidecar of int8 (or
|
|
5
|
+
float32) embeddings beside it, a ``manifest.json`` naming the embedder the
|
|
6
|
+
vectors require, and a ``golden.json`` of reference rankings the app's own
|
|
7
|
+
search must reproduce. The Swift kit in ``knowledge_press`` and
|
|
8
|
+
``kpress_family`` reads exactly this; ``gutenberg_kg``'s ``export-swift`` wrote
|
|
9
|
+
it first, and this module is that format lifted out so other KGs can write it.
|
|
10
|
+
Nothing here knows what a passage *is*: a book chunk, a diary entry and a
|
|
11
|
+
person's summary are all a :class:`Passage`.
|
|
12
|
+
|
|
13
|
+
Layout::
|
|
14
|
+
|
|
15
|
+
<out>/
|
|
16
|
+
manifest.json pack version, the embedder, every pack and its checksum
|
|
17
|
+
<name>.pack passages + FTS5 over their text
|
|
18
|
+
<name>.vectors their embeddings, row-major, memory-mappable
|
|
19
|
+
golden.json reference top-k per query, as the packs answer it
|
|
20
|
+
|
|
21
|
+
Why the vectors sit beside the pack
|
|
22
|
+
-----------------------------------
|
|
23
|
+
A ``vec0`` table cannot be read without the sqlite-vec C extension compiled into
|
|
24
|
+
the reader, and iOS ships stock SQLite. The sidecar is a 32-byte header (magic,
|
|
25
|
+
dtype, dimension, count) and then row-major vectors; ``passages.vector_index``
|
|
26
|
+
is a dense row number into it, so a passage with no vector leaves no hole. The
|
|
27
|
+
device memory-maps the file and takes dot products straight out of it.
|
|
28
|
+
|
|
29
|
+
Search
|
|
30
|
+
------
|
|
31
|
+
:func:`search_pack` is the reference the app must match: a dense channel over
|
|
32
|
+
the sidecar, a BM25 channel over FTS5 (exact phrase first, any term as the
|
|
33
|
+
fallback), fused by reciprocal rank at :data:`RRF_K`. It filters on ``genre``
|
|
34
|
+
only, as the Swift engine does; what a pack holds is the exporter's choice.
|
|
35
|
+
|
|
36
|
+
Needs NumPy for anything touching vectors (the ``semantic`` or ``viz3d``
|
|
37
|
+
extra); writing and reading passages does not.
|
|
38
|
+
|
|
39
|
+
Author: Eric G. Suchanek, PhD
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import hashlib
|
|
45
|
+
import sqlite3
|
|
46
|
+
from collections.abc import Iterable, Mapping, Sequence
|
|
47
|
+
from dataclasses import dataclass
|
|
48
|
+
from datetime import UTC, datetime
|
|
49
|
+
from pathlib import Path
|
|
50
|
+
from typing import Any, Final
|
|
51
|
+
|
|
52
|
+
#: The pack format. The app refuses a manifest newer than it understands.
|
|
53
|
+
PACK_VERSION: Final[int] = 1
|
|
54
|
+
|
|
55
|
+
#: First eight bytes of every vector sidecar.
|
|
56
|
+
VECTOR_MAGIC: Final[bytes] = b"GKGVEC01"
|
|
57
|
+
|
|
58
|
+
#: Header bytes before the vector rows. 32 keeps the rows 16-byte aligned, so
|
|
59
|
+
#: the device can map them and hand them straight to SIMD.
|
|
60
|
+
VECTOR_HEADER_BYTES: Final[int] = 32
|
|
61
|
+
|
|
62
|
+
#: Reciprocal-rank-fusion constant, shared by the worker, the packs and the app.
|
|
63
|
+
RRF_K: Final[int] = 60
|
|
64
|
+
|
|
65
|
+
DTYPES: Final[tuple[str, ...]] = ("int8", "float")
|
|
66
|
+
_DTYPE_CODES: Final[dict[str, int]] = {"int8": 0, "float": 1}
|
|
67
|
+
_DTYPE_ITEMSIZE: Final[dict[str, int]] = {"int8": 1, "float": 4}
|
|
68
|
+
|
|
69
|
+
#: Default parity tolerances written into ``golden.json``. ``max_rank_drift``
|
|
70
|
+
#: is not zero because the reference and the device differ in their last bits
|
|
71
|
+
#: (numpy over dequantised int8 against Accelerate), so near-ties may swap.
|
|
72
|
+
DEFAULT_TOLERANCE: Final[dict[str, float | int]] = {
|
|
73
|
+
"rank_overlap": 0.9,
|
|
74
|
+
"score_delta": 0.02,
|
|
75
|
+
"max_rank_drift": 2,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
PASSAGE_SCHEMA: Final[str] = """
|
|
79
|
+
CREATE TABLE pack_meta (key TEXT PRIMARY KEY, value TEXT);
|
|
80
|
+
|
|
81
|
+
CREATE TABLE passages (
|
|
82
|
+
id TEXT PRIMARY KEY,
|
|
83
|
+
kg_name TEXT NOT NULL,
|
|
84
|
+
kg_kind TEXT NOT NULL,
|
|
85
|
+
kind TEXT NOT NULL,
|
|
86
|
+
name TEXT,
|
|
87
|
+
title TEXT,
|
|
88
|
+
node_title TEXT,
|
|
89
|
+
author TEXT,
|
|
90
|
+
genre TEXT,
|
|
91
|
+
book TEXT,
|
|
92
|
+
file_path TEXT,
|
|
93
|
+
char_start INTEGER,
|
|
94
|
+
chapter INTEGER,
|
|
95
|
+
timestamp TEXT,
|
|
96
|
+
vector_index INTEGER,
|
|
97
|
+
content TEXT NOT NULL
|
|
98
|
+
);
|
|
99
|
+
CREATE INDEX idx_passages_scope ON passages(genre, kind);
|
|
100
|
+
CREATE INDEX idx_passages_vec ON passages(vector_index);
|
|
101
|
+
CREATE INDEX idx_passages_read ON passages(file_path, kind, char_start);
|
|
102
|
+
CREATE INDEX idx_passages_kg ON passages(kg_name);
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
FTS_SCHEMA: Final[str] = """
|
|
106
|
+
CREATE VIRTUAL TABLE passages_fts USING fts5(
|
|
107
|
+
content,
|
|
108
|
+
content='passages',
|
|
109
|
+
content_rowid='rowid',
|
|
110
|
+
tokenize='porter unicode61 remove_diacritics 2'
|
|
111
|
+
);
|
|
112
|
+
"""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class PackError(RuntimeError):
|
|
116
|
+
"""A pack could not be written or read, or NumPy is missing."""
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@dataclass(frozen=True)
|
|
120
|
+
class Passage:
|
|
121
|
+
"""One searchable row of a pack.
|
|
122
|
+
|
|
123
|
+
:param id: Unique within the pack; ``<kg_name>:<node_id>`` by convention.
|
|
124
|
+
:param kg_name: The KG the passage came from.
|
|
125
|
+
:param kg_kind: Its KG kind, as a hit reports it.
|
|
126
|
+
:param kind: The node's kind; the app reads ``section`` and ``chunk`` for
|
|
127
|
+
a book's chapters, and nothing else by kind.
|
|
128
|
+
:param content: The text searched and shown.
|
|
129
|
+
:param name: Short label.
|
|
130
|
+
:param title: The work's title, for hit cards.
|
|
131
|
+
:param node_title: This node's own title.
|
|
132
|
+
:param author: Author, or any person the hit card names.
|
|
133
|
+
:param genre: The scope the app filters on.
|
|
134
|
+
:param book: Book or collection slug.
|
|
135
|
+
:param file_path: Source document path.
|
|
136
|
+
:param char_start: Offset in the source document.
|
|
137
|
+
:param chapter: Chapter number, for chapter-less books.
|
|
138
|
+
:param timestamp: ISO date, for dated passages.
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
id: str
|
|
142
|
+
kg_name: str
|
|
143
|
+
kg_kind: str
|
|
144
|
+
kind: str
|
|
145
|
+
content: str
|
|
146
|
+
name: str | None = None
|
|
147
|
+
title: str | None = None
|
|
148
|
+
node_title: str | None = None
|
|
149
|
+
author: str | None = None
|
|
150
|
+
genre: str | None = None
|
|
151
|
+
book: str | None = None
|
|
152
|
+
file_path: str | None = None
|
|
153
|
+
char_start: int | None = None
|
|
154
|
+
chapter: int | None = None
|
|
155
|
+
timestamp: str | None = None
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
_PASSAGE_COLUMNS: Final[tuple[str, ...]] = (
|
|
159
|
+
"id",
|
|
160
|
+
"kg_name",
|
|
161
|
+
"kg_kind",
|
|
162
|
+
"kind",
|
|
163
|
+
"name",
|
|
164
|
+
"title",
|
|
165
|
+
"node_title",
|
|
166
|
+
"author",
|
|
167
|
+
"genre",
|
|
168
|
+
"book",
|
|
169
|
+
"file_path",
|
|
170
|
+
"char_start",
|
|
171
|
+
"chapter",
|
|
172
|
+
"timestamp",
|
|
173
|
+
"content",
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass
|
|
178
|
+
class PackStats:
|
|
179
|
+
"""What a finished pack contains.
|
|
180
|
+
|
|
181
|
+
:param path: The pack file.
|
|
182
|
+
:param passages: Rows written.
|
|
183
|
+
:param vectors: Vectors written to the sidecar.
|
|
184
|
+
:param missing_vectors: Passages with no vector.
|
|
185
|
+
:param bytes: Pack size.
|
|
186
|
+
:param sha256: Pack checksum.
|
|
187
|
+
:param sidecar: The ``.vectors`` sidecar, if any.
|
|
188
|
+
:param sidecar_bytes: Its size.
|
|
189
|
+
:param sidecar_sha256: Its checksum.
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
path: Path
|
|
193
|
+
passages: int = 0
|
|
194
|
+
vectors: int = 0
|
|
195
|
+
missing_vectors: int = 0
|
|
196
|
+
bytes: int = 0
|
|
197
|
+
sha256: str = ""
|
|
198
|
+
sidecar: Path | None = None
|
|
199
|
+
sidecar_bytes: int = 0
|
|
200
|
+
sidecar_sha256: str = ""
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def name(self) -> str:
|
|
204
|
+
"""The pack's file name."""
|
|
205
|
+
return self.path.name
|
|
206
|
+
|
|
207
|
+
@property
|
|
208
|
+
def total_bytes(self) -> int:
|
|
209
|
+
"""Pack plus sidecar: what has to reach the device."""
|
|
210
|
+
return self.bytes + self.sidecar_bytes
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _numpy() -> Any:
|
|
214
|
+
try:
|
|
215
|
+
import numpy # noqa: PLC0415 - optional, only for vectors
|
|
216
|
+
except ImportError as exc:
|
|
217
|
+
raise PackError("vectors need numpy: install kgmodule-utils[semantic]") from exc
|
|
218
|
+
return numpy
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def sha256_file(path: Path) -> str:
|
|
222
|
+
"""Streamed SHA-256 of a file.
|
|
223
|
+
|
|
224
|
+
:param path: The file.
|
|
225
|
+
:return: Hex digest.
|
|
226
|
+
"""
|
|
227
|
+
digest = hashlib.sha256()
|
|
228
|
+
with open(path, "rb") as handle:
|
|
229
|
+
for block in iter(lambda: handle.read(1 << 20), b""):
|
|
230
|
+
digest.update(block)
|
|
231
|
+
return digest.hexdigest()
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
# ---------------------------------------------------------------------------
|
|
235
|
+
# Vectors
|
|
236
|
+
# ---------------------------------------------------------------------------
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def encode_vector(vector: Any, dtype: str) -> bytes:
|
|
240
|
+
"""Encode one vector for the sidecar.
|
|
241
|
+
|
|
242
|
+
int8 vectors are L2-normalised first. Cosine is scale-invariant, so no
|
|
243
|
+
ranking changes, and the ``* 127`` scale then lands inside int8 range.
|
|
244
|
+
|
|
245
|
+
:param vector: A float vector.
|
|
246
|
+
:param dtype: ``"int8"`` or ``"float"``.
|
|
247
|
+
:return: The encoded bytes.
|
|
248
|
+
"""
|
|
249
|
+
numpy = _numpy()
|
|
250
|
+
array = numpy.asarray(vector, dtype=numpy.float32)
|
|
251
|
+
if dtype == "float":
|
|
252
|
+
return array.tobytes()
|
|
253
|
+
if dtype != "int8":
|
|
254
|
+
raise PackError(f"dtype must be one of {DTYPES}, got {dtype!r}")
|
|
255
|
+
norm = float(numpy.linalg.norm(array))
|
|
256
|
+
if norm > 0:
|
|
257
|
+
array = array / norm
|
|
258
|
+
return numpy.clip(numpy.round(array * 127.0), -128, 127).astype(numpy.int8).tobytes()
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def vector_header(dtype: str, dim: int, count: int) -> bytes:
|
|
262
|
+
"""Build the sidecar's 32-byte header.
|
|
263
|
+
|
|
264
|
+
:param dtype: ``"int8"`` or ``"float"``.
|
|
265
|
+
:param dim: Embedding dimension.
|
|
266
|
+
:param count: Vectors in the file.
|
|
267
|
+
:return: Exactly :data:`VECTOR_HEADER_BYTES` bytes.
|
|
268
|
+
"""
|
|
269
|
+
header = bytearray(VECTOR_HEADER_BYTES)
|
|
270
|
+
header[0:8] = VECTOR_MAGIC
|
|
271
|
+
header[8] = _DTYPE_CODES[dtype]
|
|
272
|
+
header[12:16] = int(dim).to_bytes(4, "little")
|
|
273
|
+
header[16:24] = int(count).to_bytes(8, "little")
|
|
274
|
+
return bytes(header)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def read_vector_sidecar(sidecar: Path) -> tuple[Any, str]:
|
|
278
|
+
"""Memory-map a sidecar.
|
|
279
|
+
|
|
280
|
+
:param sidecar: A ``.vectors`` file.
|
|
281
|
+
:return: ``(matrix, dtype)``: a ``(count, dim)`` array viewing the file,
|
|
282
|
+
and its dtype name.
|
|
283
|
+
:raises PackError: If the magic, dtype or length disagree, as a truncated
|
|
284
|
+
download would.
|
|
285
|
+
"""
|
|
286
|
+
numpy = _numpy()
|
|
287
|
+
with open(sidecar, "rb") as handle:
|
|
288
|
+
header = handle.read(VECTOR_HEADER_BYTES)
|
|
289
|
+
if len(header) < VECTOR_HEADER_BYTES or header[0:8] != VECTOR_MAGIC:
|
|
290
|
+
raise PackError(f"{sidecar} is not a vector sidecar")
|
|
291
|
+
dtype = {code: name for name, code in _DTYPE_CODES.items()}.get(header[8])
|
|
292
|
+
if dtype is None:
|
|
293
|
+
raise PackError(f"{sidecar} has an unknown vector dtype ({header[8]})")
|
|
294
|
+
dim = int.from_bytes(header[12:16], "little")
|
|
295
|
+
count = int.from_bytes(header[16:24], "little")
|
|
296
|
+
expected = VECTOR_HEADER_BYTES + count * dim * _DTYPE_ITEMSIZE[dtype]
|
|
297
|
+
actual = sidecar.stat().st_size
|
|
298
|
+
if actual != expected:
|
|
299
|
+
raise PackError(
|
|
300
|
+
f"{sidecar} is {actual} bytes, expected {expected} for {count} x {dim} {dtype}"
|
|
301
|
+
)
|
|
302
|
+
matrix = numpy.memmap(
|
|
303
|
+
sidecar,
|
|
304
|
+
dtype=numpy.int8 if dtype == "int8" else numpy.float32,
|
|
305
|
+
mode="r",
|
|
306
|
+
offset=VECTOR_HEADER_BYTES,
|
|
307
|
+
shape=(count, dim),
|
|
308
|
+
)
|
|
309
|
+
return matrix, dtype
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
# ---------------------------------------------------------------------------
|
|
313
|
+
# Writing
|
|
314
|
+
# ---------------------------------------------------------------------------
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def write_passage_pack(
|
|
318
|
+
dest: Path,
|
|
319
|
+
passages: Iterable[Passage],
|
|
320
|
+
vectors: Mapping[str, Any] | None = None,
|
|
321
|
+
*,
|
|
322
|
+
dim: int,
|
|
323
|
+
embed_model: str,
|
|
324
|
+
dtype: str = "int8",
|
|
325
|
+
meta: Mapping[str, object] | None = None,
|
|
326
|
+
) -> PackStats:
|
|
327
|
+
"""Write a passage pack and, given vectors, its sidecar.
|
|
328
|
+
|
|
329
|
+
A passage with no vector stays in the pack, findable lexically, and is
|
|
330
|
+
counted in ``missing_vectors``. Ids must be unique: a collision fails the
|
|
331
|
+
write rather than shrinking the pack.
|
|
332
|
+
|
|
333
|
+
:param dest: The ``.pack`` path; replaced if it exists. The sidecar is
|
|
334
|
+
written beside it as ``<stem>.vectors``.
|
|
335
|
+
:param passages: The rows, in the order to store them.
|
|
336
|
+
:param vectors: Passage id -> vector, or ``None`` for a lexical-only pack.
|
|
337
|
+
:param dim: Embedding dimension every vector must have.
|
|
338
|
+
:param embed_model: The model the vectors came from, stamped into the pack.
|
|
339
|
+
:param dtype: ``"int8"`` (default) or ``"float"``.
|
|
340
|
+
:param meta: Extra ``pack_meta`` rows.
|
|
341
|
+
:return: Stats for the finished pack.
|
|
342
|
+
:raises PackError: On a bad dtype or a vector of the wrong dimension.
|
|
343
|
+
"""
|
|
344
|
+
if dtype not in DTYPES:
|
|
345
|
+
raise PackError(f"dtype must be one of {DTYPES}, got {dtype!r}")
|
|
346
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
347
|
+
dest.unlink(missing_ok=True)
|
|
348
|
+
sidecar = dest.with_suffix(".vectors")
|
|
349
|
+
sidecar.unlink(missing_ok=True)
|
|
350
|
+
|
|
351
|
+
stats = PackStats(path=dest)
|
|
352
|
+
con = sqlite3.connect(str(dest))
|
|
353
|
+
try:
|
|
354
|
+
con.executescript("PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF;")
|
|
355
|
+
con.executescript(PASSAGE_SCHEMA)
|
|
356
|
+
insert = (
|
|
357
|
+
f"INSERT INTO passages({', '.join(_PASSAGE_COLUMNS)}) "
|
|
358
|
+
f"VALUES ({', '.join('?' for _ in _PASSAGE_COLUMNS)})"
|
|
359
|
+
)
|
|
360
|
+
con.executemany(insert, ([getattr(p, c) for c in _PASSAGE_COLUMNS] for p in passages))
|
|
361
|
+
stats.passages = con.execute("SELECT COUNT(*) FROM passages").fetchone()[0]
|
|
362
|
+
con.executescript(FTS_SCHEMA)
|
|
363
|
+
con.execute("INSERT INTO passages_fts(passages_fts) VALUES('rebuild')")
|
|
364
|
+
|
|
365
|
+
if vectors:
|
|
366
|
+
itemsize = _DTYPE_ITEMSIZE[dtype]
|
|
367
|
+
assignments: list[tuple[int, str]] = []
|
|
368
|
+
with open(sidecar, "wb") as handle:
|
|
369
|
+
handle.write(b"\0" * VECTOR_HEADER_BYTES)
|
|
370
|
+
for (passage_id,) in con.execute("SELECT id FROM passages ORDER BY rowid"):
|
|
371
|
+
vector = vectors.get(passage_id)
|
|
372
|
+
if vector is None:
|
|
373
|
+
continue
|
|
374
|
+
encoded = encode_vector(vector, dtype)
|
|
375
|
+
if len(encoded) != dim * itemsize:
|
|
376
|
+
raise PackError(
|
|
377
|
+
f"{passage_id} has {len(encoded) // itemsize} dimensions, "
|
|
378
|
+
f"expected {dim}"
|
|
379
|
+
)
|
|
380
|
+
handle.write(encoded)
|
|
381
|
+
assignments.append((len(assignments), passage_id))
|
|
382
|
+
handle.seek(0)
|
|
383
|
+
handle.write(vector_header(dtype, dim, len(assignments)))
|
|
384
|
+
con.executemany("UPDATE passages SET vector_index = ? WHERE id = ?", assignments)
|
|
385
|
+
stats.vectors = len(assignments)
|
|
386
|
+
stats.missing_vectors = stats.passages - stats.vectors
|
|
387
|
+
|
|
388
|
+
stamps: dict[str, object] = {
|
|
389
|
+
"pack": dest.stem,
|
|
390
|
+
"pack_version": PACK_VERSION,
|
|
391
|
+
"generated": datetime.now(UTC).isoformat(timespec="seconds"),
|
|
392
|
+
"embed_model": embed_model,
|
|
393
|
+
"embed_dim": dim,
|
|
394
|
+
"vector_dtype": dtype if stats.vectors else "none",
|
|
395
|
+
"rrf_k": RRF_K,
|
|
396
|
+
**(meta or {}),
|
|
397
|
+
}
|
|
398
|
+
con.executemany(
|
|
399
|
+
"INSERT OR REPLACE INTO pack_meta(key, value) VALUES (?, ?)",
|
|
400
|
+
[(key, str(value)) for key, value in stamps.items()],
|
|
401
|
+
)
|
|
402
|
+
con.commit()
|
|
403
|
+
con.execute("VACUUM")
|
|
404
|
+
except sqlite3.IntegrityError as exc:
|
|
405
|
+
raise PackError(f"duplicate passage id in {dest.name}: {exc}") from exc
|
|
406
|
+
finally:
|
|
407
|
+
con.close()
|
|
408
|
+
|
|
409
|
+
if stats.vectors == 0:
|
|
410
|
+
sidecar.unlink(missing_ok=True)
|
|
411
|
+
else:
|
|
412
|
+
stats.sidecar = sidecar
|
|
413
|
+
stats.sidecar_bytes = sidecar.stat().st_size
|
|
414
|
+
stats.sidecar_sha256 = sha256_file(sidecar)
|
|
415
|
+
stats.bytes = dest.stat().st_size
|
|
416
|
+
stats.sha256 = sha256_file(dest)
|
|
417
|
+
return stats
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def build_manifest(
|
|
421
|
+
packs: Sequence[PackStats],
|
|
422
|
+
*,
|
|
423
|
+
embed_model: str,
|
|
424
|
+
dim: int,
|
|
425
|
+
dtype: str,
|
|
426
|
+
golden: str | None = "golden.json",
|
|
427
|
+
extra: Mapping[str, Any] | None = None,
|
|
428
|
+
) -> dict[str, Any]:
|
|
429
|
+
"""Describe the packs for the app to check itself against.
|
|
430
|
+
|
|
431
|
+
The embedder block matters most: vectors are meaningless to a query
|
|
432
|
+
embedded by another model, so the app refuses the corpus until they agree.
|
|
433
|
+
|
|
434
|
+
:param packs: The finished packs.
|
|
435
|
+
:param embed_model: The embedder's model name.
|
|
436
|
+
:param dim: Its dimension.
|
|
437
|
+
:param dtype: The vectors' dtype.
|
|
438
|
+
:param golden: The golden file's name, or ``None``.
|
|
439
|
+
:param extra: Further top-level keys, such as privacy settings.
|
|
440
|
+
:return: The manifest document.
|
|
441
|
+
"""
|
|
442
|
+
return {
|
|
443
|
+
"pack_version": PACK_VERSION,
|
|
444
|
+
"generated": datetime.now(UTC).isoformat(timespec="seconds"),
|
|
445
|
+
"embedder": {
|
|
446
|
+
"model": embed_model,
|
|
447
|
+
"dim": dim,
|
|
448
|
+
"normalized": True,
|
|
449
|
+
"note": "Query vectors must come from this model; the packs are unusable without it.",
|
|
450
|
+
},
|
|
451
|
+
"vector_dtype": dtype if any(p.vectors for p in packs) else "none",
|
|
452
|
+
"rrf_k": RRF_K,
|
|
453
|
+
"vector_sidecar": {
|
|
454
|
+
"magic": VECTOR_MAGIC.decode("ascii"),
|
|
455
|
+
"header_bytes": VECTOR_HEADER_BYTES,
|
|
456
|
+
"layout": "row-major, one vector per row, indexed by passages.vector_index",
|
|
457
|
+
},
|
|
458
|
+
"packs": [
|
|
459
|
+
{
|
|
460
|
+
"name": p.name,
|
|
461
|
+
"bytes": p.bytes,
|
|
462
|
+
"sha256": p.sha256,
|
|
463
|
+
"passages": p.passages,
|
|
464
|
+
"vectors": p.vectors,
|
|
465
|
+
"passages_without_vectors": p.missing_vectors,
|
|
466
|
+
"sidecar": (
|
|
467
|
+
{"name": p.sidecar.name, "bytes": p.sidecar_bytes, "sha256": p.sidecar_sha256}
|
|
468
|
+
if p.sidecar is not None
|
|
469
|
+
else None
|
|
470
|
+
),
|
|
471
|
+
}
|
|
472
|
+
for p in packs
|
|
473
|
+
],
|
|
474
|
+
"total_bytes": sum(p.total_bytes for p in packs),
|
|
475
|
+
"golden": golden,
|
|
476
|
+
**(extra or {}),
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
# ---------------------------------------------------------------------------
|
|
481
|
+
# Search: the reference the app must reproduce
|
|
482
|
+
# ---------------------------------------------------------------------------
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def fts_terms(query: str) -> list[str]:
|
|
486
|
+
"""Split a query into bare alphanumeric terms, so none is read as FTS5 syntax.
|
|
487
|
+
|
|
488
|
+
:param query: The query text.
|
|
489
|
+
:return: The terms, in order.
|
|
490
|
+
"""
|
|
491
|
+
return [t for t in "".join(c if c.isalnum() else " " for c in query).split() if t]
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def fts_phrase_expression(query: str) -> str:
|
|
495
|
+
"""The exact-phrase FTS5 expression, tried before the any-term fallback.
|
|
496
|
+
|
|
497
|
+
:param query: The query text.
|
|
498
|
+
:return: A quoted phrase, or ``""`` when nothing is searchable.
|
|
499
|
+
"""
|
|
500
|
+
terms = fts_terms(query)
|
|
501
|
+
return '"' + " ".join(terms) + '"' if terms else ""
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def fts_match_expression(query: str) -> str:
|
|
505
|
+
"""The any-term FTS5 expression: every term quoted, OR-ed.
|
|
506
|
+
|
|
507
|
+
:param query: The query text.
|
|
508
|
+
:return: An FTS5 MATCH expression, or ``""``.
|
|
509
|
+
"""
|
|
510
|
+
return " OR ".join(f'"{term}"' for term in fts_terms(query))
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def rrf_fuse(
|
|
514
|
+
dense_ids: Sequence[str], lexical_ids: Sequence[str], k: int, *, rrf_k: int = RRF_K
|
|
515
|
+
) -> list[str]:
|
|
516
|
+
"""Blend two rank lists by reciprocal rank fusion.
|
|
517
|
+
|
|
518
|
+
Lexical ranks are added first, so a cross-channel tie goes to the exact
|
|
519
|
+
match.
|
|
520
|
+
|
|
521
|
+
:param dense_ids: Ids best-first by cosine.
|
|
522
|
+
:param lexical_ids: Ids best-first by BM25.
|
|
523
|
+
:param k: How many fused ids to return.
|
|
524
|
+
:param rrf_k: The fusion constant.
|
|
525
|
+
:return: Fused ids, best-first.
|
|
526
|
+
"""
|
|
527
|
+
scores: dict[str, float] = {}
|
|
528
|
+
for rank, node_id in enumerate(lexical_ids):
|
|
529
|
+
scores[node_id] = scores.get(node_id, 0.0) + 1.0 / (rrf_k + rank)
|
|
530
|
+
for rank, node_id in enumerate(dense_ids):
|
|
531
|
+
scores[node_id] = scores.get(node_id, 0.0) + 1.0 / (rrf_k + rank)
|
|
532
|
+
return sorted(scores, key=lambda i: -scores[i])[:k]
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def search_pack(
|
|
536
|
+
pack: Path,
|
|
537
|
+
query_vector: Any,
|
|
538
|
+
query_text: str,
|
|
539
|
+
*,
|
|
540
|
+
k: int = 10,
|
|
541
|
+
genre: str | None = None,
|
|
542
|
+
lexical: bool = True,
|
|
543
|
+
) -> tuple[list[dict[str, Any]], set[str]]:
|
|
544
|
+
"""Run a pack's hybrid search: dense + BM25, fused by RRF.
|
|
545
|
+
|
|
546
|
+
Both channels oversample to ``k * 3``. A hit found only lexically gets its
|
|
547
|
+
cosine from the sidecar, so every fused hit carries an honest score.
|
|
548
|
+
|
|
549
|
+
:param pack: A passage pack, its sidecar beside it.
|
|
550
|
+
:param query_vector: The query embedding.
|
|
551
|
+
:param query_text: The query, for the lexical channel.
|
|
552
|
+
:param k: Hits to return.
|
|
553
|
+
:param genre: Restrict to one genre, as the app's scope picker does.
|
|
554
|
+
:param lexical: Include the BM25 channel.
|
|
555
|
+
:return: Hits best-first (``node_id``, ``score``, ``kind``, ``genre``,
|
|
556
|
+
``title``, ``content`` and the rest of the row), and the ids the
|
|
557
|
+
lexical channel rescued from outside the dense top ``k``.
|
|
558
|
+
"""
|
|
559
|
+
numpy = _numpy()
|
|
560
|
+
con = sqlite3.connect(f"file:{pack}?mode=ro", uri=True)
|
|
561
|
+
con.row_factory = sqlite3.Row
|
|
562
|
+
try:
|
|
563
|
+
scope = " AND genre = ?" if genre else ""
|
|
564
|
+
params: list[object] = [genre] if genre else []
|
|
565
|
+
|
|
566
|
+
dense_ids: list[str] = []
|
|
567
|
+
all_scores: dict[str, float] = {}
|
|
568
|
+
sidecar = pack.with_suffix(".vectors")
|
|
569
|
+
if sidecar.exists():
|
|
570
|
+
matrix, _dtype = read_vector_sidecar(sidecar)
|
|
571
|
+
rows = con.execute(
|
|
572
|
+
f"SELECT id, vector_index FROM passages WHERE vector_index IS NOT NULL{scope}",
|
|
573
|
+
params,
|
|
574
|
+
).fetchall()
|
|
575
|
+
if rows:
|
|
576
|
+
ids = [row["id"] for row in rows]
|
|
577
|
+
indices = numpy.fromiter(
|
|
578
|
+
(row["vector_index"] for row in rows), dtype=numpy.int64, count=len(rows)
|
|
579
|
+
)
|
|
580
|
+
candidates = numpy.asarray(matrix[indices], dtype=numpy.float32)
|
|
581
|
+
norms = numpy.linalg.norm(candidates, axis=1).clip(min=1e-12)
|
|
582
|
+
query = numpy.asarray(query_vector, dtype=numpy.float32)
|
|
583
|
+
query = query / max(float(numpy.linalg.norm(query)), 1e-12)
|
|
584
|
+
similarity = (candidates @ query) / norms
|
|
585
|
+
dense_ids = [ids[i] for i in numpy.argsort(-similarity)[: k * 3]]
|
|
586
|
+
all_scores = {ids[i]: float(similarity[i]) for i in range(len(ids))}
|
|
587
|
+
|
|
588
|
+
lexical_ids: list[str] = []
|
|
589
|
+
if lexical:
|
|
590
|
+
sql = (
|
|
591
|
+
"SELECT p.id AS id FROM passages_fts f JOIN passages p ON p.rowid = f.rowid "
|
|
592
|
+
f"WHERE passages_fts MATCH ?{scope.replace('genre', 'p.genre')} "
|
|
593
|
+
"ORDER BY bm25(passages_fts) LIMIT ?"
|
|
594
|
+
)
|
|
595
|
+
|
|
596
|
+
def run(expression: str) -> list[str]:
|
|
597
|
+
if not expression:
|
|
598
|
+
return []
|
|
599
|
+
try:
|
|
600
|
+
return [r["id"] for r in con.execute(sql, [expression, *params, k * 3])]
|
|
601
|
+
except sqlite3.Error:
|
|
602
|
+
return []
|
|
603
|
+
|
|
604
|
+
lexical_ids = run(fts_phrase_expression(query_text)) or run(
|
|
605
|
+
fts_match_expression(query_text)
|
|
606
|
+
)
|
|
607
|
+
lexical_ids = [i for i in lexical_ids if i in all_scores]
|
|
608
|
+
|
|
609
|
+
ordered = rrf_fuse(dense_ids, lexical_ids, k) if lexical_ids else dense_ids[:k]
|
|
610
|
+
rescued = set(lexical_ids) - set(dense_ids[:k])
|
|
611
|
+
if not ordered:
|
|
612
|
+
return [], set()
|
|
613
|
+
|
|
614
|
+
marks = ", ".join("?" for _ in ordered)
|
|
615
|
+
by_id = {
|
|
616
|
+
row["id"]: dict(row)
|
|
617
|
+
for row in con.execute(f"SELECT * FROM passages WHERE id IN ({marks})", ordered)
|
|
618
|
+
}
|
|
619
|
+
hits = []
|
|
620
|
+
for node_id in ordered:
|
|
621
|
+
row = by_id.get(node_id)
|
|
622
|
+
if row is None:
|
|
623
|
+
continue
|
|
624
|
+
hits.append(
|
|
625
|
+
{
|
|
626
|
+
**row,
|
|
627
|
+
"node_id": node_id,
|
|
628
|
+
"score": round(all_scores.get(node_id, 0.0), 4),
|
|
629
|
+
}
|
|
630
|
+
)
|
|
631
|
+
return hits, rescued & {h["node_id"] for h in hits}
|
|
632
|
+
finally:
|
|
633
|
+
con.close()
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def build_golden(
|
|
637
|
+
packs: Mapping[str, Path],
|
|
638
|
+
queries: Sequence[str],
|
|
639
|
+
query_vectors: Sequence[Any],
|
|
640
|
+
*,
|
|
641
|
+
k: int,
|
|
642
|
+
embed_model: str,
|
|
643
|
+
dim: int,
|
|
644
|
+
dtype: str,
|
|
645
|
+
tolerance: Mapping[str, float | int] | None = None,
|
|
646
|
+
rescue_tolerance: float | None = None,
|
|
647
|
+
) -> dict[str, Any]:
|
|
648
|
+
"""Record each query's top-k as the packs answer it: the app's parity gate.
|
|
649
|
+
|
|
650
|
+
The caller embeds the queries with the same model the packs were built
|
|
651
|
+
with, so this module never loads one.
|
|
652
|
+
|
|
653
|
+
:param packs: Pack name -> path; each is searched on its own.
|
|
654
|
+
:param queries: The golden queries.
|
|
655
|
+
:param query_vectors: Their embeddings, in the same order.
|
|
656
|
+
:param k: Hits to record per query.
|
|
657
|
+
:param embed_model: The embedder's model name.
|
|
658
|
+
:param dim: Its dimension.
|
|
659
|
+
:param dtype: The packs' vector dtype.
|
|
660
|
+
:param tolerance: Parity tolerances; :data:`DEFAULT_TOLERANCE` if omitted.
|
|
661
|
+
:param rescue_tolerance: Recorded when the app checks its own constant
|
|
662
|
+
against it.
|
|
663
|
+
:return: The golden document. Each query entry has ``packs``; a caller
|
|
664
|
+
that merges packs adds its own ``all`` ranking.
|
|
665
|
+
"""
|
|
666
|
+
if len(queries) != len(query_vectors):
|
|
667
|
+
raise PackError("queries and query_vectors differ in length")
|
|
668
|
+
entries = []
|
|
669
|
+
for query, vector in zip(queries, query_vectors, strict=True):
|
|
670
|
+
ranked: dict[str, list[dict[str, Any]]] = {}
|
|
671
|
+
for name, path in packs.items():
|
|
672
|
+
hits, _rescued = search_pack(path, vector, query, k=k)
|
|
673
|
+
ranked[name] = [
|
|
674
|
+
{"rank": rank, "node_id": hit["node_id"], "score": hit["score"]}
|
|
675
|
+
for rank, hit in enumerate(hits)
|
|
676
|
+
]
|
|
677
|
+
entries.append({"query": query, "packs": ranked})
|
|
678
|
+
document: dict[str, Any] = {
|
|
679
|
+
"generated": datetime.now(UTC).isoformat(timespec="seconds"),
|
|
680
|
+
"embed_model": embed_model,
|
|
681
|
+
"embed_dim": dim,
|
|
682
|
+
"vector_dtype": dtype,
|
|
683
|
+
"rrf_k": RRF_K,
|
|
684
|
+
"k": k,
|
|
685
|
+
"tolerance": dict(tolerance or DEFAULT_TOLERANCE),
|
|
686
|
+
"queries": entries,
|
|
687
|
+
}
|
|
688
|
+
if rescue_tolerance is not None:
|
|
689
|
+
document["rescue_tolerance"] = rescue_tolerance
|
|
690
|
+
return document
|
|
691
|
+
|
|
692
|
+
|
|
693
|
+
__all__ = [
|
|
694
|
+
"DEFAULT_TOLERANCE",
|
|
695
|
+
"DTYPES",
|
|
696
|
+
"FTS_SCHEMA",
|
|
697
|
+
"PACK_VERSION",
|
|
698
|
+
"PASSAGE_SCHEMA",
|
|
699
|
+
"RRF_K",
|
|
700
|
+
"VECTOR_HEADER_BYTES",
|
|
701
|
+
"VECTOR_MAGIC",
|
|
702
|
+
"PackError",
|
|
703
|
+
"PackStats",
|
|
704
|
+
"Passage",
|
|
705
|
+
"build_golden",
|
|
706
|
+
"build_manifest",
|
|
707
|
+
"encode_vector",
|
|
708
|
+
"fts_match_expression",
|
|
709
|
+
"fts_phrase_expression",
|
|
710
|
+
"fts_terms",
|
|
711
|
+
"read_vector_sidecar",
|
|
712
|
+
"rrf_fuse",
|
|
713
|
+
"search_pack",
|
|
714
|
+
"sha256_file",
|
|
715
|
+
"vector_header",
|
|
716
|
+
"write_passage_pack",
|
|
717
|
+
]
|
|
@@ -47,7 +47,7 @@ from __future__ import annotations
|
|
|
47
47
|
|
|
48
48
|
import logging
|
|
49
49
|
import math
|
|
50
|
-
from collections.abc import Iterable, Sequence
|
|
50
|
+
from collections.abc import Iterable, Iterator, Sequence
|
|
51
51
|
from pathlib import Path
|
|
52
52
|
from typing import Any, Protocol, runtime_checkable
|
|
53
53
|
|
|
@@ -653,6 +653,30 @@ class SqliteVecBackend:
|
|
|
653
653
|
except Exception: # noqa: BLE001
|
|
654
654
|
return set()
|
|
655
655
|
|
|
656
|
+
def iter_vectors(self) -> Iterator[tuple[str, Any]]:
|
|
657
|
+
"""Yield ``(id, vector)`` for every stored vector, as float32 arrays.
|
|
658
|
+
|
|
659
|
+
An int8 store yields its values divided by 127, the inverse of
|
|
660
|
+
:meth:`_encode`; cosine is scale-invariant, so rankings are the same
|
|
661
|
+
either way. For exporters that copy a store's vectors elsewhere, such
|
|
662
|
+
as :mod:`kg_utils.devicepack`.
|
|
663
|
+
|
|
664
|
+
:return: An iterator in ``vec_meta`` rowid order; empty when the
|
|
665
|
+
store does not exist.
|
|
666
|
+
"""
|
|
667
|
+
import numpy as np # pylint: disable=import-outside-toplevel
|
|
668
|
+
|
|
669
|
+
if self._conn is None and not self.db_path.exists():
|
|
670
|
+
return
|
|
671
|
+
dtype = np.int8 if self.dtype == "int8" else np.float32
|
|
672
|
+
rows = self._c().execute(
|
|
673
|
+
"SELECT m.id, v.embedding FROM vec_meta m "
|
|
674
|
+
"JOIN vec_nodes v ON v.rowid = m.rowid ORDER BY m.rowid"
|
|
675
|
+
)
|
|
676
|
+
for node_id, blob in rows:
|
|
677
|
+
vector = np.frombuffer(blob, dtype=dtype).astype(np.float32)
|
|
678
|
+
yield node_id, vector / 127.0 if self.dtype == "int8" else vector
|
|
679
|
+
|
|
656
680
|
def count(self) -> int:
|
|
657
681
|
# Don't create a fresh db file just to count a store that isn't there.
|
|
658
682
|
if self._conn is None and not self.db_path.exists():
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|