kgmodule-utils 0.27.0__tar.gz → 0.28.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/PKG-INFO +32 -7
  2. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/README.md +29 -4
  3. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/pyproject.toml +11 -8
  4. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/__init__.py +1 -1
  5. kgmodule_utils-0.28.0/src/kg_utils/devicepack.py +717 -0
  6. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/vector_backend.py +25 -1
  7. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/LICENSE +0 -0
  8. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/analysis/__init__.py +0 -0
  9. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/analysis/scores.py +0 -0
  10. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/corpus_embedder.py +0 -0
  11. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/embed.py +0 -0
  12. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/embedder.py +0 -0
  13. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/extractor.py +0 -0
  14. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/__init__.py +0 -0
  15. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/converters.py +0 -0
  16. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/derived.py +0 -0
  17. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/manifest.py +0 -0
  18. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/ingest/pipeline.py +0 -0
  19. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/module.py +0 -0
  20. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/pipeline.py +0 -0
  21. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/py.typed +0 -0
  22. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/retrieval/__init__.py +0 -0
  23. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/retrieval/hits.py +0 -0
  24. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/semantic.py +0 -0
  25. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/snapshots/__init__.py +0 -0
  26. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/snapshots/manager.py +0 -0
  27. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/snapshots/models.py +0 -0
  28. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/specs.py +0 -0
  29. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/store.py +0 -0
  30. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/__init__.py +0 -0
  31. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/_config.py +0 -0
  32. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/_image.py +0 -0
  33. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/_text.py +0 -0
  34. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/synthesis/factory.py +0 -0
  35. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/temporal.py +0 -0
  36. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/validation.py +0 -0
  37. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz/__init__.py +0 -0
  38. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz/graph_html.py +0 -0
  39. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz/theme.py +0 -0
  40. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz/tooltip.py +0 -0
  41. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/__init__.py +0 -0
  42. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/layout.py +0 -0
  43. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/organic.py +0 -0
  44. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/qt.py +0 -0
  45. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/viz3d/species.py +0 -0
  46. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/worker/__init__.py +0 -0
  47. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/worker/client.py +0 -0
  48. {kgmodule_utils-0.27.0 → kgmodule_utils-0.28.0}/src/kg_utils/worker/ops.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kgmodule-utils
3
- Version: 0.27.0
3
+ Version: 0.28.0
4
4
  Summary: Shared types, graph store, semantic index, and pipeline base for the KGModule SDK
5
5
  License-Expression: Elastic-2.0
6
6
  License-File: LICENSE
@@ -38,8 +38,8 @@ Requires-Dist: openai (>=1.30.0) ; extra == "synthesis"
38
38
  Requires-Dist: openai (>=1.30.0) ; extra == "synthesis-mflux"
39
39
  Requires-Dist: pillow (>=10.0.0) ; extra == "synthesis"
40
40
  Requires-Dist: pillow (>=10.0.0) ; extra == "synthesis-mflux"
41
- Requires-Dist: pymupdf4llm (>=1.28.2) ; extra == "ingest"
42
- Requires-Dist: pymupdf4llm (>=1.28.2) ; extra == "pdf"
41
+ Requires-Dist: pymupdf (>=2.0) ; extra == "ingest"
42
+ Requires-Dist: pymupdf (>=2.0) ; extra == "pdf"
43
43
  Requires-Dist: pyvis (>=0.3.2) ; extra == "viz"
44
44
  Requires-Dist: pyvista (>=0.44.0) ; extra == "viz3d-qt"
45
45
  Requires-Dist: pyvista (>=0.44.0) ; extra == "viz3d-render"
@@ -56,7 +56,7 @@ Description-Content-Type: text/markdown
56
56
 
57
57
  [![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13-blue.svg)](https://www.python.org/)
58
58
  [![License: Elastic-2.0](https://img.shields.io/badge/License-Elastic%202.0-blue.svg)](https://www.elastic.co/licensing/elastic-license)
59
- [![Version](https://img.shields.io/badge/version-0.27.0-blue.svg)](https://github.com/Flux-Frontiers/KG_utils/releases)
59
+ [![Version](https://img.shields.io/badge/version-0.28.0-blue.svg)](https://github.com/Flux-Frontiers/KG_utils/releases)
60
60
  [![CI](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml/badge.svg)](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml)
61
61
  [![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
62
62
  [![DOI](https://img.shields.io/badge/DOI-10.5281%2Fzenodo.21284866-blue.svg)](https://doi.org/10.5281/zenodo.21284866)
@@ -81,6 +81,15 @@ Fleet modules use it in two ways. [PyCodeKG](https://github.com/Flux-Frontiers/p
81
81
 
82
82
  ## Latest News
83
83
 
84
+ - **0.28.0 (2026-10-23)** -- New `kg_utils.devicepack`: the on-device
85
+ search-pack format the Knowledge Press apps read (passage pack, vector
86
+ sidecar, manifest, golden rankings), lifted out of gutenberg_kg's
87
+ `export-swift` so any KG can write it. genealogy_kg's `export-swift` is its
88
+ first user. `SqliteVecBackend.iter_vectors()` reads a store's vectors back.
89
+ - **0.27.1 (2026-10-19)** -- `[pdf]` and `[ingest]` work again on a fresh
90
+ install. pymupdf4llm 2.0 is an empty placeholder (its code now ships inside
91
+ pymupdf 2.0), so 0.27.0's `pymupdf4llm>=1.28.2` resolved to no PDF support.
92
+ Both extras now declare `pymupdf>=2.0`. No API change.
84
93
  - **0.27.0 (2026-10-14)** -- One converter per format, and incremental
85
94
  vectors. `PdfConverter` is the fleet's single PDF converter (the same
86
95
  `pymupdf4llm` call `doc_kg` makes), `PandocConverter` reads `.tex` and
@@ -219,7 +228,8 @@ The SDK covers the full life of a knowledge graph: core contracts and SQLite
219
228
  storage (`specs`, `extractor`, `store`, `pipeline`), semantic indexing over
220
229
  pluggable vector backends (`semantic`, `vector_backend`, `embed`, `embedder`),
221
230
  document ingestion (`ingest`), a shared temporal contract for dated nodes
222
- (`temporal`), temporal snapshots (`snapshots`), text + image
231
+ (`temporal`), on-device search packs for the Knowledge Press apps
232
+ (`devicepack`), temporal snapshots (`snapshots`), text + image
223
233
  synthesis (`synthesis`), centrality analysis (`analysis`), and a visualization
224
234
  stack that runs from interactive HTML (`viz`) through 3-D layouts and organic
225
235
  trees grown as species (`viz3d`, `viz3d.species`) to Looking Glass light-field
@@ -575,6 +585,21 @@ about the host window.
575
585
  | `CastResult` | Outcome of one cast: `path`, `error`, `elapsed`, and the `message` a status bar shows |
576
586
  | `DEFAULT_QUILT_PRESET` / `DEFAULT_CAST_SCALE` | The preset and scale a cast uses when no `spec` is given (`"16-landscape"` at half size) |
577
587
 
588
+ ### `kg_utils.devicepack`
589
+
590
+ The on-device search-pack format: what the Knowledge Press apps open with no
591
+ server. Needs NumPy for vectors (`semantic` or `viz3d` extra).
592
+
593
+ | Class / function | Description |
594
+ |---|---|
595
+ | `Passage` | One searchable row: id, KG, kind, content, and the optional title, author, genre and location columns |
596
+ | `write_passage_pack()` | Write `<name>.pack` (passages + FTS5) and its `<name>.vectors` sidecar from a vector map; returns `PackStats` |
597
+ | `search_pack()` | The reference hybrid search: dense over the sidecar, BM25 over FTS5 (phrase, then any term), RRF-fused; filters on `genre` only |
598
+ | `build_manifest()` | `manifest.json`: pack version, the embedder, every pack with sizes and checksums |
599
+ | `build_golden()` | `golden.json`: each query's top-k as the packs answer it, the app's parity gate |
600
+ | `read_vector_sidecar()` | Memory-map a sidecar; rejects a truncated or foreign file |
601
+ | `encode_vector()`, `rrf_fuse()`, `fts_phrase_expression()` | The pieces the search is built from |
602
+
578
603
  ### `kg_utils.analysis`
579
604
 
580
605
  | Class / function | Description |
@@ -784,7 +809,7 @@ If you use kgmodule-utils in research or a project, please cite it:
784
809
 
785
810
  **APA**
786
811
 
787
- > Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.27.0) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
812
+ > Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.28.0) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
788
813
 
789
814
  **BibTeX**
790
815
 
@@ -792,7 +817,7 @@ If you use kgmodule-utils in research or a project, please cite it:
792
817
  @software{suchanek_kgmodule_utils,
793
818
  author = {Suchanek, Eric G.},
794
819
  title = {{kgmodule-utils}: Shared SDK for the KGModule Knowledge-Graph Ecosystem},
795
- version = {0.27.0},
820
+ version = {0.28.0},
796
821
  year = {2026},
797
822
  publisher = {Flux-Frontiers},
798
823
  url = {https://github.com/Flux-Frontiers/KG_utils},
@@ -1,7 +1,7 @@
1
1
 
2
2
  [![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13-blue.svg)](https://www.python.org/)
3
3
  [![License: Elastic-2.0](https://img.shields.io/badge/License-Elastic%202.0-blue.svg)](https://www.elastic.co/licensing/elastic-license)
4
- [![Version](https://img.shields.io/badge/version-0.27.0-blue.svg)](https://github.com/Flux-Frontiers/KG_utils/releases)
4
+ [![Version](https://img.shields.io/badge/version-0.28.0-blue.svg)](https://github.com/Flux-Frontiers/KG_utils/releases)
5
5
  [![CI](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml/badge.svg)](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml)
6
6
  [![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
7
7
  [![DOI](https://img.shields.io/badge/DOI-10.5281%2Fzenodo.21284866-blue.svg)](https://doi.org/10.5281/zenodo.21284866)
@@ -26,6 +26,15 @@ Fleet modules use it in two ways. [PyCodeKG](https://github.com/Flux-Frontiers/p
26
26
 
27
27
  ## Latest News
28
28
 
29
+ - **0.28.0 (2026-10-23)** -- New `kg_utils.devicepack`: the on-device
30
+ search-pack format the Knowledge Press apps read (passage pack, vector
31
+ sidecar, manifest, golden rankings), lifted out of gutenberg_kg's
32
+ `export-swift` so any KG can write it. genealogy_kg's `export-swift` is its
33
+ first user. `SqliteVecBackend.iter_vectors()` reads a store's vectors back.
34
+ - **0.27.1 (2026-10-19)** -- `[pdf]` and `[ingest]` work again on a fresh
35
+ install. pymupdf4llm 2.0 is an empty placeholder (its code now ships inside
36
+ pymupdf 2.0), so 0.27.0's `pymupdf4llm>=1.28.2` resolved to no PDF support.
37
+ Both extras now declare `pymupdf>=2.0`. No API change.
29
38
  - **0.27.0 (2026-10-14)** -- One converter per format, and incremental
30
39
  vectors. `PdfConverter` is the fleet's single PDF converter (the same
31
40
  `pymupdf4llm` call `doc_kg` makes), `PandocConverter` reads `.tex` and
@@ -164,7 +173,8 @@ The SDK covers the full life of a knowledge graph: core contracts and SQLite
164
173
  storage (`specs`, `extractor`, `store`, `pipeline`), semantic indexing over
165
174
  pluggable vector backends (`semantic`, `vector_backend`, `embed`, `embedder`),
166
175
  document ingestion (`ingest`), a shared temporal contract for dated nodes
167
- (`temporal`), temporal snapshots (`snapshots`), text + image
176
+ (`temporal`), on-device search packs for the Knowledge Press apps
177
+ (`devicepack`), temporal snapshots (`snapshots`), text + image
168
178
  synthesis (`synthesis`), centrality analysis (`analysis`), and a visualization
169
179
  stack that runs from interactive HTML (`viz`) through 3-D layouts and organic
170
180
  trees grown as species (`viz3d`, `viz3d.species`) to Looking Glass light-field
@@ -520,6 +530,21 @@ about the host window.
520
530
  | `CastResult` | Outcome of one cast: `path`, `error`, `elapsed`, and the `message` a status bar shows |
521
531
  | `DEFAULT_QUILT_PRESET` / `DEFAULT_CAST_SCALE` | The preset and scale a cast uses when no `spec` is given (`"16-landscape"` at half size) |
522
532
 
533
+ ### `kg_utils.devicepack`
534
+
535
+ The on-device search-pack format: what the Knowledge Press apps open with no
536
+ server. Needs NumPy for vectors (`semantic` or `viz3d` extra).
537
+
538
+ | Class / function | Description |
539
+ |---|---|
540
+ | `Passage` | One searchable row: id, KG, kind, content, and the optional title, author, genre and location columns |
541
+ | `write_passage_pack()` | Write `<name>.pack` (passages + FTS5) and its `<name>.vectors` sidecar from a vector map; returns `PackStats` |
542
+ | `search_pack()` | The reference hybrid search: dense over the sidecar, BM25 over FTS5 (phrase, then any term), RRF-fused; filters on `genre` only |
543
+ | `build_manifest()` | `manifest.json`: pack version, the embedder, every pack with sizes and checksums |
544
+ | `build_golden()` | `golden.json`: each query's top-k as the packs answer it, the app's parity gate |
545
+ | `read_vector_sidecar()` | Memory-map a sidecar; rejects a truncated or foreign file |
546
+ | `encode_vector()`, `rrf_fuse()`, `fts_phrase_expression()` | The pieces the search is built from |
547
+
523
548
  ### `kg_utils.analysis`
524
549
 
525
550
  | Class / function | Description |
@@ -729,7 +754,7 @@ If you use kgmodule-utils in research or a project, please cite it:
729
754
 
730
755
  **APA**
731
756
 
732
- > Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.27.0) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
757
+ > Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.28.0) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
733
758
 
734
759
  **BibTeX**
735
760
 
@@ -737,7 +762,7 @@ If you use kgmodule-utils in research or a project, please cite it:
737
762
  @software{suchanek_kgmodule_utils,
738
763
  author = {Suchanek, Eric G.},
739
764
  title = {{kgmodule-utils}: Shared SDK for the KGModule Knowledge-Graph Ecosystem},
740
- version = {0.27.0},
765
+ version = {0.28.0},
741
766
  year = {2026},
742
767
  publisher = {Flux-Frontiers},
743
768
  url = {https://github.com/Flux-Frontiers/KG_utils},
@@ -9,7 +9,7 @@ build-backend = "poetry.core.masonry.api"
9
9
 
10
10
  [project]
11
11
  name = "kgmodule-utils"
12
- version = "0.27.0"
12
+ version = "0.28.0"
13
13
  description = "Shared types, graph store, semantic index, and pipeline base for the KGModule SDK"
14
14
  readme = "README.md"
15
15
  license = "Elastic-2.0"
@@ -83,18 +83,21 @@ lancedb = [
83
83
  # Document ingestion (kg_utils.ingest): pymupdf4llm converts PDF (the same
84
84
  # call doc_kg makes, so both paths agree); anydoc converts Word, PowerPoint,
85
85
  # Excel, OpenDocument, RTF, EPUB and CSV. Kept optional because .md/.txt/.rst
86
- # ingest with no dependency at all. pymupdf4llm pulls in pymupdf (AGPL-3.0 or
87
- # Artifex commercial), already a core doc_kg dependency. .tex/.html need the
88
- # pandoc binary, which is not a Python dependency. No OCR: scanned PDFs are
89
- # recorded as failed with a reason, never silently dropped.
86
+ # ingest with no dependency at all. pymupdf (AGPL-3.0 or Artifex commercial)
87
+ # is declared, not pymupdf4llm: since 2.0 the pymupdf wheel ships the
88
+ # pymupdf4llm module, and the pymupdf4llm 2.0 distribution is an empty
89
+ # placeholder that installs no code, so declaring it alone resolves to no PDF
90
+ # support at all. .tex/.html need the pandoc binary, which is not a Python
91
+ # dependency. No OCR: scanned PDFs are recorded as failed with a reason,
92
+ # never silently dropped.
90
93
  ingest = [
91
94
  "firecrawl-anydoc>=0.1.9",
92
- "pymupdf4llm>=1.28.2",
95
+ "pymupdf>=2.0",
93
96
  ]
94
97
  # PDF only: what doc_kg needs to read PDFs through PdfConverter without
95
98
  # pulling in anydoc. [ingest] stays the "every format" extra.
96
99
  pdf = [
97
- "pymupdf4llm>=1.28.2",
100
+ "pymupdf>=2.0",
98
101
  ]
99
102
  synthesis = [
100
103
  "httpx>=0.27.0",
@@ -126,7 +129,7 @@ pytest = ">=9.0.3"
126
129
  pytest-cov = "^5.0.0"
127
130
  # Exercises the real converters in the ingest tests rather than stubs.
128
131
  firecrawl-anydoc = ">=0.1.9"
129
- pymupdf4llm = ">=1.28.2"
132
+ pymupdf = ">=2.0"
130
133
  # Capped below 0.16 deliberately: 0.16 expands the default rules and reformats
131
134
  # Python in Markdown blocks. Adopt on purpose; lift the cap in its own change.
132
135
  ruff = ">=0.15,<0.16"
@@ -68,4 +68,4 @@ The core install is stdlib-only; everything heavier is opt-in.
68
68
  pip install 'kgmodule-utils[synthesis-mflux]' # + mflux (Apple Silicon local gen)
69
69
  """
70
70
 
71
- __version__ = "0.27.0"
71
+ __version__ = "0.28.0"
@@ -0,0 +1,717 @@
1
+ """kg_utils.devicepack -- on-device search packs, the format the Knowledge Press apps read.
2
+
3
+ A device pack is a corpus cut down to what an app needs to search it with no
4
+ server: a SQLite file of passages with an FTS5 index, a sidecar of int8 (or
5
+ float32) embeddings beside it, a ``manifest.json`` naming the embedder the
6
+ vectors require, and a ``golden.json`` of reference rankings the app's own
7
+ search must reproduce. The Swift kit in ``knowledge_press`` and
8
+ ``kpress_family`` reads exactly this; ``gutenberg_kg``'s ``export-swift`` wrote
9
+ it first, and this module is that format lifted out so other KGs can write it.
10
+ Nothing here knows what a passage *is*: a book chunk, a diary entry and a
11
+ person's summary are all a :class:`Passage`.
12
+
13
+ Layout::
14
+
15
+ <out>/
16
+ manifest.json pack version, the embedder, every pack and its checksum
17
+ <name>.pack passages + FTS5 over their text
18
+ <name>.vectors their embeddings, row-major, memory-mappable
19
+ golden.json reference top-k per query, as the packs answer it
20
+
21
+ Why the vectors sit beside the pack
22
+ -----------------------------------
23
+ A ``vec0`` table cannot be read without the sqlite-vec C extension compiled into
24
+ the reader, and iOS ships stock SQLite. The sidecar is a 32-byte header (magic,
25
+ dtype, dimension, count) and then row-major vectors; ``passages.vector_index``
26
+ is a dense row number into it, so a passage with no vector leaves no hole. The
27
+ device memory-maps the file and takes dot products straight out of it.
28
+
29
+ Search
30
+ ------
31
+ :func:`search_pack` is the reference the app must match: a dense channel over
32
+ the sidecar, a BM25 channel over FTS5 (exact phrase first, any term as the
33
+ fallback), fused by reciprocal rank at :data:`RRF_K`. It filters on ``genre``
34
+ only, as the Swift engine does; what a pack holds is the exporter's choice.
35
+
36
+ Needs NumPy for anything touching vectors (the ``semantic`` or ``viz3d``
37
+ extra); writing and reading passages does not.
38
+
39
+ Author: Eric G. Suchanek, PhD
40
+ """
41
+
42
+ from __future__ import annotations
43
+
44
+ import hashlib
45
+ import sqlite3
46
+ from collections.abc import Iterable, Mapping, Sequence
47
+ from dataclasses import dataclass
48
+ from datetime import UTC, datetime
49
+ from pathlib import Path
50
+ from typing import Any, Final
51
+
52
+ #: The pack format. The app refuses a manifest newer than it understands.
53
+ PACK_VERSION: Final[int] = 1
54
+
55
+ #: First eight bytes of every vector sidecar.
56
+ VECTOR_MAGIC: Final[bytes] = b"GKGVEC01"
57
+
58
+ #: Header bytes before the vector rows. 32 keeps the rows 16-byte aligned, so
59
+ #: the device can map them and hand them straight to SIMD.
60
+ VECTOR_HEADER_BYTES: Final[int] = 32
61
+
62
+ #: Reciprocal-rank-fusion constant, shared by the worker, the packs and the app.
63
+ RRF_K: Final[int] = 60
64
+
65
+ DTYPES: Final[tuple[str, ...]] = ("int8", "float")
66
+ _DTYPE_CODES: Final[dict[str, int]] = {"int8": 0, "float": 1}
67
+ _DTYPE_ITEMSIZE: Final[dict[str, int]] = {"int8": 1, "float": 4}
68
+
69
+ #: Default parity tolerances written into ``golden.json``. ``max_rank_drift``
70
+ #: is not zero because the reference and the device differ in their last bits
71
+ #: (numpy over dequantised int8 against Accelerate), so near-ties may swap.
72
+ DEFAULT_TOLERANCE: Final[dict[str, float | int]] = {
73
+ "rank_overlap": 0.9,
74
+ "score_delta": 0.02,
75
+ "max_rank_drift": 2,
76
+ }
77
+
78
+ PASSAGE_SCHEMA: Final[str] = """
79
+ CREATE TABLE pack_meta (key TEXT PRIMARY KEY, value TEXT);
80
+
81
+ CREATE TABLE passages (
82
+ id TEXT PRIMARY KEY,
83
+ kg_name TEXT NOT NULL,
84
+ kg_kind TEXT NOT NULL,
85
+ kind TEXT NOT NULL,
86
+ name TEXT,
87
+ title TEXT,
88
+ node_title TEXT,
89
+ author TEXT,
90
+ genre TEXT,
91
+ book TEXT,
92
+ file_path TEXT,
93
+ char_start INTEGER,
94
+ chapter INTEGER,
95
+ timestamp TEXT,
96
+ vector_index INTEGER,
97
+ content TEXT NOT NULL
98
+ );
99
+ CREATE INDEX idx_passages_scope ON passages(genre, kind);
100
+ CREATE INDEX idx_passages_vec ON passages(vector_index);
101
+ CREATE INDEX idx_passages_read ON passages(file_path, kind, char_start);
102
+ CREATE INDEX idx_passages_kg ON passages(kg_name);
103
+ """
104
+
105
+ FTS_SCHEMA: Final[str] = """
106
+ CREATE VIRTUAL TABLE passages_fts USING fts5(
107
+ content,
108
+ content='passages',
109
+ content_rowid='rowid',
110
+ tokenize='porter unicode61 remove_diacritics 2'
111
+ );
112
+ """
113
+
114
+
115
+ class PackError(RuntimeError):
116
+ """A pack could not be written or read, or NumPy is missing."""
117
+
118
+
119
+ @dataclass(frozen=True)
120
+ class Passage:
121
+ """One searchable row of a pack.
122
+
123
+ :param id: Unique within the pack; ``<kg_name>:<node_id>`` by convention.
124
+ :param kg_name: The KG the passage came from.
125
+ :param kg_kind: Its KG kind, as a hit reports it.
126
+ :param kind: The node's kind; the app reads ``section`` and ``chunk`` for
127
+ a book's chapters, and nothing else by kind.
128
+ :param content: The text searched and shown.
129
+ :param name: Short label.
130
+ :param title: The work's title, for hit cards.
131
+ :param node_title: This node's own title.
132
+ :param author: Author, or any person the hit card names.
133
+ :param genre: The scope the app filters on.
134
+ :param book: Book or collection slug.
135
+ :param file_path: Source document path.
136
+ :param char_start: Offset in the source document.
137
+ :param chapter: Chapter number, for chapter-less books.
138
+ :param timestamp: ISO date, for dated passages.
139
+ """
140
+
141
+ id: str
142
+ kg_name: str
143
+ kg_kind: str
144
+ kind: str
145
+ content: str
146
+ name: str | None = None
147
+ title: str | None = None
148
+ node_title: str | None = None
149
+ author: str | None = None
150
+ genre: str | None = None
151
+ book: str | None = None
152
+ file_path: str | None = None
153
+ char_start: int | None = None
154
+ chapter: int | None = None
155
+ timestamp: str | None = None
156
+
157
+
158
+ _PASSAGE_COLUMNS: Final[tuple[str, ...]] = (
159
+ "id",
160
+ "kg_name",
161
+ "kg_kind",
162
+ "kind",
163
+ "name",
164
+ "title",
165
+ "node_title",
166
+ "author",
167
+ "genre",
168
+ "book",
169
+ "file_path",
170
+ "char_start",
171
+ "chapter",
172
+ "timestamp",
173
+ "content",
174
+ )
175
+
176
+
177
+ @dataclass
178
+ class PackStats:
179
+ """What a finished pack contains.
180
+
181
+ :param path: The pack file.
182
+ :param passages: Rows written.
183
+ :param vectors: Vectors written to the sidecar.
184
+ :param missing_vectors: Passages with no vector.
185
+ :param bytes: Pack size.
186
+ :param sha256: Pack checksum.
187
+ :param sidecar: The ``.vectors`` sidecar, if any.
188
+ :param sidecar_bytes: Its size.
189
+ :param sidecar_sha256: Its checksum.
190
+ """
191
+
192
+ path: Path
193
+ passages: int = 0
194
+ vectors: int = 0
195
+ missing_vectors: int = 0
196
+ bytes: int = 0
197
+ sha256: str = ""
198
+ sidecar: Path | None = None
199
+ sidecar_bytes: int = 0
200
+ sidecar_sha256: str = ""
201
+
202
+ @property
203
+ def name(self) -> str:
204
+ """The pack's file name."""
205
+ return self.path.name
206
+
207
+ @property
208
+ def total_bytes(self) -> int:
209
+ """Pack plus sidecar: what has to reach the device."""
210
+ return self.bytes + self.sidecar_bytes
211
+
212
+
213
+ def _numpy() -> Any:
214
+ try:
215
+ import numpy # noqa: PLC0415 - optional, only for vectors
216
+ except ImportError as exc:
217
+ raise PackError("vectors need numpy: install kgmodule-utils[semantic]") from exc
218
+ return numpy
219
+
220
+
221
+ def sha256_file(path: Path) -> str:
222
+ """Streamed SHA-256 of a file.
223
+
224
+ :param path: The file.
225
+ :return: Hex digest.
226
+ """
227
+ digest = hashlib.sha256()
228
+ with open(path, "rb") as handle:
229
+ for block in iter(lambda: handle.read(1 << 20), b""):
230
+ digest.update(block)
231
+ return digest.hexdigest()
232
+
233
+
234
+ # ---------------------------------------------------------------------------
235
+ # Vectors
236
+ # ---------------------------------------------------------------------------
237
+
238
+
239
+ def encode_vector(vector: Any, dtype: str) -> bytes:
240
+ """Encode one vector for the sidecar.
241
+
242
+ int8 vectors are L2-normalised first. Cosine is scale-invariant, so no
243
+ ranking changes, and the ``* 127`` scale then lands inside int8 range.
244
+
245
+ :param vector: A float vector.
246
+ :param dtype: ``"int8"`` or ``"float"``.
247
+ :return: The encoded bytes.
248
+ """
249
+ numpy = _numpy()
250
+ array = numpy.asarray(vector, dtype=numpy.float32)
251
+ if dtype == "float":
252
+ return array.tobytes()
253
+ if dtype != "int8":
254
+ raise PackError(f"dtype must be one of {DTYPES}, got {dtype!r}")
255
+ norm = float(numpy.linalg.norm(array))
256
+ if norm > 0:
257
+ array = array / norm
258
+ return numpy.clip(numpy.round(array * 127.0), -128, 127).astype(numpy.int8).tobytes()
259
+
260
+
261
+ def vector_header(dtype: str, dim: int, count: int) -> bytes:
262
+ """Build the sidecar's 32-byte header.
263
+
264
+ :param dtype: ``"int8"`` or ``"float"``.
265
+ :param dim: Embedding dimension.
266
+ :param count: Vectors in the file.
267
+ :return: Exactly :data:`VECTOR_HEADER_BYTES` bytes.
268
+ """
269
+ header = bytearray(VECTOR_HEADER_BYTES)
270
+ header[0:8] = VECTOR_MAGIC
271
+ header[8] = _DTYPE_CODES[dtype]
272
+ header[12:16] = int(dim).to_bytes(4, "little")
273
+ header[16:24] = int(count).to_bytes(8, "little")
274
+ return bytes(header)
275
+
276
+
277
+ def read_vector_sidecar(sidecar: Path) -> tuple[Any, str]:
278
+ """Memory-map a sidecar.
279
+
280
+ :param sidecar: A ``.vectors`` file.
281
+ :return: ``(matrix, dtype)``: a ``(count, dim)`` array viewing the file,
282
+ and its dtype name.
283
+ :raises PackError: If the magic, dtype or length disagree, as a truncated
284
+ download would.
285
+ """
286
+ numpy = _numpy()
287
+ with open(sidecar, "rb") as handle:
288
+ header = handle.read(VECTOR_HEADER_BYTES)
289
+ if len(header) < VECTOR_HEADER_BYTES or header[0:8] != VECTOR_MAGIC:
290
+ raise PackError(f"{sidecar} is not a vector sidecar")
291
+ dtype = {code: name for name, code in _DTYPE_CODES.items()}.get(header[8])
292
+ if dtype is None:
293
+ raise PackError(f"{sidecar} has an unknown vector dtype ({header[8]})")
294
+ dim = int.from_bytes(header[12:16], "little")
295
+ count = int.from_bytes(header[16:24], "little")
296
+ expected = VECTOR_HEADER_BYTES + count * dim * _DTYPE_ITEMSIZE[dtype]
297
+ actual = sidecar.stat().st_size
298
+ if actual != expected:
299
+ raise PackError(
300
+ f"{sidecar} is {actual} bytes, expected {expected} for {count} x {dim} {dtype}"
301
+ )
302
+ matrix = numpy.memmap(
303
+ sidecar,
304
+ dtype=numpy.int8 if dtype == "int8" else numpy.float32,
305
+ mode="r",
306
+ offset=VECTOR_HEADER_BYTES,
307
+ shape=(count, dim),
308
+ )
309
+ return matrix, dtype
310
+
311
+
312
+ # ---------------------------------------------------------------------------
313
+ # Writing
314
+ # ---------------------------------------------------------------------------
315
+
316
+
317
+ def write_passage_pack(
318
+ dest: Path,
319
+ passages: Iterable[Passage],
320
+ vectors: Mapping[str, Any] | None = None,
321
+ *,
322
+ dim: int,
323
+ embed_model: str,
324
+ dtype: str = "int8",
325
+ meta: Mapping[str, object] | None = None,
326
+ ) -> PackStats:
327
+ """Write a passage pack and, given vectors, its sidecar.
328
+
329
+ A passage with no vector stays in the pack, findable lexically, and is
330
+ counted in ``missing_vectors``. Ids must be unique: a collision fails the
331
+ write rather than shrinking the pack.
332
+
333
+ :param dest: The ``.pack`` path; replaced if it exists. The sidecar is
334
+ written beside it as ``<stem>.vectors``.
335
+ :param passages: The rows, in the order to store them.
336
+ :param vectors: Passage id -> vector, or ``None`` for a lexical-only pack.
337
+ :param dim: Embedding dimension every vector must have.
338
+ :param embed_model: The model the vectors came from, stamped into the pack.
339
+ :param dtype: ``"int8"`` (default) or ``"float"``.
340
+ :param meta: Extra ``pack_meta`` rows.
341
+ :return: Stats for the finished pack.
342
+ :raises PackError: On a bad dtype or a vector of the wrong dimension.
343
+ """
344
+ if dtype not in DTYPES:
345
+ raise PackError(f"dtype must be one of {DTYPES}, got {dtype!r}")
346
+ dest.parent.mkdir(parents=True, exist_ok=True)
347
+ dest.unlink(missing_ok=True)
348
+ sidecar = dest.with_suffix(".vectors")
349
+ sidecar.unlink(missing_ok=True)
350
+
351
+ stats = PackStats(path=dest)
352
+ con = sqlite3.connect(str(dest))
353
+ try:
354
+ con.executescript("PRAGMA journal_mode=OFF; PRAGMA synchronous=OFF;")
355
+ con.executescript(PASSAGE_SCHEMA)
356
+ insert = (
357
+ f"INSERT INTO passages({', '.join(_PASSAGE_COLUMNS)}) "
358
+ f"VALUES ({', '.join('?' for _ in _PASSAGE_COLUMNS)})"
359
+ )
360
+ con.executemany(insert, ([getattr(p, c) for c in _PASSAGE_COLUMNS] for p in passages))
361
+ stats.passages = con.execute("SELECT COUNT(*) FROM passages").fetchone()[0]
362
+ con.executescript(FTS_SCHEMA)
363
+ con.execute("INSERT INTO passages_fts(passages_fts) VALUES('rebuild')")
364
+
365
+ if vectors:
366
+ itemsize = _DTYPE_ITEMSIZE[dtype]
367
+ assignments: list[tuple[int, str]] = []
368
+ with open(sidecar, "wb") as handle:
369
+ handle.write(b"\0" * VECTOR_HEADER_BYTES)
370
+ for (passage_id,) in con.execute("SELECT id FROM passages ORDER BY rowid"):
371
+ vector = vectors.get(passage_id)
372
+ if vector is None:
373
+ continue
374
+ encoded = encode_vector(vector, dtype)
375
+ if len(encoded) != dim * itemsize:
376
+ raise PackError(
377
+ f"{passage_id} has {len(encoded) // itemsize} dimensions, "
378
+ f"expected {dim}"
379
+ )
380
+ handle.write(encoded)
381
+ assignments.append((len(assignments), passage_id))
382
+ handle.seek(0)
383
+ handle.write(vector_header(dtype, dim, len(assignments)))
384
+ con.executemany("UPDATE passages SET vector_index = ? WHERE id = ?", assignments)
385
+ stats.vectors = len(assignments)
386
+ stats.missing_vectors = stats.passages - stats.vectors
387
+
388
+ stamps: dict[str, object] = {
389
+ "pack": dest.stem,
390
+ "pack_version": PACK_VERSION,
391
+ "generated": datetime.now(UTC).isoformat(timespec="seconds"),
392
+ "embed_model": embed_model,
393
+ "embed_dim": dim,
394
+ "vector_dtype": dtype if stats.vectors else "none",
395
+ "rrf_k": RRF_K,
396
+ **(meta or {}),
397
+ }
398
+ con.executemany(
399
+ "INSERT OR REPLACE INTO pack_meta(key, value) VALUES (?, ?)",
400
+ [(key, str(value)) for key, value in stamps.items()],
401
+ )
402
+ con.commit()
403
+ con.execute("VACUUM")
404
+ except sqlite3.IntegrityError as exc:
405
+ raise PackError(f"duplicate passage id in {dest.name}: {exc}") from exc
406
+ finally:
407
+ con.close()
408
+
409
+ if stats.vectors == 0:
410
+ sidecar.unlink(missing_ok=True)
411
+ else:
412
+ stats.sidecar = sidecar
413
+ stats.sidecar_bytes = sidecar.stat().st_size
414
+ stats.sidecar_sha256 = sha256_file(sidecar)
415
+ stats.bytes = dest.stat().st_size
416
+ stats.sha256 = sha256_file(dest)
417
+ return stats
418
+
419
+
420
+ def build_manifest(
421
+ packs: Sequence[PackStats],
422
+ *,
423
+ embed_model: str,
424
+ dim: int,
425
+ dtype: str,
426
+ golden: str | None = "golden.json",
427
+ extra: Mapping[str, Any] | None = None,
428
+ ) -> dict[str, Any]:
429
+ """Describe the packs for the app to check itself against.
430
+
431
+ The embedder block matters most: vectors are meaningless to a query
432
+ embedded by another model, so the app refuses the corpus until they agree.
433
+
434
+ :param packs: The finished packs.
435
+ :param embed_model: The embedder's model name.
436
+ :param dim: Its dimension.
437
+ :param dtype: The vectors' dtype.
438
+ :param golden: The golden file's name, or ``None``.
439
+ :param extra: Further top-level keys, such as privacy settings.
440
+ :return: The manifest document.
441
+ """
442
+ return {
443
+ "pack_version": PACK_VERSION,
444
+ "generated": datetime.now(UTC).isoformat(timespec="seconds"),
445
+ "embedder": {
446
+ "model": embed_model,
447
+ "dim": dim,
448
+ "normalized": True,
449
+ "note": "Query vectors must come from this model; the packs are unusable without it.",
450
+ },
451
+ "vector_dtype": dtype if any(p.vectors for p in packs) else "none",
452
+ "rrf_k": RRF_K,
453
+ "vector_sidecar": {
454
+ "magic": VECTOR_MAGIC.decode("ascii"),
455
+ "header_bytes": VECTOR_HEADER_BYTES,
456
+ "layout": "row-major, one vector per row, indexed by passages.vector_index",
457
+ },
458
+ "packs": [
459
+ {
460
+ "name": p.name,
461
+ "bytes": p.bytes,
462
+ "sha256": p.sha256,
463
+ "passages": p.passages,
464
+ "vectors": p.vectors,
465
+ "passages_without_vectors": p.missing_vectors,
466
+ "sidecar": (
467
+ {"name": p.sidecar.name, "bytes": p.sidecar_bytes, "sha256": p.sidecar_sha256}
468
+ if p.sidecar is not None
469
+ else None
470
+ ),
471
+ }
472
+ for p in packs
473
+ ],
474
+ "total_bytes": sum(p.total_bytes for p in packs),
475
+ "golden": golden,
476
+ **(extra or {}),
477
+ }
478
+
479
+
480
+ # ---------------------------------------------------------------------------
481
+ # Search: the reference the app must reproduce
482
+ # ---------------------------------------------------------------------------
483
+
484
+
485
+ def fts_terms(query: str) -> list[str]:
486
+ """Split a query into bare alphanumeric terms, so none is read as FTS5 syntax.
487
+
488
+ :param query: The query text.
489
+ :return: The terms, in order.
490
+ """
491
+ return [t for t in "".join(c if c.isalnum() else " " for c in query).split() if t]
492
+
493
+
494
+ def fts_phrase_expression(query: str) -> str:
495
+ """The exact-phrase FTS5 expression, tried before the any-term fallback.
496
+
497
+ :param query: The query text.
498
+ :return: A quoted phrase, or ``""`` when nothing is searchable.
499
+ """
500
+ terms = fts_terms(query)
501
+ return '"' + " ".join(terms) + '"' if terms else ""
502
+
503
+
504
+ def fts_match_expression(query: str) -> str:
505
+ """The any-term FTS5 expression: every term quoted, OR-ed.
506
+
507
+ :param query: The query text.
508
+ :return: An FTS5 MATCH expression, or ``""``.
509
+ """
510
+ return " OR ".join(f'"{term}"' for term in fts_terms(query))
511
+
512
+
513
+ def rrf_fuse(
514
+ dense_ids: Sequence[str], lexical_ids: Sequence[str], k: int, *, rrf_k: int = RRF_K
515
+ ) -> list[str]:
516
+ """Blend two rank lists by reciprocal rank fusion.
517
+
518
+ Lexical ranks are added first, so a cross-channel tie goes to the exact
519
+ match.
520
+
521
+ :param dense_ids: Ids best-first by cosine.
522
+ :param lexical_ids: Ids best-first by BM25.
523
+ :param k: How many fused ids to return.
524
+ :param rrf_k: The fusion constant.
525
+ :return: Fused ids, best-first.
526
+ """
527
+ scores: dict[str, float] = {}
528
+ for rank, node_id in enumerate(lexical_ids):
529
+ scores[node_id] = scores.get(node_id, 0.0) + 1.0 / (rrf_k + rank)
530
+ for rank, node_id in enumerate(dense_ids):
531
+ scores[node_id] = scores.get(node_id, 0.0) + 1.0 / (rrf_k + rank)
532
+ return sorted(scores, key=lambda i: -scores[i])[:k]
533
+
534
+
535
+ def search_pack(
536
+ pack: Path,
537
+ query_vector: Any,
538
+ query_text: str,
539
+ *,
540
+ k: int = 10,
541
+ genre: str | None = None,
542
+ lexical: bool = True,
543
+ ) -> tuple[list[dict[str, Any]], set[str]]:
544
+ """Run a pack's hybrid search: dense + BM25, fused by RRF.
545
+
546
+ Both channels oversample to ``k * 3``. A hit found only lexically gets its
547
+ cosine from the sidecar, so every fused hit carries an honest score.
548
+
549
+ :param pack: A passage pack, its sidecar beside it.
550
+ :param query_vector: The query embedding.
551
+ :param query_text: The query, for the lexical channel.
552
+ :param k: Hits to return.
553
+ :param genre: Restrict to one genre, as the app's scope picker does.
554
+ :param lexical: Include the BM25 channel.
555
+ :return: Hits best-first (``node_id``, ``score``, ``kind``, ``genre``,
556
+ ``title``, ``content`` and the rest of the row), and the ids the
557
+ lexical channel rescued from outside the dense top ``k``.
558
+ """
559
+ numpy = _numpy()
560
+ con = sqlite3.connect(f"file:{pack}?mode=ro", uri=True)
561
+ con.row_factory = sqlite3.Row
562
+ try:
563
+ scope = " AND genre = ?" if genre else ""
564
+ params: list[object] = [genre] if genre else []
565
+
566
+ dense_ids: list[str] = []
567
+ all_scores: dict[str, float] = {}
568
+ sidecar = pack.with_suffix(".vectors")
569
+ if sidecar.exists():
570
+ matrix, _dtype = read_vector_sidecar(sidecar)
571
+ rows = con.execute(
572
+ f"SELECT id, vector_index FROM passages WHERE vector_index IS NOT NULL{scope}",
573
+ params,
574
+ ).fetchall()
575
+ if rows:
576
+ ids = [row["id"] for row in rows]
577
+ indices = numpy.fromiter(
578
+ (row["vector_index"] for row in rows), dtype=numpy.int64, count=len(rows)
579
+ )
580
+ candidates = numpy.asarray(matrix[indices], dtype=numpy.float32)
581
+ norms = numpy.linalg.norm(candidates, axis=1).clip(min=1e-12)
582
+ query = numpy.asarray(query_vector, dtype=numpy.float32)
583
+ query = query / max(float(numpy.linalg.norm(query)), 1e-12)
584
+ similarity = (candidates @ query) / norms
585
+ dense_ids = [ids[i] for i in numpy.argsort(-similarity)[: k * 3]]
586
+ all_scores = {ids[i]: float(similarity[i]) for i in range(len(ids))}
587
+
588
+ lexical_ids: list[str] = []
589
+ if lexical:
590
+ sql = (
591
+ "SELECT p.id AS id FROM passages_fts f JOIN passages p ON p.rowid = f.rowid "
592
+ f"WHERE passages_fts MATCH ?{scope.replace('genre', 'p.genre')} "
593
+ "ORDER BY bm25(passages_fts) LIMIT ?"
594
+ )
595
+
596
+ def run(expression: str) -> list[str]:
597
+ if not expression:
598
+ return []
599
+ try:
600
+ return [r["id"] for r in con.execute(sql, [expression, *params, k * 3])]
601
+ except sqlite3.Error:
602
+ return []
603
+
604
+ lexical_ids = run(fts_phrase_expression(query_text)) or run(
605
+ fts_match_expression(query_text)
606
+ )
607
+ lexical_ids = [i for i in lexical_ids if i in all_scores]
608
+
609
+ ordered = rrf_fuse(dense_ids, lexical_ids, k) if lexical_ids else dense_ids[:k]
610
+ rescued = set(lexical_ids) - set(dense_ids[:k])
611
+ if not ordered:
612
+ return [], set()
613
+
614
+ marks = ", ".join("?" for _ in ordered)
615
+ by_id = {
616
+ row["id"]: dict(row)
617
+ for row in con.execute(f"SELECT * FROM passages WHERE id IN ({marks})", ordered)
618
+ }
619
+ hits = []
620
+ for node_id in ordered:
621
+ row = by_id.get(node_id)
622
+ if row is None:
623
+ continue
624
+ hits.append(
625
+ {
626
+ **row,
627
+ "node_id": node_id,
628
+ "score": round(all_scores.get(node_id, 0.0), 4),
629
+ }
630
+ )
631
+ return hits, rescued & {h["node_id"] for h in hits}
632
+ finally:
633
+ con.close()
634
+
635
+
636
+ def build_golden(
637
+ packs: Mapping[str, Path],
638
+ queries: Sequence[str],
639
+ query_vectors: Sequence[Any],
640
+ *,
641
+ k: int,
642
+ embed_model: str,
643
+ dim: int,
644
+ dtype: str,
645
+ tolerance: Mapping[str, float | int] | None = None,
646
+ rescue_tolerance: float | None = None,
647
+ ) -> dict[str, Any]:
648
+ """Record each query's top-k as the packs answer it: the app's parity gate.
649
+
650
+ The caller embeds the queries with the same model the packs were built
651
+ with, so this module never loads one.
652
+
653
+ :param packs: Pack name -> path; each is searched on its own.
654
+ :param queries: The golden queries.
655
+ :param query_vectors: Their embeddings, in the same order.
656
+ :param k: Hits to record per query.
657
+ :param embed_model: The embedder's model name.
658
+ :param dim: Its dimension.
659
+ :param dtype: The packs' vector dtype.
660
+ :param tolerance: Parity tolerances; :data:`DEFAULT_TOLERANCE` if omitted.
661
+ :param rescue_tolerance: Recorded when the app checks its own constant
662
+ against it.
663
+ :return: The golden document. Each query entry has ``packs``; a caller
664
+ that merges packs adds its own ``all`` ranking.
665
+ """
666
+ if len(queries) != len(query_vectors):
667
+ raise PackError("queries and query_vectors differ in length")
668
+ entries = []
669
+ for query, vector in zip(queries, query_vectors, strict=True):
670
+ ranked: dict[str, list[dict[str, Any]]] = {}
671
+ for name, path in packs.items():
672
+ hits, _rescued = search_pack(path, vector, query, k=k)
673
+ ranked[name] = [
674
+ {"rank": rank, "node_id": hit["node_id"], "score": hit["score"]}
675
+ for rank, hit in enumerate(hits)
676
+ ]
677
+ entries.append({"query": query, "packs": ranked})
678
+ document: dict[str, Any] = {
679
+ "generated": datetime.now(UTC).isoformat(timespec="seconds"),
680
+ "embed_model": embed_model,
681
+ "embed_dim": dim,
682
+ "vector_dtype": dtype,
683
+ "rrf_k": RRF_K,
684
+ "k": k,
685
+ "tolerance": dict(tolerance or DEFAULT_TOLERANCE),
686
+ "queries": entries,
687
+ }
688
+ if rescue_tolerance is not None:
689
+ document["rescue_tolerance"] = rescue_tolerance
690
+ return document
691
+
692
+
693
+ __all__ = [
694
+ "DEFAULT_TOLERANCE",
695
+ "DTYPES",
696
+ "FTS_SCHEMA",
697
+ "PACK_VERSION",
698
+ "PASSAGE_SCHEMA",
699
+ "RRF_K",
700
+ "VECTOR_HEADER_BYTES",
701
+ "VECTOR_MAGIC",
702
+ "PackError",
703
+ "PackStats",
704
+ "Passage",
705
+ "build_golden",
706
+ "build_manifest",
707
+ "encode_vector",
708
+ "fts_match_expression",
709
+ "fts_phrase_expression",
710
+ "fts_terms",
711
+ "read_vector_sidecar",
712
+ "rrf_fuse",
713
+ "search_pack",
714
+ "sha256_file",
715
+ "vector_header",
716
+ "write_passage_pack",
717
+ ]
@@ -47,7 +47,7 @@ from __future__ import annotations
47
47
 
48
48
  import logging
49
49
  import math
50
- from collections.abc import Iterable, Sequence
50
+ from collections.abc import Iterable, Iterator, Sequence
51
51
  from pathlib import Path
52
52
  from typing import Any, Protocol, runtime_checkable
53
53
 
@@ -653,6 +653,30 @@ class SqliteVecBackend:
653
653
  except Exception: # noqa: BLE001
654
654
  return set()
655
655
 
656
+ def iter_vectors(self) -> Iterator[tuple[str, Any]]:
657
+ """Yield ``(id, vector)`` for every stored vector, as float32 arrays.
658
+
659
+ An int8 store yields its values divided by 127, the inverse of
660
+ :meth:`_encode`; cosine is scale-invariant, so rankings are the same
661
+ either way. For exporters that copy a store's vectors elsewhere, such
662
+ as :mod:`kg_utils.devicepack`.
663
+
664
+ :return: An iterator in ``vec_meta`` rowid order; empty when the
665
+ store does not exist.
666
+ """
667
+ import numpy as np # pylint: disable=import-outside-toplevel
668
+
669
+ if self._conn is None and not self.db_path.exists():
670
+ return
671
+ dtype = np.int8 if self.dtype == "int8" else np.float32
672
+ rows = self._c().execute(
673
+ "SELECT m.id, v.embedding FROM vec_meta m "
674
+ "JOIN vec_nodes v ON v.rowid = m.rowid ORDER BY m.rowid"
675
+ )
676
+ for node_id, blob in rows:
677
+ vector = np.frombuffer(blob, dtype=dtype).astype(np.float32)
678
+ yield node_id, vector / 127.0 if self.dtype == "int8" else vector
679
+
656
680
  def count(self) -> int:
657
681
  # Don't create a fresh db file just to count a store that isn't there.
658
682
  if self._conn is None and not self.db_path.exists():
File without changes