kgmodule-utils 0.26.0__tar.gz → 0.27.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/PKG-INFO +50 -10
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/README.md +45 -8
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/pyproject.toml +21 -9
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/__init__.py +1 -1
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/ingest/__init__.py +19 -1
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/ingest/converters.py +187 -11
- kgmodule_utils-0.27.1/src/kg_utils/ingest/derived.py +148 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/pipeline.py +50 -8
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/semantic.py +111 -14
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/specs.py +17 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/vector_backend.py +71 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/LICENSE +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/analysis/__init__.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/analysis/scores.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/corpus_embedder.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/embed.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/embedder.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/extractor.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/ingest/manifest.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/ingest/pipeline.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/module.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/py.typed +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/retrieval/__init__.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/retrieval/hits.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/snapshots/__init__.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/snapshots/manager.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/snapshots/models.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/store.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/__init__.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/_config.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/_image.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/_text.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/factory.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/temporal.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/validation.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz/__init__.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz/graph_html.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz/theme.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz/tooltip.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/__init__.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/layout.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/organic.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/qt.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/species.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/worker/__init__.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/worker/client.py +0 -0
- {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/worker/ops.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: kgmodule-utils
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.27.1
|
|
4
4
|
Summary: Shared types, graph store, semantic index, and pipeline base for the KGModule SDK
|
|
5
5
|
License-Expression: Elastic-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.12
|
|
|
15
15
|
Classifier: Programming Language :: Python :: 3.13
|
|
16
16
|
Provides-Extra: ingest
|
|
17
17
|
Provides-Extra: lancedb
|
|
18
|
+
Provides-Extra: pdf
|
|
18
19
|
Provides-Extra: semantic
|
|
19
20
|
Provides-Extra: sqlite-vec
|
|
20
21
|
Provides-Extra: synthesis
|
|
@@ -37,10 +38,12 @@ Requires-Dist: openai (>=1.30.0) ; extra == "synthesis"
|
|
|
37
38
|
Requires-Dist: openai (>=1.30.0) ; extra == "synthesis-mflux"
|
|
38
39
|
Requires-Dist: pillow (>=10.0.0) ; extra == "synthesis"
|
|
39
40
|
Requires-Dist: pillow (>=10.0.0) ; extra == "synthesis-mflux"
|
|
41
|
+
Requires-Dist: pymupdf (>=2.0) ; extra == "ingest"
|
|
42
|
+
Requires-Dist: pymupdf (>=2.0) ; extra == "pdf"
|
|
40
43
|
Requires-Dist: pyvis (>=0.3.2) ; extra == "viz"
|
|
41
44
|
Requires-Dist: pyvista (>=0.44.0) ; extra == "viz3d-qt"
|
|
42
45
|
Requires-Dist: pyvista (>=0.44.0) ; extra == "viz3d-render"
|
|
43
|
-
Requires-Dist: quiltwright (>=0.
|
|
46
|
+
Requires-Dist: quiltwright (>=0.16.0) ; extra == "viz3d-qt"
|
|
44
47
|
Requires-Dist: rich (>=13.0.0,<15.0.0) ; extra == "semantic"
|
|
45
48
|
Requires-Dist: sentence-transformers (>=5.4.1) ; extra == "semantic"
|
|
46
49
|
Requires-Dist: sqlite-vec (==0.1.9) ; extra == "semantic"
|
|
@@ -53,7 +56,7 @@ Description-Content-Type: text/markdown
|
|
|
53
56
|
|
|
54
57
|
[](https://www.python.org/)
|
|
55
58
|
[](https://www.elastic.co/licensing/elastic-license)
|
|
56
|
-
[](https://github.com/Flux-Frontiers/KG_utils/releases)
|
|
57
60
|
[](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml)
|
|
58
61
|
[](https://python-poetry.org/)
|
|
59
62
|
[](https://doi.org/10.5281/zenodo.21284866)
|
|
@@ -78,6 +81,16 @@ Fleet modules use it in two ways. [PyCodeKG](https://github.com/Flux-Frontiers/p
|
|
|
78
81
|
|
|
79
82
|
## Latest News
|
|
80
83
|
|
|
84
|
+
- **0.27.1 (2026-10-19)** -- `[pdf]` and `[ingest]` work again on a fresh
|
|
85
|
+
install. pymupdf4llm 2.0 is an empty placeholder (its code now ships inside
|
|
86
|
+
pymupdf 2.0), so 0.27.0's `pymupdf4llm>=1.28.2` resolved to no PDF support.
|
|
87
|
+
Both extras now declare `pymupdf>=2.0`. No API change.
|
|
88
|
+
- **0.27.0 (2026-10-14)** -- One converter per format, and incremental
|
|
89
|
+
vectors. `PdfConverter` is the fleet's single PDF converter (the same
|
|
90
|
+
`pymupdf4llm` call `doc_kg` makes), `PandocConverter` reads `.tex` and
|
|
91
|
+
`.html`, and `write_derived()` writes a converted document with provenance
|
|
92
|
+
frontmatter. `KGModule.sync()` rewrites the graph but re-embeds only the
|
|
93
|
+
nodes whose text changed. New `[pdf]` extra.
|
|
81
94
|
- **0.26.0 (2026-09-26)** -- Leaves hang as the Knowledge Press web forest hangs
|
|
82
95
|
them. New `hang_leaves()` puts each leaf's stalk on its nearest twig, its
|
|
83
96
|
blade out and lifted, its face to the sky, and returns the frame so any
|
|
@@ -302,7 +315,7 @@ pip install 'kgmodule-utils[viz3d-qt]'
|
|
|
302
315
|
|
|
303
316
|
```toml
|
|
304
317
|
[tool.poetry.dependencies]
|
|
305
|
-
kgmodule-utils = { version = ">=0.
|
|
318
|
+
kgmodule-utils = { version = ">=0.27.0", extras = ["semantic", "synthesis"] }
|
|
306
319
|
```
|
|
307
320
|
|
|
308
321
|
---
|
|
@@ -419,7 +432,7 @@ delta = mgr.diff_snapshots(snaps[-1]["key"], snaps[0]["key"])
|
|
|
419
432
|
|
|
420
433
|
| Class | Description |
|
|
421
434
|
|---|---|
|
|
422
|
-
| `KGModule` | Concrete base — implement `make_extractor()`, `kind()`, `analyze()`; get `build()`, `query()`, `pack()`, `stats()` for free. `build_graph(wipe=True)` drops the vector index it invalidates; `drop_index()` does so on demand |
|
|
435
|
+
| `KGModule` | Concrete base — implement `make_extractor()`, `kind()`, `analyze()`; get `build()`, `query()`, `pack()`, `stats()` for free. `build_graph(wipe=True)` drops the vector index it invalidates; `drop_index()` does so on demand. `sync()` rewrites the graph and re-embeds only the nodes whose index text changed |
|
|
423
436
|
|
|
424
437
|
### `kg_utils.snapshots`
|
|
425
438
|
|
|
@@ -674,9 +687,19 @@ Either way, sources are deduplicated by the SHA-256 of their *bytes* rather than
|
|
|
674
687
|
their filename, so the same document arriving twice under different names is
|
|
675
688
|
ingested once.
|
|
676
689
|
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
690
|
+
Each format has one converter:
|
|
691
|
+
|
|
692
|
+
| Formats | Converter | Needs |
|
|
693
|
+
|---|---|---|
|
|
694
|
+
| `.md`, `.markdown`, `.txt`, `.rst` | `PassthroughConverter` (no transformation) | nothing |
|
|
695
|
+
| `.pdf` | `PdfConverter` (`pymupdf4llm`, the same call `doc_kg` makes) | `[ingest]` or `[pdf]` extra |
|
|
696
|
+
| Word, PowerPoint, Excel, OpenDocument, `.rtf`, `.epub`, `.csv` | `AnydocConverter` | `[ingest]` extra |
|
|
697
|
+
| `.tex`, `.latex`, `.html`, `.htm` | `PandocConverter` | the `pandoc` binary on `PATH` |
|
|
698
|
+
|
|
699
|
+
Every file examined is accounted for, including the ones that did not make it.
|
|
700
|
+
No converter performs OCR, so a scanned PDF is recorded as failed rather than
|
|
701
|
+
converted, and a missing `pandoc` is recorded the same way; a corpus should say
|
|
702
|
+
so rather than simply lack them:
|
|
680
703
|
|
|
681
704
|
```python
|
|
682
705
|
for record in pipeline.manifest().problems():
|
|
@@ -686,6 +709,23 @@ for record in pipeline.manifest().problems():
|
|
|
686
709
|
The same records live in `<staging_root>/.ingest/manifest.json`, with the
|
|
687
710
|
converter and version that produced each staged file.
|
|
688
711
|
|
|
712
|
+
Staged files are bare Markdown; their provenance lives in the manifest. For a
|
|
713
|
+
curated import that keeps provenance in the file itself (the `vault_kg` and
|
|
714
|
+
LLM-wiki case), `write_derived` writes one converted document with YAML
|
|
715
|
+
frontmatter (`source_path` relative to an import root, `source_sha256`,
|
|
716
|
+
`source_format`, `converter`, `converter_version`, `ingested_at`, and the
|
|
717
|
+
document's own title, author, date and page count when it has them). It
|
|
718
|
+
refuses to overwrite, so `ingested_at` is written once:
|
|
719
|
+
|
|
720
|
+
```python
|
|
721
|
+
from pathlib import Path
|
|
722
|
+
from kg_utils.ingest import PdfConverter, write_derived
|
|
723
|
+
|
|
724
|
+
src = Path("~/papers/turtlend.pdf").expanduser()
|
|
725
|
+
write_derived(PdfConverter().convert(src), src, Path("RAW/articles/turtlend.md"),
|
|
726
|
+
source_root=src.parent, extra={"kind": "articles"})
|
|
727
|
+
```
|
|
728
|
+
|
|
689
729
|
For the end-to-end pipeline — converters, staging layout, manifest schema,
|
|
690
730
|
re-run semantics, and the `kgrag ingest` command that drives build and
|
|
691
731
|
registration — see
|
|
@@ -748,7 +788,7 @@ If you use kgmodule-utils in research or a project, please cite it:
|
|
|
748
788
|
|
|
749
789
|
**APA**
|
|
750
790
|
|
|
751
|
-
> Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.
|
|
791
|
+
> Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.27.1) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
|
|
752
792
|
|
|
753
793
|
**BibTeX**
|
|
754
794
|
|
|
@@ -756,7 +796,7 @@ If you use kgmodule-utils in research or a project, please cite it:
|
|
|
756
796
|
@software{suchanek_kgmodule_utils,
|
|
757
797
|
author = {Suchanek, Eric G.},
|
|
758
798
|
title = {{kgmodule-utils}: Shared SDK for the KGModule Knowledge-Graph Ecosystem},
|
|
759
|
-
version = {0.
|
|
799
|
+
version = {0.27.1},
|
|
760
800
|
year = {2026},
|
|
761
801
|
publisher = {Flux-Frontiers},
|
|
762
802
|
url = {https://github.com/Flux-Frontiers/KG_utils},
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
[](https://www.python.org/)
|
|
3
3
|
[](https://www.elastic.co/licensing/elastic-license)
|
|
4
|
-
[](https://github.com/Flux-Frontiers/KG_utils/releases)
|
|
5
5
|
[](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml)
|
|
6
6
|
[](https://python-poetry.org/)
|
|
7
7
|
[](https://doi.org/10.5281/zenodo.21284866)
|
|
@@ -26,6 +26,16 @@ Fleet modules use it in two ways. [PyCodeKG](https://github.com/Flux-Frontiers/p
|
|
|
26
26
|
|
|
27
27
|
## Latest News
|
|
28
28
|
|
|
29
|
+
- **0.27.1 (2026-10-19)** -- `[pdf]` and `[ingest]` work again on a fresh
|
|
30
|
+
install. pymupdf4llm 2.0 is an empty placeholder (its code now ships inside
|
|
31
|
+
pymupdf 2.0), so 0.27.0's `pymupdf4llm>=1.28.2` resolved to no PDF support.
|
|
32
|
+
Both extras now declare `pymupdf>=2.0`. No API change.
|
|
33
|
+
- **0.27.0 (2026-10-14)** -- One converter per format, and incremental
|
|
34
|
+
vectors. `PdfConverter` is the fleet's single PDF converter (the same
|
|
35
|
+
`pymupdf4llm` call `doc_kg` makes), `PandocConverter` reads `.tex` and
|
|
36
|
+
`.html`, and `write_derived()` writes a converted document with provenance
|
|
37
|
+
frontmatter. `KGModule.sync()` rewrites the graph but re-embeds only the
|
|
38
|
+
nodes whose text changed. New `[pdf]` extra.
|
|
29
39
|
- **0.26.0 (2026-09-26)** -- Leaves hang as the Knowledge Press web forest hangs
|
|
30
40
|
them. New `hang_leaves()` puts each leaf's stalk on its nearest twig, its
|
|
31
41
|
blade out and lifted, its face to the sky, and returns the frame so any
|
|
@@ -250,7 +260,7 @@ pip install 'kgmodule-utils[viz3d-qt]'
|
|
|
250
260
|
|
|
251
261
|
```toml
|
|
252
262
|
[tool.poetry.dependencies]
|
|
253
|
-
kgmodule-utils = { version = ">=0.
|
|
263
|
+
kgmodule-utils = { version = ">=0.27.0", extras = ["semantic", "synthesis"] }
|
|
254
264
|
```
|
|
255
265
|
|
|
256
266
|
---
|
|
@@ -367,7 +377,7 @@ delta = mgr.diff_snapshots(snaps[-1]["key"], snaps[0]["key"])
|
|
|
367
377
|
|
|
368
378
|
| Class | Description |
|
|
369
379
|
|---|---|
|
|
370
|
-
| `KGModule` | Concrete base — implement `make_extractor()`, `kind()`, `analyze()`; get `build()`, `query()`, `pack()`, `stats()` for free. `build_graph(wipe=True)` drops the vector index it invalidates; `drop_index()` does so on demand |
|
|
380
|
+
| `KGModule` | Concrete base — implement `make_extractor()`, `kind()`, `analyze()`; get `build()`, `query()`, `pack()`, `stats()` for free. `build_graph(wipe=True)` drops the vector index it invalidates; `drop_index()` does so on demand. `sync()` rewrites the graph and re-embeds only the nodes whose index text changed |
|
|
371
381
|
|
|
372
382
|
### `kg_utils.snapshots`
|
|
373
383
|
|
|
@@ -622,9 +632,19 @@ Either way, sources are deduplicated by the SHA-256 of their *bytes* rather than
|
|
|
622
632
|
their filename, so the same document arriving twice under different names is
|
|
623
633
|
ingested once.
|
|
624
634
|
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
635
|
+
Each format has one converter:
|
|
636
|
+
|
|
637
|
+
| Formats | Converter | Needs |
|
|
638
|
+
|---|---|---|
|
|
639
|
+
| `.md`, `.markdown`, `.txt`, `.rst` | `PassthroughConverter` (no transformation) | nothing |
|
|
640
|
+
| `.pdf` | `PdfConverter` (`pymupdf4llm`, the same call `doc_kg` makes) | `[ingest]` or `[pdf]` extra |
|
|
641
|
+
| Word, PowerPoint, Excel, OpenDocument, `.rtf`, `.epub`, `.csv` | `AnydocConverter` | `[ingest]` extra |
|
|
642
|
+
| `.tex`, `.latex`, `.html`, `.htm` | `PandocConverter` | the `pandoc` binary on `PATH` |
|
|
643
|
+
|
|
644
|
+
Every file examined is accounted for, including the ones that did not make it.
|
|
645
|
+
No converter performs OCR, so a scanned PDF is recorded as failed rather than
|
|
646
|
+
converted, and a missing `pandoc` is recorded the same way; a corpus should say
|
|
647
|
+
so rather than simply lack them:
|
|
628
648
|
|
|
629
649
|
```python
|
|
630
650
|
for record in pipeline.manifest().problems():
|
|
@@ -634,6 +654,23 @@ for record in pipeline.manifest().problems():
|
|
|
634
654
|
The same records live in `<staging_root>/.ingest/manifest.json`, with the
|
|
635
655
|
converter and version that produced each staged file.
|
|
636
656
|
|
|
657
|
+
Staged files are bare Markdown; their provenance lives in the manifest. For a
|
|
658
|
+
curated import that keeps provenance in the file itself (the `vault_kg` and
|
|
659
|
+
LLM-wiki case), `write_derived` writes one converted document with YAML
|
|
660
|
+
frontmatter (`source_path` relative to an import root, `source_sha256`,
|
|
661
|
+
`source_format`, `converter`, `converter_version`, `ingested_at`, and the
|
|
662
|
+
document's own title, author, date and page count when it has them). It
|
|
663
|
+
refuses to overwrite, so `ingested_at` is written once:
|
|
664
|
+
|
|
665
|
+
```python
|
|
666
|
+
from pathlib import Path
|
|
667
|
+
from kg_utils.ingest import PdfConverter, write_derived
|
|
668
|
+
|
|
669
|
+
src = Path("~/papers/turtlend.pdf").expanduser()
|
|
670
|
+
write_derived(PdfConverter().convert(src), src, Path("RAW/articles/turtlend.md"),
|
|
671
|
+
source_root=src.parent, extra={"kind": "articles"})
|
|
672
|
+
```
|
|
673
|
+
|
|
637
674
|
For the end-to-end pipeline — converters, staging layout, manifest schema,
|
|
638
675
|
re-run semantics, and the `kgrag ingest` command that drives build and
|
|
639
676
|
registration — see
|
|
@@ -696,7 +733,7 @@ If you use kgmodule-utils in research or a project, please cite it:
|
|
|
696
733
|
|
|
697
734
|
**APA**
|
|
698
735
|
|
|
699
|
-
> Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.
|
|
736
|
+
> Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.27.1) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
|
|
700
737
|
|
|
701
738
|
**BibTeX**
|
|
702
739
|
|
|
@@ -704,7 +741,7 @@ If you use kgmodule-utils in research or a project, please cite it:
|
|
|
704
741
|
@software{suchanek_kgmodule_utils,
|
|
705
742
|
author = {Suchanek, Eric G.},
|
|
706
743
|
title = {{kgmodule-utils}: Shared SDK for the KGModule Knowledge-Graph Ecosystem},
|
|
707
|
-
version = {0.
|
|
744
|
+
version = {0.27.1},
|
|
708
745
|
year = {2026},
|
|
709
746
|
publisher = {Flux-Frontiers},
|
|
710
747
|
url = {https://github.com/Flux-Frontiers/KG_utils},
|
|
@@ -9,7 +9,7 @@ build-backend = "poetry.core.masonry.api"
|
|
|
9
9
|
|
|
10
10
|
[project]
|
|
11
11
|
name = "kgmodule-utils"
|
|
12
|
-
version = "0.
|
|
12
|
+
version = "0.27.1"
|
|
13
13
|
description = "Shared types, graph store, semantic index, and pipeline base for the KGModule SDK"
|
|
14
14
|
readme = "README.md"
|
|
15
15
|
license = "Elastic-2.0"
|
|
@@ -67,8 +67,8 @@ viz3d-qt = [
|
|
|
67
67
|
"PyQt5>=5.15.0",
|
|
68
68
|
# For quiltwright.quilt.resolve_view_cone: the view-cone cap as one shared
|
|
69
69
|
# function rather than a copy of 35.0 per caller. That landed in 0.14.0;
|
|
70
|
-
# the floor
|
|
71
|
-
"quiltwright>=0.
|
|
70
|
+
# the floor tracks the latest release (0.16.0), per the fleet pin rule.
|
|
71
|
+
"quiltwright>=0.16.0",
|
|
72
72
|
]
|
|
73
73
|
# Exact pin: pre-1.0, breaking minors possible. Prebuilt wheels everywhere;
|
|
74
74
|
# also bundled in `semantic` since "auto" defaults to sqlite-vec.
|
|
@@ -80,13 +80,24 @@ sqlite-vec = [
|
|
|
80
80
|
lancedb = [
|
|
81
81
|
"lancedb>=0.19.0",
|
|
82
82
|
]
|
|
83
|
-
# Document ingestion (kg_utils.ingest):
|
|
84
|
-
#
|
|
85
|
-
# optional because .md/.txt/.rst
|
|
86
|
-
#
|
|
87
|
-
#
|
|
83
|
+
# Document ingestion (kg_utils.ingest): pymupdf4llm converts PDF (the same
|
|
84
|
+
# call doc_kg makes, so both paths agree); anydoc converts Word, PowerPoint,
|
|
85
|
+
# Excel, OpenDocument, RTF, EPUB and CSV. Kept optional because .md/.txt/.rst
|
|
86
|
+
# ingest with no dependency at all. pymupdf (AGPL-3.0 or Artifex commercial)
|
|
87
|
+
# is declared, not pymupdf4llm: since 2.0 the pymupdf wheel ships the
|
|
88
|
+
# pymupdf4llm module, and the pymupdf4llm 2.0 distribution is an empty
|
|
89
|
+
# placeholder that installs no code, so declaring it alone resolves to no PDF
|
|
90
|
+
# support at all. .tex/.html need the pandoc binary, which is not a Python
|
|
91
|
+
# dependency. No OCR: scanned PDFs are recorded as failed with a reason,
|
|
92
|
+
# never silently dropped.
|
|
88
93
|
ingest = [
|
|
89
94
|
"firecrawl-anydoc>=0.1.9",
|
|
95
|
+
"pymupdf>=2.0",
|
|
96
|
+
]
|
|
97
|
+
# PDF only: what doc_kg needs to read PDFs through PdfConverter without
|
|
98
|
+
# pulling in anydoc. [ingest] stays the "every format" extra.
|
|
99
|
+
pdf = [
|
|
100
|
+
"pymupdf>=2.0",
|
|
90
101
|
]
|
|
91
102
|
synthesis = [
|
|
92
103
|
"httpx>=0.27.0",
|
|
@@ -116,8 +127,9 @@ optional = true
|
|
|
116
127
|
[tool.poetry.group.dev.dependencies]
|
|
117
128
|
pytest = ">=9.0.3"
|
|
118
129
|
pytest-cov = "^5.0.0"
|
|
119
|
-
# Exercises the real
|
|
130
|
+
# Exercises the real converters in the ingest tests rather than stubs.
|
|
120
131
|
firecrawl-anydoc = ">=0.1.9"
|
|
132
|
+
pymupdf = ">=2.0"
|
|
121
133
|
# Capped below 0.16 deliberately: 0.16 expands the default rules and reformats
|
|
122
134
|
# Python in Markdown blocks. Adopt on purpose; lift the cap in its own change.
|
|
123
135
|
ruff = ">=0.15,<0.16"
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
The acquisition stage that sits *in front of* every KGModule builder. Point it
|
|
4
4
|
at ``.md``, ``.pdf``, ``.docx``, ``.epub``, ``.pptx``, ``.xlsx``, ``.csv`` (and
|
|
5
|
-
the rest of the ``anydoc``
|
|
5
|
+
the rest of the formats ``pymupdf4llm``, ``anydoc`` and ``pandoc`` read); it converts each to Markdown,
|
|
6
6
|
materializes a staging corpus, and records provenance for every file it saw —
|
|
7
7
|
including the ones it could not convert.
|
|
8
8
|
|
|
@@ -36,16 +36,26 @@ from __future__ import annotations
|
|
|
36
36
|
|
|
37
37
|
from kg_utils.ingest.converters import (
|
|
38
38
|
ANYDOC_EXTENSIONS,
|
|
39
|
+
PANDOC_EXTENSIONS,
|
|
39
40
|
PASSTHROUGH_EXTENSIONS,
|
|
41
|
+
PDF_EXTENSIONS,
|
|
40
42
|
AnydocConverter,
|
|
41
43
|
ConversionError,
|
|
42
44
|
ConversionResult,
|
|
43
45
|
Converter,
|
|
46
|
+
PandocConverter,
|
|
44
47
|
PassthroughConverter,
|
|
48
|
+
PdfConverter,
|
|
45
49
|
default_converters,
|
|
46
50
|
resolve_converter,
|
|
47
51
|
supported_extensions,
|
|
48
52
|
)
|
|
53
|
+
from kg_utils.ingest.derived import (
|
|
54
|
+
PROVENANCE_KEYS,
|
|
55
|
+
derived_frontmatter,
|
|
56
|
+
render_derived,
|
|
57
|
+
write_derived,
|
|
58
|
+
)
|
|
49
59
|
from kg_utils.ingest.manifest import (
|
|
50
60
|
INGEST_DIR,
|
|
51
61
|
MANIFEST_NAME,
|
|
@@ -63,7 +73,10 @@ __all__ = [
|
|
|
63
73
|
"INGEST_DIR",
|
|
64
74
|
"MANIFEST_NAME",
|
|
65
75
|
"MANIFEST_VERSION",
|
|
76
|
+
"PANDOC_EXTENSIONS",
|
|
66
77
|
"PASSTHROUGH_EXTENSIONS",
|
|
78
|
+
"PDF_EXTENSIONS",
|
|
79
|
+
"PROVENANCE_KEYS",
|
|
67
80
|
"SKIP_DIRS",
|
|
68
81
|
"AnydocConverter",
|
|
69
82
|
"ConversionError",
|
|
@@ -74,10 +87,15 @@ __all__ = [
|
|
|
74
87
|
"IngestRecord",
|
|
75
88
|
"IngestStats",
|
|
76
89
|
"IngestStatus",
|
|
90
|
+
"PandocConverter",
|
|
77
91
|
"PassthroughConverter",
|
|
92
|
+
"PdfConverter",
|
|
78
93
|
"default_converters",
|
|
94
|
+
"derived_frontmatter",
|
|
79
95
|
"ingest",
|
|
96
|
+
"render_derived",
|
|
80
97
|
"resolve_converter",
|
|
81
98
|
"sha256_file",
|
|
82
99
|
"supported_extensions",
|
|
100
|
+
"write_derived",
|
|
83
101
|
]
|
|
@@ -9,14 +9,25 @@ provenance needed to reproduce that conversion later. Two are shipped:
|
|
|
9
9
|
downstream chunkers keep applying the right structural parse — a ``.txt``
|
|
10
10
|
file must not become ``.md`` or it would suddenly be parsed for headings.
|
|
11
11
|
|
|
12
|
+
``PdfConverter``
|
|
13
|
+
PDF, via ``pymupdf4llm``: the fleet's one PDF authority (sweep item 48).
|
|
14
|
+
The call is the same bare ``pymupdf4llm.to_markdown(path)`` that
|
|
15
|
+
``doc_kg`` makes, so a PDF converts to the same Markdown whether it
|
|
16
|
+
arrives through ``kgrag ingest`` or ``dockg build``.
|
|
17
|
+
|
|
12
18
|
``AnydocConverter``
|
|
13
|
-
|
|
19
|
+
Office and publishing formats, via `anydoc <https://github.com/firecrawl/anydoc>`_
|
|
14
20
|
(PyPI: ``firecrawl-anydoc``) — a Rust converter with Python bindings that
|
|
15
21
|
emits consistent GitHub-Flavored Markdown for Word, PowerPoint, Excel,
|
|
16
|
-
OpenDocument, RTF, EPUB
|
|
22
|
+
OpenDocument, RTF, EPUB and CSV.
|
|
23
|
+
|
|
24
|
+
``PandocConverter``
|
|
25
|
+
LaTeX and HTML, via the ``pandoc`` command, which neither of the above
|
|
26
|
+
reads. Pandoc is a system binary, not a Python dependency: when it is
|
|
27
|
+
absent the file is recorded as failed with that reason.
|
|
17
28
|
|
|
18
|
-
|
|
19
|
-
:class:`ConversionError`; the pipeline records that
|
|
29
|
+
No converter performs OCR. A scanned PDF converts to empty output, which
|
|
30
|
+
raises :class:`ConversionError`; the pipeline records that with a reason
|
|
20
31
|
rather than dropping the file silently.
|
|
21
32
|
|
|
22
33
|
Author: Eric G. Suchanek, PhD
|
|
@@ -24,6 +35,8 @@ Author: Eric G. Suchanek, PhD
|
|
|
24
35
|
|
|
25
36
|
from __future__ import annotations
|
|
26
37
|
|
|
38
|
+
import shutil
|
|
39
|
+
import subprocess
|
|
27
40
|
from dataclasses import dataclass, field
|
|
28
41
|
from pathlib import Path
|
|
29
42
|
from typing import Protocol, runtime_checkable
|
|
@@ -62,10 +75,16 @@ ANYDOC_EXTENSIONS: frozenset[str] = frozenset(
|
|
|
62
75
|
".rtf",
|
|
63
76
|
".epub",
|
|
64
77
|
".csv",
|
|
65
|
-
".pdf",
|
|
66
78
|
}
|
|
67
79
|
)
|
|
68
80
|
|
|
81
|
+
#: Extensions :class:`PdfConverter` handles.
|
|
82
|
+
PDF_EXTENSIONS: frozenset[str] = frozenset({".pdf"})
|
|
83
|
+
|
|
84
|
+
#: Extensions :class:`PandocConverter` handles: formats no Python converter
|
|
85
|
+
#: here reads.
|
|
86
|
+
PANDOC_EXTENSIONS: frozenset[str] = frozenset({".tex", ".latex", ".html", ".htm"})
|
|
87
|
+
|
|
69
88
|
|
|
70
89
|
class ConversionError(RuntimeError):
|
|
71
90
|
"""Raised when a source file cannot be converted to Markdown.
|
|
@@ -269,6 +288,161 @@ class AnydocConverter:
|
|
|
269
288
|
)
|
|
270
289
|
|
|
271
290
|
|
|
291
|
+
class PdfConverter:
|
|
292
|
+
"""Convert PDF to Markdown with ``pymupdf4llm``.
|
|
293
|
+
|
|
294
|
+
The conversion is a bare ``pymupdf4llm.to_markdown(str(path))`` -- no page
|
|
295
|
+
chunks, no anchors -- exactly what ``doc_kg.pdf_reader`` does, so the two
|
|
296
|
+
paths produce byte-identical Markdown. Page anchors are roadmap Phase 2.
|
|
297
|
+
|
|
298
|
+
Title, author, creation date and page count are read from the PDF's own
|
|
299
|
+
metadata into :attr:`ConversionResult.metadata` when present.
|
|
300
|
+
"""
|
|
301
|
+
|
|
302
|
+
name = "pymupdf4llm"
|
|
303
|
+
|
|
304
|
+
def __init__(self, extensions: frozenset[str] | None = None) -> None:
|
|
305
|
+
""":param extensions: Suffixes to accept (default: :data:`PDF_EXTENSIONS`)."""
|
|
306
|
+
self.extensions = extensions if extensions is not None else PDF_EXTENSIONS
|
|
307
|
+
self._version = ""
|
|
308
|
+
|
|
309
|
+
@property
|
|
310
|
+
def version(self) -> str:
|
|
311
|
+
"""Installed ``pymupdf4llm`` version, read without importing it."""
|
|
312
|
+
if not self._version:
|
|
313
|
+
self._version = _installed_version("pymupdf4llm")
|
|
314
|
+
return self._version
|
|
315
|
+
|
|
316
|
+
def handles(self, path: Path) -> bool:
|
|
317
|
+
"""Return ``True`` for suffixes in this converter's extension set."""
|
|
318
|
+
return path.suffix.lower() in self.extensions
|
|
319
|
+
|
|
320
|
+
def convert(self, path: Path) -> ConversionResult:
|
|
321
|
+
"""Convert *path* to Markdown with ``pymupdf4llm``.
|
|
322
|
+
|
|
323
|
+
:param path: PDF to convert.
|
|
324
|
+
:return: Markdown plus provenance, staged as ``.md``.
|
|
325
|
+
:raises ConversionError: If ``pymupdf4llm`` is not installed, cannot
|
|
326
|
+
open or parse the file, or the file has no text layer.
|
|
327
|
+
"""
|
|
328
|
+
try:
|
|
329
|
+
import pymupdf # pylint: disable=import-outside-toplevel
|
|
330
|
+
import pymupdf4llm # pylint: disable=import-outside-toplevel
|
|
331
|
+
except ImportError as exc: # pragma: no cover — depends on install
|
|
332
|
+
raise ConversionError(
|
|
333
|
+
"pymupdf4llm is not installed; install the ingest extra: "
|
|
334
|
+
"pip install 'kgmodule-utils[ingest]'"
|
|
335
|
+
) from exc
|
|
336
|
+
try:
|
|
337
|
+
markdown = pymupdf4llm.to_markdown(str(path))
|
|
338
|
+
except Exception as exc: # noqa: BLE001 — any parser failure is a recorded skip
|
|
339
|
+
raise ConversionError(f"pymupdf4llm could not parse {path.name}: {exc}") from exc
|
|
340
|
+
if not markdown.strip():
|
|
341
|
+
raise ConversionError(
|
|
342
|
+
"converted to empty output (likely a scanned/image-only document needing OCR)"
|
|
343
|
+
)
|
|
344
|
+
|
|
345
|
+
metadata = {"source_format": "pdf"}
|
|
346
|
+
try:
|
|
347
|
+
with pymupdf.open(str(path)) as doc:
|
|
348
|
+
metadata["pages"] = str(doc.page_count)
|
|
349
|
+
info = doc.metadata or {}
|
|
350
|
+
for key, field_name in (
|
|
351
|
+
("title", "title"),
|
|
352
|
+
("author", "author"),
|
|
353
|
+
("creationDate", "created"),
|
|
354
|
+
):
|
|
355
|
+
value = (info.get(key) or "").strip()
|
|
356
|
+
if value:
|
|
357
|
+
metadata[field_name] = value
|
|
358
|
+
except Exception: # noqa: BLE001 — metadata is optional; the text already converted
|
|
359
|
+
pass
|
|
360
|
+
return ConversionResult(
|
|
361
|
+
markdown=markdown,
|
|
362
|
+
converter=self.name,
|
|
363
|
+
converter_version=self.version,
|
|
364
|
+
suffix=".md",
|
|
365
|
+
metadata=metadata,
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
class PandocConverter:
|
|
370
|
+
"""Convert LaTeX and HTML to GitHub-Flavored Markdown with ``pandoc``.
|
|
371
|
+
|
|
372
|
+
``pandoc`` is found on ``PATH`` at conversion time. It is a system
|
|
373
|
+
binary rather than a Python dependency, so a missing ``pandoc`` is a
|
|
374
|
+
recorded failure with an install hint, never a silent skip.
|
|
375
|
+
"""
|
|
376
|
+
|
|
377
|
+
name = "pandoc"
|
|
378
|
+
|
|
379
|
+
def __init__(self, extensions: frozenset[str] | None = None) -> None:
|
|
380
|
+
""":param extensions: Suffixes to accept (default: :data:`PANDOC_EXTENSIONS`)."""
|
|
381
|
+
self.extensions = extensions if extensions is not None else PANDOC_EXTENSIONS
|
|
382
|
+
self._version = ""
|
|
383
|
+
|
|
384
|
+
@property
|
|
385
|
+
def version(self) -> str:
|
|
386
|
+
"""Version reported by ``pandoc --version``, or ``"unknown"``."""
|
|
387
|
+
if not self._version:
|
|
388
|
+
exe = shutil.which("pandoc")
|
|
389
|
+
self._version = "unknown"
|
|
390
|
+
if exe:
|
|
391
|
+
try:
|
|
392
|
+
out = subprocess.run(
|
|
393
|
+
[exe, "--version"], capture_output=True, text=True, check=True, timeout=30
|
|
394
|
+
).stdout
|
|
395
|
+
first = out.splitlines()[0] if out else ""
|
|
396
|
+
self._version = first.split()[-1] if first else "unknown"
|
|
397
|
+
except (OSError, subprocess.SubprocessError):
|
|
398
|
+
pass
|
|
399
|
+
return self._version
|
|
400
|
+
|
|
401
|
+
def handles(self, path: Path) -> bool:
|
|
402
|
+
"""Return ``True`` for suffixes in this converter's extension set."""
|
|
403
|
+
return path.suffix.lower() in self.extensions
|
|
404
|
+
|
|
405
|
+
def convert(self, path: Path) -> ConversionResult:
|
|
406
|
+
"""Convert *path* to Markdown with ``pandoc -t gfm --wrap=none``.
|
|
407
|
+
|
|
408
|
+
:param path: LaTeX or HTML file to convert.
|
|
409
|
+
:return: Markdown plus provenance, staged as ``.md``.
|
|
410
|
+
:raises ConversionError: If ``pandoc`` is not on ``PATH``, fails, or
|
|
411
|
+
produces empty output.
|
|
412
|
+
"""
|
|
413
|
+
exe = shutil.which("pandoc")
|
|
414
|
+
if exe is None:
|
|
415
|
+
raise ConversionError(
|
|
416
|
+
"pandoc is not installed; install it (https://pandoc.org/installing.html, "
|
|
417
|
+
"or `brew install pandoc`) to convert .tex and .html"
|
|
418
|
+
)
|
|
419
|
+
try:
|
|
420
|
+
proc = subprocess.run(
|
|
421
|
+
[exe, str(path), "-t", "gfm", "--wrap=none"],
|
|
422
|
+
capture_output=True,
|
|
423
|
+
text=True,
|
|
424
|
+
check=True,
|
|
425
|
+
timeout=300,
|
|
426
|
+
)
|
|
427
|
+
except subprocess.CalledProcessError as exc:
|
|
428
|
+
detail = (exc.stderr or "").strip().splitlines()
|
|
429
|
+
raise ConversionError(
|
|
430
|
+
f"pandoc failed on {path.name}: {detail[-1] if detail else exc}"
|
|
431
|
+
) from exc
|
|
432
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
433
|
+
raise ConversionError(f"pandoc failed on {path.name}: {exc}") from exc
|
|
434
|
+
if not proc.stdout.strip():
|
|
435
|
+
raise ConversionError("converted to empty output")
|
|
436
|
+
fmt = "latex" if path.suffix.lower() in (".tex", ".latex") else "html"
|
|
437
|
+
return ConversionResult(
|
|
438
|
+
markdown=proc.stdout,
|
|
439
|
+
converter=self.name,
|
|
440
|
+
converter_version=self.version,
|
|
441
|
+
suffix=".md",
|
|
442
|
+
metadata={"source_format": fmt},
|
|
443
|
+
)
|
|
444
|
+
|
|
445
|
+
|
|
272
446
|
def _installed_version(distribution: str) -> str:
|
|
273
447
|
"""Return the installed version of *distribution*, or ``"unknown"``.
|
|
274
448
|
|
|
@@ -289,12 +463,13 @@ def _installed_version(distribution: str) -> str:
|
|
|
289
463
|
def default_converters() -> list[Converter]:
|
|
290
464
|
"""Return the standard converter chain, most specific first.
|
|
291
465
|
|
|
292
|
-
Passthrough is tried
|
|
293
|
-
|
|
466
|
+
Passthrough is tried first so that Markdown and plain text are never
|
|
467
|
+
round-tripped through a converter that would rewrite them. The other
|
|
468
|
+
three handle disjoint extension sets, so their order does not matter.
|
|
294
469
|
|
|
295
470
|
:return: Converters in resolution order.
|
|
296
471
|
"""
|
|
297
|
-
return [PassthroughConverter(), AnydocConverter()]
|
|
472
|
+
return [PassthroughConverter(), PdfConverter(), AnydocConverter(), PandocConverter()]
|
|
298
473
|
|
|
299
474
|
|
|
300
475
|
def resolve_converter(path: Path, converters: list[Converter]) -> Converter | None:
|
|
@@ -313,9 +488,10 @@ def resolve_converter(path: Path, converters: list[Converter]) -> Converter | No
|
|
|
313
488
|
def supported_extensions() -> frozenset[str]:
|
|
314
489
|
"""Return every suffix the default converter chain accepts.
|
|
315
490
|
|
|
316
|
-
Answers without importing
|
|
317
|
-
install.
|
|
491
|
+
Answers without importing any converter library, so it is safe to call on
|
|
492
|
+
a core-only install. ``pandoc``'s formats are included whether or not the
|
|
493
|
+
binary is installed; a missing binary is reported per file.
|
|
318
494
|
|
|
319
495
|
:return: Lower-case suffixes including the leading dot.
|
|
320
496
|
"""
|
|
321
|
-
return PASSTHROUGH_EXTENSIONS | ANYDOC_EXTENSIONS
|
|
497
|
+
return PASSTHROUGH_EXTENSIONS | PDF_EXTENSIONS | ANYDOC_EXTENSIONS | PANDOC_EXTENSIONS
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""kg_utils/ingest/derived.py — Write one converted document with provenance.
|
|
2
|
+
|
|
3
|
+
A *derived note* is the Markdown a converter produced from one source file,
|
|
4
|
+
written to disk with YAML frontmatter that records where it came from:
|
|
5
|
+
|
|
6
|
+
.. code-block:: yaml
|
|
7
|
+
|
|
8
|
+
---
|
|
9
|
+
source_path: "papers/turtlend.pdf"
|
|
10
|
+
source_sha256: "9f2c..."
|
|
11
|
+
source_format: "pdf"
|
|
12
|
+
converter: "pymupdf4llm"
|
|
13
|
+
converter_version: "1.28.2"
|
|
14
|
+
ingested_at: "2026-10-14T21:03:11Z"
|
|
15
|
+
title: "TurtleND"
|
|
16
|
+
pages: "12"
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
The schema is rule 3 of ``kgrag_priv/docs/INGESTION_ROADMAP.md``. Rule 5
|
|
20
|
+
(same bytes and converter version give the same file) holds for everything
|
|
21
|
+
except ``ingested_at``, which is written once: :func:`write_derived` refuses
|
|
22
|
+
to overwrite, so a derived note is never regenerated in place.
|
|
23
|
+
|
|
24
|
+
This is the writer for curated imports such as ``vaultkg add``. The staging
|
|
25
|
+
pipeline (:class:`~kg_utils.ingest.pipeline.IngestPipeline`) does *not* use
|
|
26
|
+
it yet: DocKG cannot parse frontmatter until roadmap Phase 1, so staged files
|
|
27
|
+
stay bare Markdown with their provenance in the ingest manifest.
|
|
28
|
+
|
|
29
|
+
Author: Eric G. Suchanek, PhD
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import json
|
|
35
|
+
from collections.abc import Mapping
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
|
|
38
|
+
from kg_utils.ingest.converters import ConversionResult
|
|
39
|
+
from kg_utils.ingest.manifest import sha256_file, utc_now
|
|
40
|
+
|
|
41
|
+
#: Frontmatter keys written first, in this order; converter metadata and
|
|
42
|
+
#: caller extras follow, sorted.
|
|
43
|
+
PROVENANCE_KEYS: tuple[str, ...] = (
|
|
44
|
+
"source_path",
|
|
45
|
+
"source_sha256",
|
|
46
|
+
"source_format",
|
|
47
|
+
"converter",
|
|
48
|
+
"converter_version",
|
|
49
|
+
"ingested_at",
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def derived_frontmatter(
|
|
54
|
+
result: ConversionResult,
|
|
55
|
+
source: Path,
|
|
56
|
+
*,
|
|
57
|
+
source_root: Path | None = None,
|
|
58
|
+
ingested_at: str | None = None,
|
|
59
|
+
extra: Mapping[str, str] | None = None,
|
|
60
|
+
) -> dict[str, str]:
|
|
61
|
+
"""Build the provenance mapping for one derived note.
|
|
62
|
+
|
|
63
|
+
:param result: The conversion to describe.
|
|
64
|
+
:param source: The source file that was converted.
|
|
65
|
+
:param source_root: Directory ``source_path`` is recorded relative to.
|
|
66
|
+
When ``None``, or when *source* is not under it, only the file name is
|
|
67
|
+
recorded, so no absolute home-directory path reaches a tracked file.
|
|
68
|
+
:param ingested_at: ISO timestamp to record; defaults to now (UTC).
|
|
69
|
+
:param extra: Caller fields (for example ``kind``). They may not replace a
|
|
70
|
+
provenance key.
|
|
71
|
+
:return: Ordered mapping of frontmatter keys to string values.
|
|
72
|
+
:raises ValueError: If *extra* names a provenance key.
|
|
73
|
+
"""
|
|
74
|
+
source = Path(source)
|
|
75
|
+
rel = source.name
|
|
76
|
+
if source_root is not None:
|
|
77
|
+
try:
|
|
78
|
+
rel = source.resolve().relative_to(Path(source_root).resolve()).as_posix()
|
|
79
|
+
except ValueError:
|
|
80
|
+
rel = source.name
|
|
81
|
+
fields: dict[str, str] = {
|
|
82
|
+
"source_path": rel,
|
|
83
|
+
"source_sha256": sha256_file(source),
|
|
84
|
+
"source_format": result.metadata.get("source_format") or source.suffix.lower().lstrip("."),
|
|
85
|
+
"converter": result.converter,
|
|
86
|
+
"converter_version": result.converter_version,
|
|
87
|
+
"ingested_at": ingested_at or utc_now(),
|
|
88
|
+
}
|
|
89
|
+
clash = sorted(set(extra or {}) & set(PROVENANCE_KEYS))
|
|
90
|
+
if clash:
|
|
91
|
+
raise ValueError(f"extra may not replace provenance keys: {clash}")
|
|
92
|
+
tail = {k: v for k, v in result.metadata.items() if k not in fields}
|
|
93
|
+
tail.update(extra or {})
|
|
94
|
+
for key in sorted(tail):
|
|
95
|
+
fields[key] = str(tail[key])
|
|
96
|
+
return fields
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def render_derived(fields: Mapping[str, str], markdown: str) -> str:
|
|
100
|
+
"""Render frontmatter plus body as one document.
|
|
101
|
+
|
|
102
|
+
Values are written as JSON strings, which are valid double-quoted YAML
|
|
103
|
+
scalars, so a title containing ``: `` or quotes cannot break the block.
|
|
104
|
+
|
|
105
|
+
:param fields: Frontmatter keys and values.
|
|
106
|
+
:param markdown: Document body.
|
|
107
|
+
:return: The complete file text, ending in one newline.
|
|
108
|
+
"""
|
|
109
|
+
lines = ["---"]
|
|
110
|
+
lines += [
|
|
111
|
+
f"{key}: {json.dumps(str(value), ensure_ascii=False)}" for key, value in fields.items()
|
|
112
|
+
]
|
|
113
|
+
lines += ["---", ""]
|
|
114
|
+
return "\n".join(lines) + markdown.rstrip("\n") + "\n"
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def write_derived(
|
|
118
|
+
result: ConversionResult,
|
|
119
|
+
source: Path,
|
|
120
|
+
dest: Path,
|
|
121
|
+
*,
|
|
122
|
+
source_root: Path | None = None,
|
|
123
|
+
ingested_at: str | None = None,
|
|
124
|
+
extra: Mapping[str, str] | None = None,
|
|
125
|
+
) -> Path:
|
|
126
|
+
"""Write *result* to *dest* as a derived note with provenance frontmatter.
|
|
127
|
+
|
|
128
|
+
:param result: The conversion to write.
|
|
129
|
+
:param source: The source file that was converted.
|
|
130
|
+
:param dest: Where to write the note. Its parent is created if needed.
|
|
131
|
+
:param source_root: See :func:`derived_frontmatter`.
|
|
132
|
+
:param ingested_at: See :func:`derived_frontmatter`.
|
|
133
|
+
:param extra: See :func:`derived_frontmatter`.
|
|
134
|
+
:return: *dest*.
|
|
135
|
+
:raises FileExistsError: If *dest* already exists. A derived note is
|
|
136
|
+
written once; deciding what an existing file means (an identical
|
|
137
|
+
re-import, or a name collision) is the caller's job.
|
|
138
|
+
"""
|
|
139
|
+
dest = Path(dest)
|
|
140
|
+
if dest.exists():
|
|
141
|
+
raise FileExistsError(f"derived note already exists: {dest}")
|
|
142
|
+
fields = derived_frontmatter(
|
|
143
|
+
result, source, source_root=source_root, ingested_at=ingested_at, extra=extra
|
|
144
|
+
)
|
|
145
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
146
|
+
with dest.open("x", encoding="utf-8") as fh:
|
|
147
|
+
fh.write(render_derived(fields, result.markdown))
|
|
148
|
+
return dest
|
|
@@ -499,14 +499,7 @@ class KGModule(ABC):
|
|
|
499
499
|
index (:meth:`drop_index`), which no longer matches the new graph.
|
|
500
500
|
:return: :class:`~kg_utils.types.BuildStats` (``indexed_rows`` will be ``None``).
|
|
501
501
|
"""
|
|
502
|
-
|
|
503
|
-
node_specs: list[NodeSpec] = []
|
|
504
|
-
edge_specs: list[EdgeSpec] = []
|
|
505
|
-
for item in extractor.extract():
|
|
506
|
-
if isinstance(item, NodeSpec):
|
|
507
|
-
node_specs.append(item)
|
|
508
|
-
else:
|
|
509
|
-
edge_specs.append(item)
|
|
502
|
+
node_specs, edge_specs = self._extract_specs()
|
|
510
503
|
|
|
511
504
|
if wipe:
|
|
512
505
|
# A wiped graph invalidates every vector: an index left from the
|
|
@@ -527,6 +520,55 @@ class KGModule(ABC):
|
|
|
527
520
|
edge_counts=s["edge_counts"],
|
|
528
521
|
)
|
|
529
522
|
|
|
523
|
+
def _extract_specs(self) -> tuple[list[NodeSpec], list[EdgeSpec]]:
|
|
524
|
+
"""Drain the extractor into node and edge specs.
|
|
525
|
+
|
|
526
|
+
:return: ``(node_specs, edge_specs)`` in extraction order.
|
|
527
|
+
"""
|
|
528
|
+
node_specs: list[NodeSpec] = []
|
|
529
|
+
edge_specs: list[EdgeSpec] = []
|
|
530
|
+
for item in self.make_extractor().extract():
|
|
531
|
+
if isinstance(item, NodeSpec):
|
|
532
|
+
node_specs.append(item)
|
|
533
|
+
else:
|
|
534
|
+
edge_specs.append(item)
|
|
535
|
+
return node_specs, edge_specs
|
|
536
|
+
|
|
537
|
+
def sync(self) -> BuildStats:
|
|
538
|
+
"""Rebuild the graph, then update the vector index incrementally.
|
|
539
|
+
|
|
540
|
+
The graph is always rewritten in full (``wipe=True``): extraction is
|
|
541
|
+
cheap and a full rewrite keeps "same source, same graph" exact. Only
|
|
542
|
+
the vectors are incremental: :meth:`SemanticIndex.sync` embeds the
|
|
543
|
+
nodes whose index text changed, deletes the vectors of nodes that are
|
|
544
|
+
gone, and leaves the rest. Unlike :meth:`build_graph` with
|
|
545
|
+
``wipe=True``, this does not drop the index first.
|
|
546
|
+
|
|
547
|
+
The first sync of a store embeds every node once, because no
|
|
548
|
+
per-node hashes are recorded yet.
|
|
549
|
+
|
|
550
|
+
:return: :class:`~kg_utils.specs.BuildStats` with ``indexed_rows``,
|
|
551
|
+
``index_dim``, ``embedded_rows`` and ``deleted_rows`` set.
|
|
552
|
+
"""
|
|
553
|
+
node_specs, edge_specs = self._extract_specs()
|
|
554
|
+
self.store.write(node_specs, edge_specs, wipe=True)
|
|
555
|
+
self._post_build_hook(self.store)
|
|
556
|
+
|
|
557
|
+
idx = self.index.sync(self.store, signature=f"{self.model_name}:{self.embedder.dim}")
|
|
558
|
+
s = self.store.stats()
|
|
559
|
+
return BuildStats(
|
|
560
|
+
repo_root=str(self.repo_root),
|
|
561
|
+
db_path=str(self.db_path),
|
|
562
|
+
total_nodes=s["total_nodes"],
|
|
563
|
+
total_edges=s["total_edges"],
|
|
564
|
+
node_counts=s["node_counts"],
|
|
565
|
+
edge_counts=s["edge_counts"],
|
|
566
|
+
indexed_rows=idx["indexed_rows"],
|
|
567
|
+
index_dim=idx["dim"],
|
|
568
|
+
embedded_rows=idx["embedded_rows"],
|
|
569
|
+
deleted_rows=idx["deleted_rows"],
|
|
570
|
+
)
|
|
571
|
+
|
|
530
572
|
def drop_index(self) -> list[Path]:
|
|
531
573
|
"""Delete the vector index, closing it first.
|
|
532
574
|
|
|
@@ -27,6 +27,7 @@ constructs it any more — see :meth:`SemanticIndex._get_backend`.
|
|
|
27
27
|
|
|
28
28
|
from __future__ import annotations
|
|
29
29
|
|
|
30
|
+
import hashlib
|
|
30
31
|
import logging
|
|
31
32
|
import os
|
|
32
33
|
import re
|
|
@@ -200,20 +201,7 @@ class SemanticIndex:
|
|
|
200
201
|
for i in range(0, len(nodes), batch_size):
|
|
201
202
|
chunk = nodes[i : i + batch_size]
|
|
202
203
|
texts = [_build_index_text(n) for n in chunk]
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
rows = [
|
|
206
|
-
{
|
|
207
|
-
"id": n["id"],
|
|
208
|
-
"kind": n["kind"],
|
|
209
|
-
"name": n["name"],
|
|
210
|
-
"qualname": n["qualname"] or "",
|
|
211
|
-
"module_path": n["module_path"] or "",
|
|
212
|
-
"text": text,
|
|
213
|
-
"vector": vec,
|
|
214
|
-
}
|
|
215
|
-
for n, text, vec in zip(chunk, texts, vecs)
|
|
216
|
-
]
|
|
204
|
+
rows = self._rows(chunk, texts)
|
|
217
205
|
indexed += backend.upsert(rows, batch_size=len(rows))
|
|
218
206
|
|
|
219
207
|
# No-op unless the backend is a LanceDBBackend with ANN enabled.
|
|
@@ -228,6 +216,115 @@ class SemanticIndex:
|
|
|
228
216
|
"kinds": list(self.index_kinds),
|
|
229
217
|
}
|
|
230
218
|
|
|
219
|
+
def sync(
|
|
220
|
+
self,
|
|
221
|
+
store: GraphStore,
|
|
222
|
+
*,
|
|
223
|
+
signature: str | None = None,
|
|
224
|
+
batch_size: int = 256,
|
|
225
|
+
quiet: bool = True,
|
|
226
|
+
) -> dict[str, Any]:
|
|
227
|
+
"""Bring the vector index in line with *store*, embedding only changes.
|
|
228
|
+
|
|
229
|
+
Each node's index text is hashed. A node is embedded when it is new or
|
|
230
|
+
its hash differs from the one recorded at the last sync; vectors for
|
|
231
|
+
nodes no longer in *store* are deleted; everything else is left as it
|
|
232
|
+
is. The hashes and the embedder *signature* live in sidecar tables of
|
|
233
|
+
the sqlite-vec store, separate from ``vec_meta``.
|
|
234
|
+
|
|
235
|
+
The first sync after a full :meth:`build` (or on a store that has
|
|
236
|
+
never been synced) embeds every node once, because no hashes are
|
|
237
|
+
recorded yet; later syncs are incremental. A changed *signature*
|
|
238
|
+
(another model or dimension) wipes the store and embeds everything.
|
|
239
|
+
|
|
240
|
+
Only :class:`SqliteVecBackend` keeps sync state. With any other
|
|
241
|
+
backend this falls back to ``build(store, wipe=True)``.
|
|
242
|
+
|
|
243
|
+
A vector embedded alone can differ in its last bits from the same text
|
|
244
|
+
embedded in a batch, so ties in a ranking may order differently than
|
|
245
|
+
after a full build. The graph itself is unaffected.
|
|
246
|
+
|
|
247
|
+
:param store: Authoritative :class:`~kg_utils.store.GraphStore`.
|
|
248
|
+
:param signature: Embedder identity; defaults to the embedder's class
|
|
249
|
+
name, its ``model_name`` if it has one, and its dimension.
|
|
250
|
+
:param batch_size: Number of nodes to embed per batch.
|
|
251
|
+
:param quiet: Suppress progress output during ingestion.
|
|
252
|
+
:return: The :meth:`build` stats plus ``embedded_rows`` and
|
|
253
|
+
``deleted_rows``.
|
|
254
|
+
"""
|
|
255
|
+
backend = self._get_backend()
|
|
256
|
+
if not isinstance(backend, SqliteVecBackend):
|
|
257
|
+
stats = self.build(store, wipe=True, batch_size=batch_size, quiet=quiet)
|
|
258
|
+
return {**stats, "embedded_rows": stats["indexed_rows"], "deleted_rows": 0}
|
|
259
|
+
if quiet:
|
|
260
|
+
suppress_ingestion_logging()
|
|
261
|
+
|
|
262
|
+
if signature is None:
|
|
263
|
+
signature = ":".join(
|
|
264
|
+
(
|
|
265
|
+
type(self.embedder).__name__,
|
|
266
|
+
str(getattr(self.embedder, "model_name", "")),
|
|
267
|
+
str(self.embedder.dim),
|
|
268
|
+
)
|
|
269
|
+
)
|
|
270
|
+
nodes = self._read_nodes(store)
|
|
271
|
+
texts = {n["id"]: _build_index_text(n) for n in nodes}
|
|
272
|
+
shas = {nid: hashlib.sha256(t.encode("utf-8")).hexdigest() for nid, t in texts.items()}
|
|
273
|
+
|
|
274
|
+
backend.open(wipe=False)
|
|
275
|
+
stored_signature, stored = backend.load_sync_state()
|
|
276
|
+
if stored_signature and stored_signature != signature:
|
|
277
|
+
backend.close()
|
|
278
|
+
backend.open(wipe=True)
|
|
279
|
+
stored = {}
|
|
280
|
+
present = backend.existing_ids()
|
|
281
|
+
|
|
282
|
+
gone = sorted(present - shas.keys())
|
|
283
|
+
deleted = backend.delete_ids(gone) if gone else 0
|
|
284
|
+
todo = [n for n in nodes if n["id"] not in present or stored.get(n["id"]) != shas[n["id"]]]
|
|
285
|
+
|
|
286
|
+
embedded = 0
|
|
287
|
+
for i in range(0, len(todo), batch_size):
|
|
288
|
+
chunk = todo[i : i + batch_size]
|
|
289
|
+
rows = self._rows(chunk, [texts[n["id"]] for n in chunk])
|
|
290
|
+
embedded += backend.upsert(rows, batch_size=len(rows))
|
|
291
|
+
backend.save_sync_state(
|
|
292
|
+
signature,
|
|
293
|
+
{n["id"]: shas[n["id"]] for n in todo},
|
|
294
|
+
[*gone, *(stored.keys() - shas.keys())],
|
|
295
|
+
)
|
|
296
|
+
|
|
297
|
+
return {
|
|
298
|
+
"indexed_rows": backend.count(),
|
|
299
|
+
"embedded_rows": embedded,
|
|
300
|
+
"deleted_rows": deleted,
|
|
301
|
+
"dim": self.embedder.dim,
|
|
302
|
+
"table": self.table_name,
|
|
303
|
+
"vectors_path": str(self.vectors_path),
|
|
304
|
+
"kinds": list(self.index_kinds),
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
def _rows(self, nodes: Sequence[dict[str, Any]], texts: Sequence[str]) -> list[dict[str, Any]]:
|
|
308
|
+
"""Embed *texts* and pair each vector with its node's metadata row.
|
|
309
|
+
|
|
310
|
+
:param nodes: Node dicts from the graph store.
|
|
311
|
+
:param texts: Index text for each node, in the same order.
|
|
312
|
+
:return: Backend rows (``id``, the code-KG meta columns, ``text``, ``vector``).
|
|
313
|
+
"""
|
|
314
|
+
vecs = self.embedder.embed_texts(list(texts))
|
|
315
|
+
return [
|
|
316
|
+
{
|
|
317
|
+
"id": n["id"],
|
|
318
|
+
"kind": n["kind"],
|
|
319
|
+
"name": n["name"],
|
|
320
|
+
"qualname": n["qualname"] or "",
|
|
321
|
+
"module_path": n["module_path"] or "",
|
|
322
|
+
"text": text,
|
|
323
|
+
"vector": vec,
|
|
324
|
+
}
|
|
325
|
+
for n, text, vec in zip(nodes, texts, vecs)
|
|
326
|
+
]
|
|
327
|
+
|
|
231
328
|
# ------------------------------------------------------------------
|
|
232
329
|
# Search
|
|
233
330
|
# ------------------------------------------------------------------
|
|
@@ -64,6 +64,10 @@ class BuildStats:
|
|
|
64
64
|
:param edge_counts: Edge counts broken down by relation.
|
|
65
65
|
:param indexed_rows: Number of nodes embedded into LanceDB (``None`` if not built).
|
|
66
66
|
:param index_dim: Embedding dimension (``None`` if not built).
|
|
67
|
+
:param embedded_rows: Vectors embedded by :meth:`KGModule.sync` (``None``
|
|
68
|
+
for a full build, which embeds every row).
|
|
69
|
+
:param deleted_rows: Vectors deleted by :meth:`KGModule.sync` (``None``
|
|
70
|
+
for a full build).
|
|
67
71
|
"""
|
|
68
72
|
|
|
69
73
|
repo_root: str
|
|
@@ -74,6 +78,8 @@ class BuildStats:
|
|
|
74
78
|
edge_counts: dict[str, int]
|
|
75
79
|
indexed_rows: int | None = None
|
|
76
80
|
index_dim: int | None = None
|
|
81
|
+
embedded_rows: int | None = None
|
|
82
|
+
deleted_rows: int | None = None
|
|
77
83
|
|
|
78
84
|
def to_dict(self) -> dict[str, Any]:
|
|
79
85
|
"""Serialise the stats to a plain dictionary."""
|
|
@@ -86,6 +92,13 @@ class BuildStats:
|
|
|
86
92
|
"edge_counts": self.edge_counts,
|
|
87
93
|
"indexed_rows": self.indexed_rows,
|
|
88
94
|
"index_dim": self.index_dim,
|
|
95
|
+
# Sync-only fields appear only after a sync, so a full build's
|
|
96
|
+
# dict is unchanged for callers that compare it whole.
|
|
97
|
+
**(
|
|
98
|
+
{"embedded_rows": self.embedded_rows, "deleted_rows": self.deleted_rows}
|
|
99
|
+
if self.embedded_rows is not None
|
|
100
|
+
else {}
|
|
101
|
+
),
|
|
89
102
|
}
|
|
90
103
|
|
|
91
104
|
def __str__(self) -> str:
|
|
@@ -97,6 +110,10 @@ class BuildStats:
|
|
|
97
110
|
]
|
|
98
111
|
if self.indexed_rows is not None:
|
|
99
112
|
lines.append(f"indexed : {self.indexed_rows} vectors dim={self.index_dim}")
|
|
113
|
+
if self.embedded_rows is not None:
|
|
114
|
+
lines.append(
|
|
115
|
+
f"synced : {self.embedded_rows} embedded, {self.deleted_rows} deleted"
|
|
116
|
+
)
|
|
100
117
|
return "\n".join(lines)
|
|
101
118
|
|
|
102
119
|
|
|
@@ -456,6 +456,9 @@ class SqliteVecBackend:
|
|
|
456
456
|
if wipe:
|
|
457
457
|
conn.execute("DROP TABLE IF EXISTS vec_nodes")
|
|
458
458
|
conn.execute("DROP TABLE IF EXISTS vec_meta")
|
|
459
|
+
# sync() state describes the vectors just dropped.
|
|
460
|
+
conn.execute(f"DROP TABLE IF EXISTS {SYNC_HASH_TABLE}")
|
|
461
|
+
conn.execute(f"DROP TABLE IF EXISTS {SYNC_INFO_TABLE}")
|
|
459
462
|
existed = bool(
|
|
460
463
|
conn.execute(
|
|
461
464
|
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='vec_meta'"
|
|
@@ -577,6 +580,68 @@ class SqliteVecBackend:
|
|
|
577
580
|
removed += self._delete_ids_conn(conn, ids[start : start + 1000])
|
|
578
581
|
return removed
|
|
579
582
|
|
|
583
|
+
# -- sync state -----------------------------------------------------
|
|
584
|
+
#
|
|
585
|
+
# Two sidecar tables that only :meth:`SemanticIndex.sync` reads and
|
|
586
|
+
# writes. They are separate from ``vec_meta`` on purpose: every consumer
|
|
587
|
+
# shares that table and some read it directly, so a full build stays
|
|
588
|
+
# exactly what it was. A store that has never been synced has neither.
|
|
589
|
+
|
|
590
|
+
def load_sync_state(self) -> tuple[str, dict[str, str]]:
|
|
591
|
+
"""Return the stored embedder signature and per-id index-text hashes.
|
|
592
|
+
|
|
593
|
+
:return: ``(signature, {node_id: text_sha256})``; ``("", {})`` when
|
|
594
|
+
the store has never been synced.
|
|
595
|
+
"""
|
|
596
|
+
conn = self._c()
|
|
597
|
+
have = {
|
|
598
|
+
r[0]
|
|
599
|
+
for r in conn.execute(
|
|
600
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name IN (?, ?)",
|
|
601
|
+
(SYNC_HASH_TABLE, SYNC_INFO_TABLE),
|
|
602
|
+
).fetchall()
|
|
603
|
+
}
|
|
604
|
+
if have != {SYNC_HASH_TABLE, SYNC_INFO_TABLE}:
|
|
605
|
+
return "", {}
|
|
606
|
+
row = conn.execute(
|
|
607
|
+
f"SELECT value FROM {SYNC_INFO_TABLE} WHERE key = 'signature'"
|
|
608
|
+
).fetchone()
|
|
609
|
+
hashes = dict(conn.execute(f"SELECT id, sha FROM {SYNC_HASH_TABLE}").fetchall())
|
|
610
|
+
return (row[0] if row else ""), hashes
|
|
611
|
+
|
|
612
|
+
def save_sync_state(
|
|
613
|
+
self, signature: str, written: dict[str, str], removed: Iterable[str]
|
|
614
|
+
) -> None:
|
|
615
|
+
"""Record the embedder signature and update the per-id hashes.
|
|
616
|
+
|
|
617
|
+
:param signature: Embedder identity the stored vectors were made with.
|
|
618
|
+
:param written: ``{node_id: text_sha256}`` for ids embedded this run.
|
|
619
|
+
:param removed: Ids whose vectors were deleted this run.
|
|
620
|
+
"""
|
|
621
|
+
conn = self._c()
|
|
622
|
+
with conn:
|
|
623
|
+
conn.execute(
|
|
624
|
+
f"CREATE TABLE IF NOT EXISTS {SYNC_HASH_TABLE}(id TEXT PRIMARY KEY, sha TEXT NOT NULL)"
|
|
625
|
+
)
|
|
626
|
+
conn.execute(
|
|
627
|
+
f"CREATE TABLE IF NOT EXISTS {SYNC_INFO_TABLE}(key TEXT PRIMARY KEY, value TEXT NOT NULL)"
|
|
628
|
+
)
|
|
629
|
+
conn.execute(
|
|
630
|
+
f"INSERT OR REPLACE INTO {SYNC_INFO_TABLE}(key, value) VALUES ('signature', ?)",
|
|
631
|
+
(signature,),
|
|
632
|
+
)
|
|
633
|
+
gone = [i for i in removed if i]
|
|
634
|
+
for start in range(0, len(gone), 1000):
|
|
635
|
+
chunk = gone[start : start + 1000]
|
|
636
|
+
conn.execute(
|
|
637
|
+
f"DELETE FROM {SYNC_HASH_TABLE} WHERE id IN ({', '.join('?' for _ in chunk)})",
|
|
638
|
+
chunk,
|
|
639
|
+
)
|
|
640
|
+
conn.executemany(
|
|
641
|
+
f"INSERT OR REPLACE INTO {SYNC_HASH_TABLE}(id, sha) VALUES (?, ?)",
|
|
642
|
+
list(written.items()),
|
|
643
|
+
)
|
|
644
|
+
|
|
580
645
|
# -- reads ----------------------------------------------------------
|
|
581
646
|
|
|
582
647
|
def existing_ids(self) -> set[str]:
|
|
@@ -622,6 +687,12 @@ class SqliteVecBackend:
|
|
|
622
687
|
return out
|
|
623
688
|
|
|
624
689
|
|
|
690
|
+
#: Sidecar tables holding :meth:`SemanticIndex.sync` state (see
|
|
691
|
+
#: :meth:`SqliteVecBackend.load_sync_state`).
|
|
692
|
+
SYNC_HASH_TABLE = "vec_sync_hash"
|
|
693
|
+
SYNC_INFO_TABLE = "vec_sync_info"
|
|
694
|
+
|
|
695
|
+
|
|
625
696
|
def _escape(s: str) -> str:
|
|
626
697
|
return s.replace("'", "''")
|
|
627
698
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|