kgmodule-utils 0.26.0__tar.gz → 0.27.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/PKG-INFO +50 -10
  2. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/README.md +45 -8
  3. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/pyproject.toml +21 -9
  4. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/__init__.py +1 -1
  5. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/ingest/__init__.py +19 -1
  6. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/ingest/converters.py +187 -11
  7. kgmodule_utils-0.27.1/src/kg_utils/ingest/derived.py +148 -0
  8. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/pipeline.py +50 -8
  9. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/semantic.py +111 -14
  10. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/specs.py +17 -0
  11. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/vector_backend.py +71 -0
  12. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/LICENSE +0 -0
  13. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/analysis/__init__.py +0 -0
  14. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/analysis/scores.py +0 -0
  15. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/corpus_embedder.py +0 -0
  16. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/embed.py +0 -0
  17. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/embedder.py +0 -0
  18. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/extractor.py +0 -0
  19. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/ingest/manifest.py +0 -0
  20. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/ingest/pipeline.py +0 -0
  21. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/module.py +0 -0
  22. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/py.typed +0 -0
  23. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/retrieval/__init__.py +0 -0
  24. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/retrieval/hits.py +0 -0
  25. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/snapshots/__init__.py +0 -0
  26. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/snapshots/manager.py +0 -0
  27. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/snapshots/models.py +0 -0
  28. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/store.py +0 -0
  29. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/__init__.py +0 -0
  30. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/_config.py +0 -0
  31. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/_image.py +0 -0
  32. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/_text.py +0 -0
  33. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/synthesis/factory.py +0 -0
  34. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/temporal.py +0 -0
  35. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/validation.py +0 -0
  36. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz/__init__.py +0 -0
  37. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz/graph_html.py +0 -0
  38. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz/theme.py +0 -0
  39. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz/tooltip.py +0 -0
  40. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/__init__.py +0 -0
  41. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/layout.py +0 -0
  42. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/organic.py +0 -0
  43. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/qt.py +0 -0
  44. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/viz3d/species.py +0 -0
  45. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/worker/__init__.py +0 -0
  46. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/worker/client.py +0 -0
  47. {kgmodule_utils-0.26.0 → kgmodule_utils-0.27.1}/src/kg_utils/worker/ops.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: kgmodule-utils
3
- Version: 0.26.0
3
+ Version: 0.27.1
4
4
  Summary: Shared types, graph store, semantic index, and pipeline base for the KGModule SDK
5
5
  License-Expression: Elastic-2.0
6
6
  License-File: LICENSE
@@ -15,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.12
15
15
  Classifier: Programming Language :: Python :: 3.13
16
16
  Provides-Extra: ingest
17
17
  Provides-Extra: lancedb
18
+ Provides-Extra: pdf
18
19
  Provides-Extra: semantic
19
20
  Provides-Extra: sqlite-vec
20
21
  Provides-Extra: synthesis
@@ -37,10 +38,12 @@ Requires-Dist: openai (>=1.30.0) ; extra == "synthesis"
37
38
  Requires-Dist: openai (>=1.30.0) ; extra == "synthesis-mflux"
38
39
  Requires-Dist: pillow (>=10.0.0) ; extra == "synthesis"
39
40
  Requires-Dist: pillow (>=10.0.0) ; extra == "synthesis-mflux"
41
+ Requires-Dist: pymupdf (>=2.0) ; extra == "ingest"
42
+ Requires-Dist: pymupdf (>=2.0) ; extra == "pdf"
40
43
  Requires-Dist: pyvis (>=0.3.2) ; extra == "viz"
41
44
  Requires-Dist: pyvista (>=0.44.0) ; extra == "viz3d-qt"
42
45
  Requires-Dist: pyvista (>=0.44.0) ; extra == "viz3d-render"
43
- Requires-Dist: quiltwright (>=0.15.0) ; extra == "viz3d-qt"
46
+ Requires-Dist: quiltwright (>=0.16.0) ; extra == "viz3d-qt"
44
47
  Requires-Dist: rich (>=13.0.0,<15.0.0) ; extra == "semantic"
45
48
  Requires-Dist: sentence-transformers (>=5.4.1) ; extra == "semantic"
46
49
  Requires-Dist: sqlite-vec (==0.1.9) ; extra == "semantic"
@@ -53,7 +56,7 @@ Description-Content-Type: text/markdown
53
56
 
54
57
  [![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13-blue.svg)](https://www.python.org/)
55
58
  [![License: Elastic-2.0](https://img.shields.io/badge/License-Elastic%202.0-blue.svg)](https://www.elastic.co/licensing/elastic-license)
56
- [![Version](https://img.shields.io/badge/version-0.26.0-blue.svg)](https://github.com/Flux-Frontiers/KG_utils/releases)
59
+ [![Version](https://img.shields.io/badge/version-0.27.1-blue.svg)](https://github.com/Flux-Frontiers/KG_utils/releases)
57
60
  [![CI](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml/badge.svg)](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml)
58
61
  [![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
59
62
  [![DOI](https://img.shields.io/badge/DOI-10.5281%2Fzenodo.21284866-blue.svg)](https://doi.org/10.5281/zenodo.21284866)
@@ -78,6 +81,16 @@ Fleet modules use it in two ways. [PyCodeKG](https://github.com/Flux-Frontiers/p
78
81
 
79
82
  ## Latest News
80
83
 
84
+ - **0.27.1 (2026-10-19)** -- `[pdf]` and `[ingest]` work again on a fresh
85
+ install. pymupdf4llm 2.0 is an empty placeholder (its code now ships inside
86
+ pymupdf 2.0), so 0.27.0's `pymupdf4llm>=1.28.2` resolved to no PDF support.
87
+ Both extras now declare `pymupdf>=2.0`. No API change.
88
+ - **0.27.0 (2026-10-14)** -- One converter per format, and incremental
89
+ vectors. `PdfConverter` is the fleet's single PDF converter (the same
90
+ `pymupdf4llm` call `doc_kg` makes), `PandocConverter` reads `.tex` and
91
+ `.html`, and `write_derived()` writes a converted document with provenance
92
+ frontmatter. `KGModule.sync()` rewrites the graph but re-embeds only the
93
+ nodes whose text changed. New `[pdf]` extra.
81
94
  - **0.26.0 (2026-09-26)** -- Leaves hang as the Knowledge Press web forest hangs
82
95
  them. New `hang_leaves()` puts each leaf's stalk on its nearest twig, its
83
96
  blade out and lifted, its face to the sky, and returns the frame so any
@@ -302,7 +315,7 @@ pip install 'kgmodule-utils[viz3d-qt]'
302
315
 
303
316
  ```toml
304
317
  [tool.poetry.dependencies]
305
- kgmodule-utils = { version = ">=0.26.0", extras = ["semantic", "synthesis"] }
318
+ kgmodule-utils = { version = ">=0.27.0", extras = ["semantic", "synthesis"] }
306
319
  ```
307
320
 
308
321
  ---
@@ -419,7 +432,7 @@ delta = mgr.diff_snapshots(snaps[-1]["key"], snaps[0]["key"])
419
432
 
420
433
  | Class | Description |
421
434
  |---|---|
422
- | `KGModule` | Concrete base — implement `make_extractor()`, `kind()`, `analyze()`; get `build()`, `query()`, `pack()`, `stats()` for free. `build_graph(wipe=True)` drops the vector index it invalidates; `drop_index()` does so on demand |
435
+ | `KGModule` | Concrete base — implement `make_extractor()`, `kind()`, `analyze()`; get `build()`, `query()`, `pack()`, `stats()` for free. `build_graph(wipe=True)` drops the vector index it invalidates; `drop_index()` does so on demand. `sync()` rewrites the graph and re-embeds only the nodes whose index text changed |
423
436
 
424
437
  ### `kg_utils.snapshots`
425
438
 
@@ -674,9 +687,19 @@ Either way, sources are deduplicated by the SHA-256 of their *bytes* rather than
674
687
  their filename, so the same document arriving twice under different names is
675
688
  ingested once.
676
689
 
677
- Every file examined is accounted for, including the ones that did not make it —
678
- `anydoc` performs no OCR, so scanned PDFs are skipped rather than converted, and
679
- a corpus should say so rather than simply lack them:
690
+ Each format has one converter:
691
+
692
+ | Formats | Converter | Needs |
693
+ |---|---|---|
694
+ | `.md`, `.markdown`, `.txt`, `.rst` | `PassthroughConverter` (no transformation) | nothing |
695
+ | `.pdf` | `PdfConverter` (`pymupdf4llm`, the same call `doc_kg` makes) | `[ingest]` or `[pdf]` extra |
696
+ | Word, PowerPoint, Excel, OpenDocument, `.rtf`, `.epub`, `.csv` | `AnydocConverter` | `[ingest]` extra |
697
+ | `.tex`, `.latex`, `.html`, `.htm` | `PandocConverter` | the `pandoc` binary on `PATH` |
698
+
699
+ Every file examined is accounted for, including the ones that did not make it.
700
+ No converter performs OCR, so a scanned PDF is recorded as failed rather than
701
+ converted, and a missing `pandoc` is recorded the same way; a corpus should say
702
+ so rather than simply lack them:
680
703
 
681
704
  ```python
682
705
  for record in pipeline.manifest().problems():
@@ -686,6 +709,23 @@ for record in pipeline.manifest().problems():
686
709
  The same records live in `<staging_root>/.ingest/manifest.json`, with the
687
710
  converter and version that produced each staged file.
688
711
 
712
+ Staged files are bare Markdown; their provenance lives in the manifest. For a
713
+ curated import that keeps provenance in the file itself (the `vault_kg` and
714
+ LLM-wiki case), `write_derived` writes one converted document with YAML
715
+ frontmatter (`source_path` relative to an import root, `source_sha256`,
716
+ `source_format`, `converter`, `converter_version`, `ingested_at`, and the
717
+ document's own title, author, date and page count when it has them). It
718
+ refuses to overwrite, so `ingested_at` is written once:
719
+
720
+ ```python
721
+ from pathlib import Path
722
+ from kg_utils.ingest import PdfConverter, write_derived
723
+
724
+ src = Path("~/papers/turtlend.pdf").expanduser()
725
+ write_derived(PdfConverter().convert(src), src, Path("RAW/articles/turtlend.md"),
726
+ source_root=src.parent, extra={"kind": "articles"})
727
+ ```
728
+
689
729
  For the end-to-end pipeline — converters, staging layout, manifest schema,
690
730
  re-run semantics, and the `kgrag ingest` command that drives build and
691
731
  registration — see
@@ -748,7 +788,7 @@ If you use kgmodule-utils in research or a project, please cite it:
748
788
 
749
789
  **APA**
750
790
 
751
- > Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.26.0) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
791
+ > Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.27.1) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
752
792
 
753
793
  **BibTeX**
754
794
 
@@ -756,7 +796,7 @@ If you use kgmodule-utils in research or a project, please cite it:
756
796
  @software{suchanek_kgmodule_utils,
757
797
  author = {Suchanek, Eric G.},
758
798
  title = {{kgmodule-utils}: Shared SDK for the KGModule Knowledge-Graph Ecosystem},
759
- version = {0.26.0},
799
+ version = {0.27.1},
760
800
  year = {2026},
761
801
  publisher = {Flux-Frontiers},
762
802
  url = {https://github.com/Flux-Frontiers/KG_utils},
@@ -1,7 +1,7 @@
1
1
 
2
2
  [![Python](https://img.shields.io/badge/python-3.12%20%7C%203.13-blue.svg)](https://www.python.org/)
3
3
  [![License: Elastic-2.0](https://img.shields.io/badge/License-Elastic%202.0-blue.svg)](https://www.elastic.co/licensing/elastic-license)
4
- [![Version](https://img.shields.io/badge/version-0.26.0-blue.svg)](https://github.com/Flux-Frontiers/KG_utils/releases)
4
+ [![Version](https://img.shields.io/badge/version-0.27.1-blue.svg)](https://github.com/Flux-Frontiers/KG_utils/releases)
5
5
  [![CI](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml/badge.svg)](https://github.com/Flux-Frontiers/KG_utils/actions/workflows/ci.yml)
6
6
  [![Poetry](https://img.shields.io/endpoint?url=https://python-poetry.org/badge/v0.json)](https://python-poetry.org/)
7
7
  [![DOI](https://img.shields.io/badge/DOI-10.5281%2Fzenodo.21284866-blue.svg)](https://doi.org/10.5281/zenodo.21284866)
@@ -26,6 +26,16 @@ Fleet modules use it in two ways. [PyCodeKG](https://github.com/Flux-Frontiers/p
26
26
 
27
27
  ## Latest News
28
28
 
29
+ - **0.27.1 (2026-10-19)** -- `[pdf]` and `[ingest]` work again on a fresh
30
+ install. pymupdf4llm 2.0 is an empty placeholder (its code now ships inside
31
+ pymupdf 2.0), so 0.27.0's `pymupdf4llm>=1.28.2` resolved to no PDF support.
32
+ Both extras now declare `pymupdf>=2.0`. No API change.
33
+ - **0.27.0 (2026-10-14)** -- One converter per format, and incremental
34
+ vectors. `PdfConverter` is the fleet's single PDF converter (the same
35
+ `pymupdf4llm` call `doc_kg` makes), `PandocConverter` reads `.tex` and
36
+ `.html`, and `write_derived()` writes a converted document with provenance
37
+ frontmatter. `KGModule.sync()` rewrites the graph but re-embeds only the
38
+ nodes whose text changed. New `[pdf]` extra.
29
39
  - **0.26.0 (2026-09-26)** -- Leaves hang as the Knowledge Press web forest hangs
30
40
  them. New `hang_leaves()` puts each leaf's stalk on its nearest twig, its
31
41
  blade out and lifted, its face to the sky, and returns the frame so any
@@ -250,7 +260,7 @@ pip install 'kgmodule-utils[viz3d-qt]'
250
260
 
251
261
  ```toml
252
262
  [tool.poetry.dependencies]
253
- kgmodule-utils = { version = ">=0.26.0", extras = ["semantic", "synthesis"] }
263
+ kgmodule-utils = { version = ">=0.27.0", extras = ["semantic", "synthesis"] }
254
264
  ```
255
265
 
256
266
  ---
@@ -367,7 +377,7 @@ delta = mgr.diff_snapshots(snaps[-1]["key"], snaps[0]["key"])
367
377
 
368
378
  | Class | Description |
369
379
  |---|---|
370
- | `KGModule` | Concrete base — implement `make_extractor()`, `kind()`, `analyze()`; get `build()`, `query()`, `pack()`, `stats()` for free. `build_graph(wipe=True)` drops the vector index it invalidates; `drop_index()` does so on demand |
380
+ | `KGModule` | Concrete base — implement `make_extractor()`, `kind()`, `analyze()`; get `build()`, `query()`, `pack()`, `stats()` for free. `build_graph(wipe=True)` drops the vector index it invalidates; `drop_index()` does so on demand. `sync()` rewrites the graph and re-embeds only the nodes whose index text changed |
371
381
 
372
382
  ### `kg_utils.snapshots`
373
383
 
@@ -622,9 +632,19 @@ Either way, sources are deduplicated by the SHA-256 of their *bytes* rather than
622
632
  their filename, so the same document arriving twice under different names is
623
633
  ingested once.
624
634
 
625
- Every file examined is accounted for, including the ones that did not make it —
626
- `anydoc` performs no OCR, so scanned PDFs are skipped rather than converted, and
627
- a corpus should say so rather than simply lack them:
635
+ Each format has one converter:
636
+
637
+ | Formats | Converter | Needs |
638
+ |---|---|---|
639
+ | `.md`, `.markdown`, `.txt`, `.rst` | `PassthroughConverter` (no transformation) | nothing |
640
+ | `.pdf` | `PdfConverter` (`pymupdf4llm`, the same call `doc_kg` makes) | `[ingest]` or `[pdf]` extra |
641
+ | Word, PowerPoint, Excel, OpenDocument, `.rtf`, `.epub`, `.csv` | `AnydocConverter` | `[ingest]` extra |
642
+ | `.tex`, `.latex`, `.html`, `.htm` | `PandocConverter` | the `pandoc` binary on `PATH` |
643
+
644
+ Every file examined is accounted for, including the ones that did not make it.
645
+ No converter performs OCR, so a scanned PDF is recorded as failed rather than
646
+ converted, and a missing `pandoc` is recorded the same way; a corpus should say
647
+ so rather than simply lack them:
628
648
 
629
649
  ```python
630
650
  for record in pipeline.manifest().problems():
@@ -634,6 +654,23 @@ for record in pipeline.manifest().problems():
634
654
  The same records live in `<staging_root>/.ingest/manifest.json`, with the
635
655
  converter and version that produced each staged file.
636
656
 
657
+ Staged files are bare Markdown; their provenance lives in the manifest. For a
658
+ curated import that keeps provenance in the file itself (the `vault_kg` and
659
+ LLM-wiki case), `write_derived` writes one converted document with YAML
660
+ frontmatter (`source_path` relative to an import root, `source_sha256`,
661
+ `source_format`, `converter`, `converter_version`, `ingested_at`, and the
662
+ document's own title, author, date and page count when it has them). It
663
+ refuses to overwrite, so `ingested_at` is written once:
664
+
665
+ ```python
666
+ from pathlib import Path
667
+ from kg_utils.ingest import PdfConverter, write_derived
668
+
669
+ src = Path("~/papers/turtlend.pdf").expanduser()
670
+ write_derived(PdfConverter().convert(src), src, Path("RAW/articles/turtlend.md"),
671
+ source_root=src.parent, extra={"kind": "articles"})
672
+ ```
673
+
637
674
  For the end-to-end pipeline — converters, staging layout, manifest schema,
638
675
  re-run semantics, and the `kgrag ingest` command that drives build and
639
676
  registration — see
@@ -696,7 +733,7 @@ If you use kgmodule-utils in research or a project, please cite it:
696
733
 
697
734
  **APA**
698
735
 
699
- > Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.26.0) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
736
+ > Suchanek, E. G. (2026). *kgmodule-utils: Shared SDK for the KGModule Knowledge-Graph Ecosystem* (Version 0.27.1) [Software]. Flux-Frontiers. https://doi.org/10.5281/zenodo.21284866
700
737
 
701
738
  **BibTeX**
702
739
 
@@ -704,7 +741,7 @@ If you use kgmodule-utils in research or a project, please cite it:
704
741
  @software{suchanek_kgmodule_utils,
705
742
  author = {Suchanek, Eric G.},
706
743
  title = {{kgmodule-utils}: Shared SDK for the KGModule Knowledge-Graph Ecosystem},
707
- version = {0.26.0},
744
+ version = {0.27.1},
708
745
  year = {2026},
709
746
  publisher = {Flux-Frontiers},
710
747
  url = {https://github.com/Flux-Frontiers/KG_utils},
@@ -9,7 +9,7 @@ build-backend = "poetry.core.masonry.api"
9
9
 
10
10
  [project]
11
11
  name = "kgmodule-utils"
12
- version = "0.26.0"
12
+ version = "0.27.1"
13
13
  description = "Shared types, graph store, semantic index, and pipeline base for the KGModule SDK"
14
14
  readme = "README.md"
15
15
  license = "Elastic-2.0"
@@ -67,8 +67,8 @@ viz3d-qt = [
67
67
  "PyQt5>=5.15.0",
68
68
  # For quiltwright.quilt.resolve_view_cone: the view-cone cap as one shared
69
69
  # function rather than a copy of 35.0 per caller. That landed in 0.14.0;
70
- # the floor sits at 0.14.1, the version the cast path is tested against.
71
- "quiltwright>=0.15.0",
70
+ # the floor tracks the latest release (0.16.0), per the fleet pin rule.
71
+ "quiltwright>=0.16.0",
72
72
  ]
73
73
  # Exact pin: pre-1.0, breaking minors possible. Prebuilt wheels everywhere;
74
74
  # also bundled in `semantic` since "auto" defaults to sqlite-vec.
@@ -80,13 +80,24 @@ sqlite-vec = [
80
80
  lancedb = [
81
81
  "lancedb>=0.19.0",
82
82
  ]
83
- # Document ingestion (kg_utils.ingest): anydoc converts Word, PowerPoint,
84
- # Excel, OpenDocument, RTF, EPUB, CSV and text-based PDF to Markdown. Kept
85
- # optional because .md/.txt/.rst ingest with no dependency at all — only
86
- # non-textual formats need the Rust converter. No OCR: scanned PDFs are
87
- # recorded as skipped, never silently dropped.
83
+ # Document ingestion (kg_utils.ingest): pymupdf4llm converts PDF (the same
84
+ # call doc_kg makes, so both paths agree); anydoc converts Word, PowerPoint,
85
+ # Excel, OpenDocument, RTF, EPUB and CSV. Kept optional because .md/.txt/.rst
86
+ # ingest with no dependency at all. pymupdf (AGPL-3.0 or Artifex commercial)
87
+ # is declared, not pymupdf4llm: since 2.0 the pymupdf wheel ships the
88
+ # pymupdf4llm module, and the pymupdf4llm 2.0 distribution is an empty
89
+ # placeholder that installs no code, so declaring it alone resolves to no PDF
90
+ # support at all. .tex/.html need the pandoc binary, which is not a Python
91
+ # dependency. No OCR: scanned PDFs are recorded as failed with a reason,
92
+ # never silently dropped.
88
93
  ingest = [
89
94
  "firecrawl-anydoc>=0.1.9",
95
+ "pymupdf>=2.0",
96
+ ]
97
+ # PDF only: what doc_kg needs to read PDFs through PdfConverter without
98
+ # pulling in anydoc. [ingest] stays the "every format" extra.
99
+ pdf = [
100
+ "pymupdf>=2.0",
90
101
  ]
91
102
  synthesis = [
92
103
  "httpx>=0.27.0",
@@ -116,8 +127,9 @@ optional = true
116
127
  [tool.poetry.group.dev.dependencies]
117
128
  pytest = ">=9.0.3"
118
129
  pytest-cov = "^5.0.0"
119
- # Exercises the real converter in the ingest tests rather than a stub.
130
+ # Exercises the real converters in the ingest tests rather than stubs.
120
131
  firecrawl-anydoc = ">=0.1.9"
132
+ pymupdf = ">=2.0"
121
133
  # Capped below 0.16 deliberately: 0.16 expands the default rules and reformats
122
134
  # Python in Markdown blocks. Adopt on purpose; lift the cap in its own change.
123
135
  ruff = ">=0.15,<0.16"
@@ -68,4 +68,4 @@ The core install is stdlib-only; everything heavier is opt-in.
68
68
  pip install 'kgmodule-utils[synthesis-mflux]' # + mflux (Apple Silicon local gen)
69
69
  """
70
70
 
71
- __version__ = "0.26.0"
71
+ __version__ = "0.27.1"
@@ -2,7 +2,7 @@
2
2
 
3
3
  The acquisition stage that sits *in front of* every KGModule builder. Point it
4
4
  at ``.md``, ``.pdf``, ``.docx``, ``.epub``, ``.pptx``, ``.xlsx``, ``.csv`` (and
5
- the rest of the ``anydoc`` format set); it converts each to Markdown,
5
+ the rest of the formats ``pymupdf4llm``, ``anydoc`` and ``pandoc`` read); it converts each to Markdown,
6
6
  materializes a staging corpus, and records provenance for every file it saw —
7
7
  including the ones it could not convert.
8
8
 
@@ -36,16 +36,26 @@ from __future__ import annotations
36
36
 
37
37
  from kg_utils.ingest.converters import (
38
38
  ANYDOC_EXTENSIONS,
39
+ PANDOC_EXTENSIONS,
39
40
  PASSTHROUGH_EXTENSIONS,
41
+ PDF_EXTENSIONS,
40
42
  AnydocConverter,
41
43
  ConversionError,
42
44
  ConversionResult,
43
45
  Converter,
46
+ PandocConverter,
44
47
  PassthroughConverter,
48
+ PdfConverter,
45
49
  default_converters,
46
50
  resolve_converter,
47
51
  supported_extensions,
48
52
  )
53
+ from kg_utils.ingest.derived import (
54
+ PROVENANCE_KEYS,
55
+ derived_frontmatter,
56
+ render_derived,
57
+ write_derived,
58
+ )
49
59
  from kg_utils.ingest.manifest import (
50
60
  INGEST_DIR,
51
61
  MANIFEST_NAME,
@@ -63,7 +73,10 @@ __all__ = [
63
73
  "INGEST_DIR",
64
74
  "MANIFEST_NAME",
65
75
  "MANIFEST_VERSION",
76
+ "PANDOC_EXTENSIONS",
66
77
  "PASSTHROUGH_EXTENSIONS",
78
+ "PDF_EXTENSIONS",
79
+ "PROVENANCE_KEYS",
67
80
  "SKIP_DIRS",
68
81
  "AnydocConverter",
69
82
  "ConversionError",
@@ -74,10 +87,15 @@ __all__ = [
74
87
  "IngestRecord",
75
88
  "IngestStats",
76
89
  "IngestStatus",
90
+ "PandocConverter",
77
91
  "PassthroughConverter",
92
+ "PdfConverter",
78
93
  "default_converters",
94
+ "derived_frontmatter",
79
95
  "ingest",
96
+ "render_derived",
80
97
  "resolve_converter",
81
98
  "sha256_file",
82
99
  "supported_extensions",
100
+ "write_derived",
83
101
  ]
@@ -9,14 +9,25 @@ provenance needed to reproduce that conversion later. Two are shipped:
9
9
  downstream chunkers keep applying the right structural parse — a ``.txt``
10
10
  file must not become ``.md`` or it would suddenly be parsed for headings.
11
11
 
12
+ ``PdfConverter``
13
+ PDF, via ``pymupdf4llm``: the fleet's one PDF authority (sweep item 48).
14
+ The call is the same bare ``pymupdf4llm.to_markdown(path)`` that
15
+ ``doc_kg`` makes, so a PDF converts to the same Markdown whether it
16
+ arrives through ``kgrag ingest`` or ``dockg build``.
17
+
12
18
  ``AnydocConverter``
13
- Everything else, via `anydoc <https://github.com/firecrawl/anydoc>`_
19
+ Office and publishing formats, via `anydoc <https://github.com/firecrawl/anydoc>`_
14
20
  (PyPI: ``firecrawl-anydoc``) — a Rust converter with Python bindings that
15
21
  emits consistent GitHub-Flavored Markdown for Word, PowerPoint, Excel,
16
- OpenDocument, RTF, EPUB, CSV and text-based PDF.
22
+ OpenDocument, RTF, EPUB and CSV.
23
+
24
+ ``PandocConverter``
25
+ LaTeX and HTML, via the ``pandoc`` command, which neither of the above
26
+ reads. Pandoc is a system binary, not a Python dependency: when it is
27
+ absent the file is recorded as failed with that reason.
17
28
 
18
- ``anydoc`` performs no OCR. Image-only PDFs raise
19
- :class:`ConversionError`; the pipeline records that as a skip with a reason
29
+ No converter performs OCR. A scanned PDF converts to empty output, which
30
+ raises :class:`ConversionError`; the pipeline records that with a reason
20
31
  rather than dropping the file silently.
21
32
 
22
33
  Author: Eric G. Suchanek, PhD
@@ -24,6 +35,8 @@ Author: Eric G. Suchanek, PhD
24
35
 
25
36
  from __future__ import annotations
26
37
 
38
+ import shutil
39
+ import subprocess
27
40
  from dataclasses import dataclass, field
28
41
  from pathlib import Path
29
42
  from typing import Protocol, runtime_checkable
@@ -62,10 +75,16 @@ ANYDOC_EXTENSIONS: frozenset[str] = frozenset(
62
75
  ".rtf",
63
76
  ".epub",
64
77
  ".csv",
65
- ".pdf",
66
78
  }
67
79
  )
68
80
 
81
+ #: Extensions :class:`PdfConverter` handles.
82
+ PDF_EXTENSIONS: frozenset[str] = frozenset({".pdf"})
83
+
84
+ #: Extensions :class:`PandocConverter` handles: formats no Python converter
85
+ #: here reads.
86
+ PANDOC_EXTENSIONS: frozenset[str] = frozenset({".tex", ".latex", ".html", ".htm"})
87
+
69
88
 
70
89
  class ConversionError(RuntimeError):
71
90
  """Raised when a source file cannot be converted to Markdown.
@@ -269,6 +288,161 @@ class AnydocConverter:
269
288
  )
270
289
 
271
290
 
291
+ class PdfConverter:
292
+ """Convert PDF to Markdown with ``pymupdf4llm``.
293
+
294
+ The conversion is a bare ``pymupdf4llm.to_markdown(str(path))`` -- no page
295
+ chunks, no anchors -- exactly what ``doc_kg.pdf_reader`` does, so the two
296
+ paths produce byte-identical Markdown. Page anchors are roadmap Phase 2.
297
+
298
+ Title, author, creation date and page count are read from the PDF's own
299
+ metadata into :attr:`ConversionResult.metadata` when present.
300
+ """
301
+
302
+ name = "pymupdf4llm"
303
+
304
+ def __init__(self, extensions: frozenset[str] | None = None) -> None:
305
+ """:param extensions: Suffixes to accept (default: :data:`PDF_EXTENSIONS`)."""
306
+ self.extensions = extensions if extensions is not None else PDF_EXTENSIONS
307
+ self._version = ""
308
+
309
+ @property
310
+ def version(self) -> str:
311
+ """Installed ``pymupdf4llm`` version, read without importing it."""
312
+ if not self._version:
313
+ self._version = _installed_version("pymupdf4llm")
314
+ return self._version
315
+
316
+ def handles(self, path: Path) -> bool:
317
+ """Return ``True`` for suffixes in this converter's extension set."""
318
+ return path.suffix.lower() in self.extensions
319
+
320
+ def convert(self, path: Path) -> ConversionResult:
321
+ """Convert *path* to Markdown with ``pymupdf4llm``.
322
+
323
+ :param path: PDF to convert.
324
+ :return: Markdown plus provenance, staged as ``.md``.
325
+ :raises ConversionError: If ``pymupdf4llm`` is not installed, cannot
326
+ open or parse the file, or the file has no text layer.
327
+ """
328
+ try:
329
+ import pymupdf # pylint: disable=import-outside-toplevel
330
+ import pymupdf4llm # pylint: disable=import-outside-toplevel
331
+ except ImportError as exc: # pragma: no cover — depends on install
332
+ raise ConversionError(
333
+ "pymupdf4llm is not installed; install the ingest extra: "
334
+ "pip install 'kgmodule-utils[ingest]'"
335
+ ) from exc
336
+ try:
337
+ markdown = pymupdf4llm.to_markdown(str(path))
338
+ except Exception as exc: # noqa: BLE001 — any parser failure is a recorded skip
339
+ raise ConversionError(f"pymupdf4llm could not parse {path.name}: {exc}") from exc
340
+ if not markdown.strip():
341
+ raise ConversionError(
342
+ "converted to empty output (likely a scanned/image-only document needing OCR)"
343
+ )
344
+
345
+ metadata = {"source_format": "pdf"}
346
+ try:
347
+ with pymupdf.open(str(path)) as doc:
348
+ metadata["pages"] = str(doc.page_count)
349
+ info = doc.metadata or {}
350
+ for key, field_name in (
351
+ ("title", "title"),
352
+ ("author", "author"),
353
+ ("creationDate", "created"),
354
+ ):
355
+ value = (info.get(key) or "").strip()
356
+ if value:
357
+ metadata[field_name] = value
358
+ except Exception: # noqa: BLE001 — metadata is optional; the text already converted
359
+ pass
360
+ return ConversionResult(
361
+ markdown=markdown,
362
+ converter=self.name,
363
+ converter_version=self.version,
364
+ suffix=".md",
365
+ metadata=metadata,
366
+ )
367
+
368
+
369
+ class PandocConverter:
370
+ """Convert LaTeX and HTML to GitHub-Flavored Markdown with ``pandoc``.
371
+
372
+ ``pandoc`` is found on ``PATH`` at conversion time. It is a system
373
+ binary rather than a Python dependency, so a missing ``pandoc`` is a
374
+ recorded failure with an install hint, never a silent skip.
375
+ """
376
+
377
+ name = "pandoc"
378
+
379
+ def __init__(self, extensions: frozenset[str] | None = None) -> None:
380
+ """:param extensions: Suffixes to accept (default: :data:`PANDOC_EXTENSIONS`)."""
381
+ self.extensions = extensions if extensions is not None else PANDOC_EXTENSIONS
382
+ self._version = ""
383
+
384
+ @property
385
+ def version(self) -> str:
386
+ """Version reported by ``pandoc --version``, or ``"unknown"``."""
387
+ if not self._version:
388
+ exe = shutil.which("pandoc")
389
+ self._version = "unknown"
390
+ if exe:
391
+ try:
392
+ out = subprocess.run(
393
+ [exe, "--version"], capture_output=True, text=True, check=True, timeout=30
394
+ ).stdout
395
+ first = out.splitlines()[0] if out else ""
396
+ self._version = first.split()[-1] if first else "unknown"
397
+ except (OSError, subprocess.SubprocessError):
398
+ pass
399
+ return self._version
400
+
401
+ def handles(self, path: Path) -> bool:
402
+ """Return ``True`` for suffixes in this converter's extension set."""
403
+ return path.suffix.lower() in self.extensions
404
+
405
+ def convert(self, path: Path) -> ConversionResult:
406
+ """Convert *path* to Markdown with ``pandoc -t gfm --wrap=none``.
407
+
408
+ :param path: LaTeX or HTML file to convert.
409
+ :return: Markdown plus provenance, staged as ``.md``.
410
+ :raises ConversionError: If ``pandoc`` is not on ``PATH``, fails, or
411
+ produces empty output.
412
+ """
413
+ exe = shutil.which("pandoc")
414
+ if exe is None:
415
+ raise ConversionError(
416
+ "pandoc is not installed; install it (https://pandoc.org/installing.html, "
417
+ "or `brew install pandoc`) to convert .tex and .html"
418
+ )
419
+ try:
420
+ proc = subprocess.run(
421
+ [exe, str(path), "-t", "gfm", "--wrap=none"],
422
+ capture_output=True,
423
+ text=True,
424
+ check=True,
425
+ timeout=300,
426
+ )
427
+ except subprocess.CalledProcessError as exc:
428
+ detail = (exc.stderr or "").strip().splitlines()
429
+ raise ConversionError(
430
+ f"pandoc failed on {path.name}: {detail[-1] if detail else exc}"
431
+ ) from exc
432
+ except (OSError, subprocess.SubprocessError) as exc:
433
+ raise ConversionError(f"pandoc failed on {path.name}: {exc}") from exc
434
+ if not proc.stdout.strip():
435
+ raise ConversionError("converted to empty output")
436
+ fmt = "latex" if path.suffix.lower() in (".tex", ".latex") else "html"
437
+ return ConversionResult(
438
+ markdown=proc.stdout,
439
+ converter=self.name,
440
+ converter_version=self.version,
441
+ suffix=".md",
442
+ metadata={"source_format": fmt},
443
+ )
444
+
445
+
272
446
  def _installed_version(distribution: str) -> str:
273
447
  """Return the installed version of *distribution*, or ``"unknown"``.
274
448
 
@@ -289,12 +463,13 @@ def _installed_version(distribution: str) -> str:
289
463
  def default_converters() -> list[Converter]:
290
464
  """Return the standard converter chain, most specific first.
291
465
 
292
- Passthrough is tried before ``anydoc`` so that Markdown and plain text are
293
- never round-tripped through a converter that would rewrite them.
466
+ Passthrough is tried first so that Markdown and plain text are never
467
+ round-tripped through a converter that would rewrite them. The other
468
+ three handle disjoint extension sets, so their order does not matter.
294
469
 
295
470
  :return: Converters in resolution order.
296
471
  """
297
- return [PassthroughConverter(), AnydocConverter()]
472
+ return [PassthroughConverter(), PdfConverter(), AnydocConverter(), PandocConverter()]
298
473
 
299
474
 
300
475
  def resolve_converter(path: Path, converters: list[Converter]) -> Converter | None:
@@ -313,9 +488,10 @@ def resolve_converter(path: Path, converters: list[Converter]) -> Converter | No
313
488
  def supported_extensions() -> frozenset[str]:
314
489
  """Return every suffix the default converter chain accepts.
315
490
 
316
- Answers without importing ``anydoc``, so it is safe to call on a core-only
317
- install.
491
+ Answers without importing any converter library, so it is safe to call on
492
+ a core-only install. ``pandoc``'s formats are included whether or not the
493
+ binary is installed; a missing binary is reported per file.
318
494
 
319
495
  :return: Lower-case suffixes including the leading dot.
320
496
  """
321
- return PASSTHROUGH_EXTENSIONS | ANYDOC_EXTENSIONS
497
+ return PASSTHROUGH_EXTENSIONS | PDF_EXTENSIONS | ANYDOC_EXTENSIONS | PANDOC_EXTENSIONS
@@ -0,0 +1,148 @@
1
+ """kg_utils/ingest/derived.py — Write one converted document with provenance.
2
+
3
+ A *derived note* is the Markdown a converter produced from one source file,
4
+ written to disk with YAML frontmatter that records where it came from:
5
+
6
+ .. code-block:: yaml
7
+
8
+ ---
9
+ source_path: "papers/turtlend.pdf"
10
+ source_sha256: "9f2c..."
11
+ source_format: "pdf"
12
+ converter: "pymupdf4llm"
13
+ converter_version: "1.28.2"
14
+ ingested_at: "2026-10-14T21:03:11Z"
15
+ title: "TurtleND"
16
+ pages: "12"
17
+ ---
18
+
19
+ The schema is rule 3 of ``kgrag_priv/docs/INGESTION_ROADMAP.md``. Rule 5
20
+ (same bytes and converter version give the same file) holds for everything
21
+ except ``ingested_at``, which is written once: :func:`write_derived` refuses
22
+ to overwrite, so a derived note is never regenerated in place.
23
+
24
+ This is the writer for curated imports such as ``vaultkg add``. The staging
25
+ pipeline (:class:`~kg_utils.ingest.pipeline.IngestPipeline`) does *not* use
26
+ it yet: DocKG cannot parse frontmatter until roadmap Phase 1, so staged files
27
+ stay bare Markdown with their provenance in the ingest manifest.
28
+
29
+ Author: Eric G. Suchanek, PhD
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ import json
35
+ from collections.abc import Mapping
36
+ from pathlib import Path
37
+
38
+ from kg_utils.ingest.converters import ConversionResult
39
+ from kg_utils.ingest.manifest import sha256_file, utc_now
40
+
41
+ #: Frontmatter keys written first, in this order; converter metadata and
42
+ #: caller extras follow, sorted.
43
+ PROVENANCE_KEYS: tuple[str, ...] = (
44
+ "source_path",
45
+ "source_sha256",
46
+ "source_format",
47
+ "converter",
48
+ "converter_version",
49
+ "ingested_at",
50
+ )
51
+
52
+
53
+ def derived_frontmatter(
54
+ result: ConversionResult,
55
+ source: Path,
56
+ *,
57
+ source_root: Path | None = None,
58
+ ingested_at: str | None = None,
59
+ extra: Mapping[str, str] | None = None,
60
+ ) -> dict[str, str]:
61
+ """Build the provenance mapping for one derived note.
62
+
63
+ :param result: The conversion to describe.
64
+ :param source: The source file that was converted.
65
+ :param source_root: Directory ``source_path`` is recorded relative to.
66
+ When ``None``, or when *source* is not under it, only the file name is
67
+ recorded, so no absolute home-directory path reaches a tracked file.
68
+ :param ingested_at: ISO timestamp to record; defaults to now (UTC).
69
+ :param extra: Caller fields (for example ``kind``). They may not replace a
70
+ provenance key.
71
+ :return: Ordered mapping of frontmatter keys to string values.
72
+ :raises ValueError: If *extra* names a provenance key.
73
+ """
74
+ source = Path(source)
75
+ rel = source.name
76
+ if source_root is not None:
77
+ try:
78
+ rel = source.resolve().relative_to(Path(source_root).resolve()).as_posix()
79
+ except ValueError:
80
+ rel = source.name
81
+ fields: dict[str, str] = {
82
+ "source_path": rel,
83
+ "source_sha256": sha256_file(source),
84
+ "source_format": result.metadata.get("source_format") or source.suffix.lower().lstrip("."),
85
+ "converter": result.converter,
86
+ "converter_version": result.converter_version,
87
+ "ingested_at": ingested_at or utc_now(),
88
+ }
89
+ clash = sorted(set(extra or {}) & set(PROVENANCE_KEYS))
90
+ if clash:
91
+ raise ValueError(f"extra may not replace provenance keys: {clash}")
92
+ tail = {k: v for k, v in result.metadata.items() if k not in fields}
93
+ tail.update(extra or {})
94
+ for key in sorted(tail):
95
+ fields[key] = str(tail[key])
96
+ return fields
97
+
98
+
99
+ def render_derived(fields: Mapping[str, str], markdown: str) -> str:
100
+ """Render frontmatter plus body as one document.
101
+
102
+ Values are written as JSON strings, which are valid double-quoted YAML
103
+ scalars, so a title containing ``: `` or quotes cannot break the block.
104
+
105
+ :param fields: Frontmatter keys and values.
106
+ :param markdown: Document body.
107
+ :return: The complete file text, ending in one newline.
108
+ """
109
+ lines = ["---"]
110
+ lines += [
111
+ f"{key}: {json.dumps(str(value), ensure_ascii=False)}" for key, value in fields.items()
112
+ ]
113
+ lines += ["---", ""]
114
+ return "\n".join(lines) + markdown.rstrip("\n") + "\n"
115
+
116
+
117
+ def write_derived(
118
+ result: ConversionResult,
119
+ source: Path,
120
+ dest: Path,
121
+ *,
122
+ source_root: Path | None = None,
123
+ ingested_at: str | None = None,
124
+ extra: Mapping[str, str] | None = None,
125
+ ) -> Path:
126
+ """Write *result* to *dest* as a derived note with provenance frontmatter.
127
+
128
+ :param result: The conversion to write.
129
+ :param source: The source file that was converted.
130
+ :param dest: Where to write the note. Its parent is created if needed.
131
+ :param source_root: See :func:`derived_frontmatter`.
132
+ :param ingested_at: See :func:`derived_frontmatter`.
133
+ :param extra: See :func:`derived_frontmatter`.
134
+ :return: *dest*.
135
+ :raises FileExistsError: If *dest* already exists. A derived note is
136
+ written once; deciding what an existing file means (an identical
137
+ re-import, or a name collision) is the caller's job.
138
+ """
139
+ dest = Path(dest)
140
+ if dest.exists():
141
+ raise FileExistsError(f"derived note already exists: {dest}")
142
+ fields = derived_frontmatter(
143
+ result, source, source_root=source_root, ingested_at=ingested_at, extra=extra
144
+ )
145
+ dest.parent.mkdir(parents=True, exist_ok=True)
146
+ with dest.open("x", encoding="utf-8") as fh:
147
+ fh.write(render_derived(fields, result.markdown))
148
+ return dest
@@ -499,14 +499,7 @@ class KGModule(ABC):
499
499
  index (:meth:`drop_index`), which no longer matches the new graph.
500
500
  :return: :class:`~kg_utils.types.BuildStats` (``indexed_rows`` will be ``None``).
501
501
  """
502
- extractor = self.make_extractor()
503
- node_specs: list[NodeSpec] = []
504
- edge_specs: list[EdgeSpec] = []
505
- for item in extractor.extract():
506
- if isinstance(item, NodeSpec):
507
- node_specs.append(item)
508
- else:
509
- edge_specs.append(item)
502
+ node_specs, edge_specs = self._extract_specs()
510
503
 
511
504
  if wipe:
512
505
  # A wiped graph invalidates every vector: an index left from the
@@ -527,6 +520,55 @@ class KGModule(ABC):
527
520
  edge_counts=s["edge_counts"],
528
521
  )
529
522
 
523
+ def _extract_specs(self) -> tuple[list[NodeSpec], list[EdgeSpec]]:
524
+ """Drain the extractor into node and edge specs.
525
+
526
+ :return: ``(node_specs, edge_specs)`` in extraction order.
527
+ """
528
+ node_specs: list[NodeSpec] = []
529
+ edge_specs: list[EdgeSpec] = []
530
+ for item in self.make_extractor().extract():
531
+ if isinstance(item, NodeSpec):
532
+ node_specs.append(item)
533
+ else:
534
+ edge_specs.append(item)
535
+ return node_specs, edge_specs
536
+
537
+ def sync(self) -> BuildStats:
538
+ """Rebuild the graph, then update the vector index incrementally.
539
+
540
+ The graph is always rewritten in full (``wipe=True``): extraction is
541
+ cheap and a full rewrite keeps "same source, same graph" exact. Only
542
+ the vectors are incremental: :meth:`SemanticIndex.sync` embeds the
543
+ nodes whose index text changed, deletes the vectors of nodes that are
544
+ gone, and leaves the rest. Unlike :meth:`build_graph` with
545
+ ``wipe=True``, this does not drop the index first.
546
+
547
+ The first sync of a store embeds every node once, because no
548
+ per-node hashes are recorded yet.
549
+
550
+ :return: :class:`~kg_utils.specs.BuildStats` with ``indexed_rows``,
551
+ ``index_dim``, ``embedded_rows`` and ``deleted_rows`` set.
552
+ """
553
+ node_specs, edge_specs = self._extract_specs()
554
+ self.store.write(node_specs, edge_specs, wipe=True)
555
+ self._post_build_hook(self.store)
556
+
557
+ idx = self.index.sync(self.store, signature=f"{self.model_name}:{self.embedder.dim}")
558
+ s = self.store.stats()
559
+ return BuildStats(
560
+ repo_root=str(self.repo_root),
561
+ db_path=str(self.db_path),
562
+ total_nodes=s["total_nodes"],
563
+ total_edges=s["total_edges"],
564
+ node_counts=s["node_counts"],
565
+ edge_counts=s["edge_counts"],
566
+ indexed_rows=idx["indexed_rows"],
567
+ index_dim=idx["dim"],
568
+ embedded_rows=idx["embedded_rows"],
569
+ deleted_rows=idx["deleted_rows"],
570
+ )
571
+
530
572
  def drop_index(self) -> list[Path]:
531
573
  """Delete the vector index, closing it first.
532
574
 
@@ -27,6 +27,7 @@ constructs it any more — see :meth:`SemanticIndex._get_backend`.
27
27
 
28
28
  from __future__ import annotations
29
29
 
30
+ import hashlib
30
31
  import logging
31
32
  import os
32
33
  import re
@@ -200,20 +201,7 @@ class SemanticIndex:
200
201
  for i in range(0, len(nodes), batch_size):
201
202
  chunk = nodes[i : i + batch_size]
202
203
  texts = [_build_index_text(n) for n in chunk]
203
- vecs = self.embedder.embed_texts(texts)
204
-
205
- rows = [
206
- {
207
- "id": n["id"],
208
- "kind": n["kind"],
209
- "name": n["name"],
210
- "qualname": n["qualname"] or "",
211
- "module_path": n["module_path"] or "",
212
- "text": text,
213
- "vector": vec,
214
- }
215
- for n, text, vec in zip(chunk, texts, vecs)
216
- ]
204
+ rows = self._rows(chunk, texts)
217
205
  indexed += backend.upsert(rows, batch_size=len(rows))
218
206
 
219
207
  # No-op unless the backend is a LanceDBBackend with ANN enabled.
@@ -228,6 +216,115 @@ class SemanticIndex:
228
216
  "kinds": list(self.index_kinds),
229
217
  }
230
218
 
219
+ def sync(
220
+ self,
221
+ store: GraphStore,
222
+ *,
223
+ signature: str | None = None,
224
+ batch_size: int = 256,
225
+ quiet: bool = True,
226
+ ) -> dict[str, Any]:
227
+ """Bring the vector index in line with *store*, embedding only changes.
228
+
229
+ Each node's index text is hashed. A node is embedded when it is new or
230
+ its hash differs from the one recorded at the last sync; vectors for
231
+ nodes no longer in *store* are deleted; everything else is left as it
232
+ is. The hashes and the embedder *signature* live in sidecar tables of
233
+ the sqlite-vec store, separate from ``vec_meta``.
234
+
235
+ The first sync after a full :meth:`build` (or on a store that has
236
+ never been synced) embeds every node once, because no hashes are
237
+ recorded yet; later syncs are incremental. A changed *signature*
238
+ (another model or dimension) wipes the store and embeds everything.
239
+
240
+ Only :class:`SqliteVecBackend` keeps sync state. With any other
241
+ backend this falls back to ``build(store, wipe=True)``.
242
+
243
+ A vector embedded alone can differ in its last bits from the same text
244
+ embedded in a batch, so ties in a ranking may order differently than
245
+ after a full build. The graph itself is unaffected.
246
+
247
+ :param store: Authoritative :class:`~kg_utils.store.GraphStore`.
248
+ :param signature: Embedder identity; defaults to the embedder's class
249
+ name, its ``model_name`` if it has one, and its dimension.
250
+ :param batch_size: Number of nodes to embed per batch.
251
+ :param quiet: Suppress progress output during ingestion.
252
+ :return: The :meth:`build` stats plus ``embedded_rows`` and
253
+ ``deleted_rows``.
254
+ """
255
+ backend = self._get_backend()
256
+ if not isinstance(backend, SqliteVecBackend):
257
+ stats = self.build(store, wipe=True, batch_size=batch_size, quiet=quiet)
258
+ return {**stats, "embedded_rows": stats["indexed_rows"], "deleted_rows": 0}
259
+ if quiet:
260
+ suppress_ingestion_logging()
261
+
262
+ if signature is None:
263
+ signature = ":".join(
264
+ (
265
+ type(self.embedder).__name__,
266
+ str(getattr(self.embedder, "model_name", "")),
267
+ str(self.embedder.dim),
268
+ )
269
+ )
270
+ nodes = self._read_nodes(store)
271
+ texts = {n["id"]: _build_index_text(n) for n in nodes}
272
+ shas = {nid: hashlib.sha256(t.encode("utf-8")).hexdigest() for nid, t in texts.items()}
273
+
274
+ backend.open(wipe=False)
275
+ stored_signature, stored = backend.load_sync_state()
276
+ if stored_signature and stored_signature != signature:
277
+ backend.close()
278
+ backend.open(wipe=True)
279
+ stored = {}
280
+ present = backend.existing_ids()
281
+
282
+ gone = sorted(present - shas.keys())
283
+ deleted = backend.delete_ids(gone) if gone else 0
284
+ todo = [n for n in nodes if n["id"] not in present or stored.get(n["id"]) != shas[n["id"]]]
285
+
286
+ embedded = 0
287
+ for i in range(0, len(todo), batch_size):
288
+ chunk = todo[i : i + batch_size]
289
+ rows = self._rows(chunk, [texts[n["id"]] for n in chunk])
290
+ embedded += backend.upsert(rows, batch_size=len(rows))
291
+ backend.save_sync_state(
292
+ signature,
293
+ {n["id"]: shas[n["id"]] for n in todo},
294
+ [*gone, *(stored.keys() - shas.keys())],
295
+ )
296
+
297
+ return {
298
+ "indexed_rows": backend.count(),
299
+ "embedded_rows": embedded,
300
+ "deleted_rows": deleted,
301
+ "dim": self.embedder.dim,
302
+ "table": self.table_name,
303
+ "vectors_path": str(self.vectors_path),
304
+ "kinds": list(self.index_kinds),
305
+ }
306
+
307
+ def _rows(self, nodes: Sequence[dict[str, Any]], texts: Sequence[str]) -> list[dict[str, Any]]:
308
+ """Embed *texts* and pair each vector with its node's metadata row.
309
+
310
+ :param nodes: Node dicts from the graph store.
311
+ :param texts: Index text for each node, in the same order.
312
+ :return: Backend rows (``id``, the code-KG meta columns, ``text``, ``vector``).
313
+ """
314
+ vecs = self.embedder.embed_texts(list(texts))
315
+ return [
316
+ {
317
+ "id": n["id"],
318
+ "kind": n["kind"],
319
+ "name": n["name"],
320
+ "qualname": n["qualname"] or "",
321
+ "module_path": n["module_path"] or "",
322
+ "text": text,
323
+ "vector": vec,
324
+ }
325
+ for n, text, vec in zip(nodes, texts, vecs)
326
+ ]
327
+
231
328
  # ------------------------------------------------------------------
232
329
  # Search
233
330
  # ------------------------------------------------------------------
@@ -64,6 +64,10 @@ class BuildStats:
64
64
  :param edge_counts: Edge counts broken down by relation.
65
65
  :param indexed_rows: Number of nodes embedded into LanceDB (``None`` if not built).
66
66
  :param index_dim: Embedding dimension (``None`` if not built).
67
+ :param embedded_rows: Vectors embedded by :meth:`KGModule.sync` (``None``
68
+ for a full build, which embeds every row).
69
+ :param deleted_rows: Vectors deleted by :meth:`KGModule.sync` (``None``
70
+ for a full build).
67
71
  """
68
72
 
69
73
  repo_root: str
@@ -74,6 +78,8 @@ class BuildStats:
74
78
  edge_counts: dict[str, int]
75
79
  indexed_rows: int | None = None
76
80
  index_dim: int | None = None
81
+ embedded_rows: int | None = None
82
+ deleted_rows: int | None = None
77
83
 
78
84
  def to_dict(self) -> dict[str, Any]:
79
85
  """Serialise the stats to a plain dictionary."""
@@ -86,6 +92,13 @@ class BuildStats:
86
92
  "edge_counts": self.edge_counts,
87
93
  "indexed_rows": self.indexed_rows,
88
94
  "index_dim": self.index_dim,
95
+ # Sync-only fields appear only after a sync, so a full build's
96
+ # dict is unchanged for callers that compare it whole.
97
+ **(
98
+ {"embedded_rows": self.embedded_rows, "deleted_rows": self.deleted_rows}
99
+ if self.embedded_rows is not None
100
+ else {}
101
+ ),
89
102
  }
90
103
 
91
104
  def __str__(self) -> str:
@@ -97,6 +110,10 @@ class BuildStats:
97
110
  ]
98
111
  if self.indexed_rows is not None:
99
112
  lines.append(f"indexed : {self.indexed_rows} vectors dim={self.index_dim}")
113
+ if self.embedded_rows is not None:
114
+ lines.append(
115
+ f"synced : {self.embedded_rows} embedded, {self.deleted_rows} deleted"
116
+ )
100
117
  return "\n".join(lines)
101
118
 
102
119
 
@@ -456,6 +456,9 @@ class SqliteVecBackend:
456
456
  if wipe:
457
457
  conn.execute("DROP TABLE IF EXISTS vec_nodes")
458
458
  conn.execute("DROP TABLE IF EXISTS vec_meta")
459
+ # sync() state describes the vectors just dropped.
460
+ conn.execute(f"DROP TABLE IF EXISTS {SYNC_HASH_TABLE}")
461
+ conn.execute(f"DROP TABLE IF EXISTS {SYNC_INFO_TABLE}")
459
462
  existed = bool(
460
463
  conn.execute(
461
464
  "SELECT 1 FROM sqlite_master WHERE type='table' AND name='vec_meta'"
@@ -577,6 +580,68 @@ class SqliteVecBackend:
577
580
  removed += self._delete_ids_conn(conn, ids[start : start + 1000])
578
581
  return removed
579
582
 
583
+ # -- sync state -----------------------------------------------------
584
+ #
585
+ # Two sidecar tables that only :meth:`SemanticIndex.sync` reads and
586
+ # writes. They are separate from ``vec_meta`` on purpose: every consumer
587
+ # shares that table and some read it directly, so a full build stays
588
+ # exactly what it was. A store that has never been synced has neither.
589
+
590
+ def load_sync_state(self) -> tuple[str, dict[str, str]]:
591
+ """Return the stored embedder signature and per-id index-text hashes.
592
+
593
+ :return: ``(signature, {node_id: text_sha256})``; ``("", {})`` when
594
+ the store has never been synced.
595
+ """
596
+ conn = self._c()
597
+ have = {
598
+ r[0]
599
+ for r in conn.execute(
600
+ "SELECT name FROM sqlite_master WHERE type='table' AND name IN (?, ?)",
601
+ (SYNC_HASH_TABLE, SYNC_INFO_TABLE),
602
+ ).fetchall()
603
+ }
604
+ if have != {SYNC_HASH_TABLE, SYNC_INFO_TABLE}:
605
+ return "", {}
606
+ row = conn.execute(
607
+ f"SELECT value FROM {SYNC_INFO_TABLE} WHERE key = 'signature'"
608
+ ).fetchone()
609
+ hashes = dict(conn.execute(f"SELECT id, sha FROM {SYNC_HASH_TABLE}").fetchall())
610
+ return (row[0] if row else ""), hashes
611
+
612
+ def save_sync_state(
613
+ self, signature: str, written: dict[str, str], removed: Iterable[str]
614
+ ) -> None:
615
+ """Record the embedder signature and update the per-id hashes.
616
+
617
+ :param signature: Embedder identity the stored vectors were made with.
618
+ :param written: ``{node_id: text_sha256}`` for ids embedded this run.
619
+ :param removed: Ids whose vectors were deleted this run.
620
+ """
621
+ conn = self._c()
622
+ with conn:
623
+ conn.execute(
624
+ f"CREATE TABLE IF NOT EXISTS {SYNC_HASH_TABLE}(id TEXT PRIMARY KEY, sha TEXT NOT NULL)"
625
+ )
626
+ conn.execute(
627
+ f"CREATE TABLE IF NOT EXISTS {SYNC_INFO_TABLE}(key TEXT PRIMARY KEY, value TEXT NOT NULL)"
628
+ )
629
+ conn.execute(
630
+ f"INSERT OR REPLACE INTO {SYNC_INFO_TABLE}(key, value) VALUES ('signature', ?)",
631
+ (signature,),
632
+ )
633
+ gone = [i for i in removed if i]
634
+ for start in range(0, len(gone), 1000):
635
+ chunk = gone[start : start + 1000]
636
+ conn.execute(
637
+ f"DELETE FROM {SYNC_HASH_TABLE} WHERE id IN ({', '.join('?' for _ in chunk)})",
638
+ chunk,
639
+ )
640
+ conn.executemany(
641
+ f"INSERT OR REPLACE INTO {SYNC_HASH_TABLE}(id, sha) VALUES (?, ?)",
642
+ list(written.items()),
643
+ )
644
+
580
645
  # -- reads ----------------------------------------------------------
581
646
 
582
647
  def existing_ids(self) -> set[str]:
@@ -622,6 +687,12 @@ class SqliteVecBackend:
622
687
  return out
623
688
 
624
689
 
690
+ #: Sidecar tables holding :meth:`SemanticIndex.sync` state (see
691
+ #: :meth:`SqliteVecBackend.load_sync_state`).
692
+ SYNC_HASH_TABLE = "vec_sync_hash"
693
+ SYNC_INFO_TABLE = "vec_sync_info"
694
+
695
+
625
696
  def _escape(s: str) -> str:
626
697
  return s.replace("'", "''")
627
698
 
File without changes