okfgraph 0.7.0__tar.gz → 0.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. {okfgraph-0.7.0 → okfgraph-0.9.0}/PKG-INFO +19 -11
  2. {okfgraph-0.7.0 → okfgraph-0.9.0}/README.md +15 -8
  3. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/cli.py +77 -8
  4. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/diff.py +6 -0
  5. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/doctor.py +31 -0
  6. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/embedding.py +276 -2
  7. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/image_assets.py +106 -12
  8. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/import_.py +57 -11
  9. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/ingest.py +21 -4
  10. okfgraph-0.9.0/okfgraph/components/producers.py +515 -0
  11. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/config.py +31 -4
  12. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/images.py +83 -37
  13. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/mcp_server.py +3 -3
  14. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/router.py +97 -14
  15. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/tools.py +10 -8
  16. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/PKG-INFO +19 -11
  17. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/SOURCES.txt +7 -1
  18. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/requires.txt +3 -2
  19. {okfgraph-0.7.0 → okfgraph-0.9.0}/pyproject.toml +11 -5
  20. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_config.py +3 -5
  21. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_gpu_integration.py +17 -14
  22. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_images.py +55 -24
  23. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_lazy_encoder.py +8 -3
  24. okfgraph-0.9.0/tests/test_observations.py +204 -0
  25. okfgraph-0.9.0/tests/test_produce.py +211 -0
  26. okfgraph-0.9.0/tests/test_rootless.py +133 -0
  27. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_router.py +2 -2
  28. okfgraph-0.9.0/tests/test_thought_ids.py +132 -0
  29. okfgraph-0.9.0/tests/test_vision.py +210 -0
  30. {okfgraph-0.7.0 → okfgraph-0.9.0}/LICENSE +0 -0
  31. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/__init__.py +0 -0
  32. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/__init__.py +0 -0
  33. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/converters.py +0 -0
  34. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/delta.py +0 -0
  35. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/export.py +0 -0
  36. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/links.py +0 -0
  37. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/lint.py +0 -0
  38. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/purge.py +0 -0
  39. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/ranking.py +0 -0
  40. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/roots.py +0 -0
  41. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/schema.py +0 -0
  42. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/search.py +0 -0
  43. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/models.py +0 -0
  44. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/security.py +0 -0
  45. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/dependency_links.txt +0 -0
  46. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/entry_points.txt +0 -0
  47. {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/top_level.txt +0 -0
  48. {okfgraph-0.7.0 → okfgraph-0.9.0}/setup.cfg +0 -0
  49. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_chunk_cap.py +0 -0
  50. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_chunk_search.py +0 -0
  51. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_chunking.py +0 -0
  52. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_cli.py +0 -0
  53. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_converter.py +0 -0
  54. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_delta.py +0 -0
  55. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_detach.py +0 -0
  56. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_diff.py +0 -0
  57. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_directory_hash.py +0 -0
  58. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_doctor.py +0 -0
  59. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_encode_bucketing.py +0 -0
  60. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_error_paths.py +0 -0
  61. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_explicit_files.py +0 -0
  62. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_export_compliance.py +0 -0
  63. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_golden_vectors.py +0 -0
  64. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_graph_enrichment.py +0 -0
  65. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_image_ingest.py +0 -0
  66. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ingest.py +0 -0
  67. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ingest_tool.py +0 -0
  68. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_integration.py +0 -0
  69. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_lint.py +0 -0
  70. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_logging.py +0 -0
  71. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_max_length.py +0 -0
  72. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_mcp_server.py +0 -0
  73. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_mcp_stdio.py +0 -0
  74. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_model.py +0 -0
  75. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_models.py +0 -0
  76. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_multiroot.py +0 -0
  77. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_obsidian.py +0 -0
  78. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_okf_ingest_tool.py +0 -0
  79. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ort.py +0 -0
  80. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_packaging.py +0 -0
  81. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_parity.py +0 -0
  82. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_pdf_e2e.py +0 -0
  83. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_phase0.py +0 -0
  84. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ppr_search.py +0 -0
  85. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_precision.py +0 -0
  86. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ranking.py +0 -0
  87. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_reconstruction.py +0 -0
  88. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_reserved.py +0 -0
  89. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_retrieval_conformance.py +0 -0
  90. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_roundup_cli.py +0 -0
  91. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_router_misc.py +0 -0
  92. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_rust_backend.py +0 -0
  93. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_rust_e2e.py +0 -0
  94. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_sanitize.py +0 -0
  95. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_search_browser.py +0 -0
  96. {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_security.py +0 -0
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: okfgraph
3
- Version: 0.7.0
3
+ Version: 0.9.0
4
4
  Summary: Ladybug-backed OKF knowledge graph with ONNX + Jina v5 embeddings
5
5
  License-Expression: Apache-2.0 OR MIT
6
6
  Requires-Python: >=3.11
7
7
  Description-Content-Type: text/markdown
8
8
  License-File: LICENSE
9
- Requires-Dist: ladybug==0.20.3
10
- Requires-Dist: embroider<0.3,>=0.2.0
9
+ Requires-Dist: ladybug==0.21.2
10
+ Requires-Dist: embroider<0.4,>=0.3.0
11
11
  Requires-Dist: pydantic>=2.0
12
12
  Requires-Dist: python-frontmatter>=1.0
13
13
  Requires-Dist: pyyaml>=6.0
@@ -15,6 +15,7 @@ Requires-Dist: numpy>=1.26
15
15
  Requires-Dist: mordant>=0.9
16
16
  Requires-Dist: mcp>=2.0
17
17
  Requires-Dist: fasteners>=0.19
18
+ Requires-Dist: Pillow>=11
18
19
  Provides-Extra: dev
19
20
  Requires-Dist: pytest>=8.0; extra == "dev"
20
21
  Provides-Extra: cpu
@@ -25,7 +26,7 @@ Provides-Extra: pdf
25
26
  Requires-Dist: bobine>=0.5; extra == "pdf"
26
27
  Dynamic: license-file
27
28
 
28
- # OKFgraph 0.6.0
29
+ # OKFgraph
29
30
 
30
31
  [![PyPI](https://img.shields.io/pypi/v/okfgraph)](https://pypi.org/project/okfgraph/)
31
32
  [![Python](https://img.shields.io/badge/python-%3E%3D3.11-blue)](https://www.python.org/)
@@ -57,14 +58,14 @@ a swappable `DocumentConverter` seam. What isn't needed isn't installed.
57
58
  | **Embeddings** | Jina v5 (default `…-text-small-retrieval`, `--model` selects registry: text-nano) via the `embroider` Rust wheel; last-token pooling, Matryoshka truncation (default 512, per-model ladder); device `auto` (CUDA→FP16 mirror weights, CPU→FP32, `--precision` pins incl. explicit `int8`); CPU arena off by default (`--cpu-arena`); token ceiling configurable (`--max-length`); model + precision pinned per graph, never mixed. No torch anywhere (0.7.0+) |
58
59
  | **Search** | Hybrid RRF fusion (vector + FTS) at concept and chunk granularity; `rank=none\|hub\|ppr` — including **PPR**: lexical seeds → exact Personalized PageRank, zero model load, deterministic |
59
60
  | **Read** | Body / chunks / rebuilt document / graph context, with optional **token budgets** (`max_tokens`): self first, then PPR-ranked neighbours, index-first for context |
60
- | **Storage** | LadybugDB `==0.20.3` (pinned — newer 0.20.x segfaults index builds): graph + vector + FTS in one file |
61
+ | **Storage** | LadybugDB `==0.21.2` (pinned — 0.20.x segfaulted 2nd in-process vector-index builds): graph + vector + FTS in one file |
61
62
  | **Links** | Path links (`](doc.md)`) + `[[wikilinks]]` resolved by name (`uid` → `aliases` → `title` → filename stem); ambiguous names never resolve; broken links tracked and repairable |
62
- | **Import** | Single files, whole bundles (delta-aware: only changed files re-embed, `--purge` drops deleted concepts), markdown / PDF / raw thoughts; mordant lint on the way in |
63
+ | **Import** | Single files, whole bundles (delta-aware: only changed files re-embed, `--purge` drops deleted concepts), markdown / PDF+Office / raw thoughts; mordant lint on the way in |
63
64
  | **PDF** | `DocumentConverter` seam with `BobineConverter` default (routing `auto\|surgical\|always\|never`); bring your own converter, no code changes |
64
65
  | **Diff** | `okf diff`: structural snapshot (dir vs dir, no model load) and drift (graph vs dir) modes; CI exit codes + `--json` |
65
66
  | **Doctor** | `okf doctor`: 0–100 health score (broken/orphan/stale/duplicate-title/missing-description), safe `--fix` that never touches `reviewed: true`, `--strict` CI gate |
66
67
  | **Export** | OKF round-trip with See Also / Cited By enrichment + index files, or `--flavor obsidian` (`[[Title]]` wikilinks, no index files, edge-lossless re-import) |
67
- | **Images** | Caption-based (`mode=text` only since 0.7.0), content-hash dedup, `okf-asset://` protocol |
68
+ | **Images** | Caption-based (`mode=text`) or image-content (`mode=optional`/`omni` via the ONNX vision model, needs a text-nano graph; 0.8.0+) content-hash dedup, `okf-asset://` protocol |
68
69
  | **MCP** | 5 tools (`search`, `read`, `traverse`, `ingest`, `export_bundle`), MCP ≥ 2.0 (`MCPServer` + `ToolAnnotations`), stdio transport |
69
70
  | **CLI** | Same 5 verbs plus maintenance (`init`, `import`, `diff`, `doctor`, `shell`, `reindex`, `broken-links`, `deleted-*`, …), slim per-command help, `okfgraph.toml` config |
70
71
  | **Skills** | 3 harness-neutral skills (`okfgraph-mcp`, `okfgraph-cli`, `okfgraph-ingest`), `.mcp.json` wiring included |
@@ -90,7 +91,12 @@ uv sync --extra pdf # bobine PDF converter
90
91
  uv sync --extra dev # pytest
91
92
  ```
92
93
 
93
- Core dependencies are deliberately few: `ladybug==0.20.3`, `embroider>=0.2,<0.3`
94
+ Prefer a system-wide ONNX Runtime (conda, distro package, self-built)?
95
+ Skip both extras and point `ORT_DYLIB_PATH` at your onnxruntime 1.29
96
+ library instead — but never rely on bare OS-loader discovery: a stale
97
+ system DLL (e.g. in `System32`) fails the load instead of being used.
98
+
99
+ Core dependencies are deliberately few: `ladybug==0.21.2`, `embroider>=0.3,<0.4`
94
100
  (the shared embedding engine — github.com/opticsWolf/embroider, also used by
95
101
  bobine), `onnxruntime==1.29.0` (one pinned ORT binary shared by bobine +
96
102
  embroider; `ORT_DYLIB_PATH`-overridable), `mordant`, `mcp>=2.0`, `pydantic`,
@@ -111,6 +117,7 @@ okf traverse savanna --relationship LINKS_TO --direction BOTH
111
117
  okf diff # drift: graph vs bundle dir
112
118
  okf doctor # health score + findings
113
119
  okf lint kb/ # pre-import gate: frontmatter + links, no model
120
+ okf produce --from sqlite --source shop.db --output kb/ # SQLite → bundle (lint pre-flighted)
114
121
  ```
115
122
 
116
123
  ### Programmatic usage
@@ -332,7 +339,7 @@ okfgraph/
332
339
  │ ├── cli.py # okf: 5 verbs + maintenance, slim help, okfgraph.toml
333
340
  │ ├── mcp_server.py # okf-mcp: 5 tools, MCP ≥ 2.0, lifespan-managed router
334
341
  │ ├── config.py # okfgraph.toml + env + CLI merge
335
- │ ├── images.py # IngestMode (text-only since 0.7.0), planning helpers
342
+ │ ├── images.py # IngestMode (text + vision-onnx since 0.8.0), planning helpers
336
343
  │ ├── security.py # SSRF/domain guards for remote images
337
344
  │ ├── tools.py # legacy tool definitions (superseded by mcp_server)
338
345
  │ └── components/ # ranking, links, search, lint, import_, export, diff, doctor,
@@ -341,7 +348,7 @@ okfgraph/
341
348
  ├── tests/fixtures/ # conformance corpus: ppr, diff, doctor, obsidian, bundles
342
349
  ├── docs/ # converters, harness-integration, plan-retrieval-roundup, diagnostics…
343
350
  ├── .mcp.json # ready MCP wiring (uv run --project . okf-mcp)
344
- ├── architecture.md # long-form architecture spec (v6.0, as-built for 0.2.12)
351
+ ├── architecture.md # long-form architecture spec (v6.5, as-built for 0.8.x)
345
352
  └── pyproject.toml # slim core deps + pdf/dev extras
346
353
  ```
347
354
 
@@ -349,7 +356,7 @@ okfgraph/
349
356
 
350
357
  ## Requirements
351
358
 
352
- Core (`uv sync`): `ladybug==0.20.3`, `embroider>=0.2,<0.3` (PyPI wheels,
359
+ Core (`uv sync`): `ladybug==0.21.2`, `embroider>=0.3,<0.4` (PyPI wheels,
353
360
  github.com/opticsWolf/embroider), `mordant>=0.9`,
354
361
  `mcp>=2.0`, `pydantic>=2`, `pyyaml>=6`,
355
362
  `numpy>=1.26`, `python-frontmatter>=1`, `fasteners>=0.19`. Python ≥ 3.11.
@@ -374,6 +381,7 @@ Check them before any commercial use.
374
381
  | Text embeddings (fp16) | `opticsWolf/jina-embeddings-v5-text-small-retrieval-onnx-fp16` (ONNX conversion of the above) | embroider | CC BY-NC 4.0 |
375
382
  | Text embeddings (nano) | `jinaai/jina-embeddings-v5-text-nano-retrieval` | embroider | CC BY-NC 4.0 |
376
383
  | Image embeddings (≤0.6.x `--extra omni`, removed 0.7.0) | `jinaai/jina-embeddings-v5-omni-small-retrieval` | sentence-transformers (torch path, removed) | CC BY-NC 4.0 |
384
+ | Image embeddings (vision, 0.8.0+) | `opticsWolf/jina-embeddings-v5-omni-nano-retrieval-onnx` (dynamic-grid ONNX export of `jinaai/jina-embeddings-v5-omni-nano-retrieval`) | embroider (`JinaV5Vision`) | CC BY-NC 4.0 |
377
385
  | PDF layout (`--extra pdf`) | `wybxc/DocLayout-YOLO-DocStructBench-onnx` | bobine | Apache-2.0 |
378
386
  | PDF OCR det/rec (`--extra pdf`) | `SWHL/RapidOCR` (PP-OCRv4) | bobine | Apache-2.0 |
379
387
  | PDF tables (`--extra pdf`) | `opendatalab/PDF-Extract-Kit-1.0` (`models/TabRec/SlanetPlus/slanet-plus.onnx`) | bobine | repo declares AGPL-3.0 |
@@ -1,4 +1,4 @@
1
- # OKFgraph 0.6.0
1
+ # OKFgraph
2
2
 
3
3
  [![PyPI](https://img.shields.io/pypi/v/okfgraph)](https://pypi.org/project/okfgraph/)
4
4
  [![Python](https://img.shields.io/badge/python-%3E%3D3.11-blue)](https://www.python.org/)
@@ -30,14 +30,14 @@ a swappable `DocumentConverter` seam. What isn't needed isn't installed.
30
30
  | **Embeddings** | Jina v5 (default `…-text-small-retrieval`, `--model` selects registry: text-nano) via the `embroider` Rust wheel; last-token pooling, Matryoshka truncation (default 512, per-model ladder); device `auto` (CUDA→FP16 mirror weights, CPU→FP32, `--precision` pins incl. explicit `int8`); CPU arena off by default (`--cpu-arena`); token ceiling configurable (`--max-length`); model + precision pinned per graph, never mixed. No torch anywhere (0.7.0+) |
31
31
  | **Search** | Hybrid RRF fusion (vector + FTS) at concept and chunk granularity; `rank=none\|hub\|ppr` — including **PPR**: lexical seeds → exact Personalized PageRank, zero model load, deterministic |
32
32
  | **Read** | Body / chunks / rebuilt document / graph context, with optional **token budgets** (`max_tokens`): self first, then PPR-ranked neighbours, index-first for context |
33
- | **Storage** | LadybugDB `==0.20.3` (pinned — newer 0.20.x segfaults index builds): graph + vector + FTS in one file |
33
+ | **Storage** | LadybugDB `==0.21.2` (pinned — 0.20.x segfaulted 2nd in-process vector-index builds): graph + vector + FTS in one file |
34
34
  | **Links** | Path links (`](doc.md)`) + `[[wikilinks]]` resolved by name (`uid` → `aliases` → `title` → filename stem); ambiguous names never resolve; broken links tracked and repairable |
35
- | **Import** | Single files, whole bundles (delta-aware: only changed files re-embed, `--purge` drops deleted concepts), markdown / PDF / raw thoughts; mordant lint on the way in |
35
+ | **Import** | Single files, whole bundles (delta-aware: only changed files re-embed, `--purge` drops deleted concepts), markdown / PDF+Office / raw thoughts; mordant lint on the way in |
36
36
  | **PDF** | `DocumentConverter` seam with `BobineConverter` default (routing `auto\|surgical\|always\|never`); bring your own converter, no code changes |
37
37
  | **Diff** | `okf diff`: structural snapshot (dir vs dir, no model load) and drift (graph vs dir) modes; CI exit codes + `--json` |
38
38
  | **Doctor** | `okf doctor`: 0–100 health score (broken/orphan/stale/duplicate-title/missing-description), safe `--fix` that never touches `reviewed: true`, `--strict` CI gate |
39
39
  | **Export** | OKF round-trip with See Also / Cited By enrichment + index files, or `--flavor obsidian` (`[[Title]]` wikilinks, no index files, edge-lossless re-import) |
40
- | **Images** | Caption-based (`mode=text` only since 0.7.0), content-hash dedup, `okf-asset://` protocol |
40
+ | **Images** | Caption-based (`mode=text`) or image-content (`mode=optional`/`omni` via the ONNX vision model, needs a text-nano graph; 0.8.0+) content-hash dedup, `okf-asset://` protocol |
41
41
  | **MCP** | 5 tools (`search`, `read`, `traverse`, `ingest`, `export_bundle`), MCP ≥ 2.0 (`MCPServer` + `ToolAnnotations`), stdio transport |
42
42
  | **CLI** | Same 5 verbs plus maintenance (`init`, `import`, `diff`, `doctor`, `shell`, `reindex`, `broken-links`, `deleted-*`, …), slim per-command help, `okfgraph.toml` config |
43
43
  | **Skills** | 3 harness-neutral skills (`okfgraph-mcp`, `okfgraph-cli`, `okfgraph-ingest`), `.mcp.json` wiring included |
@@ -63,7 +63,12 @@ uv sync --extra pdf # bobine PDF converter
63
63
  uv sync --extra dev # pytest
64
64
  ```
65
65
 
66
- Core dependencies are deliberately few: `ladybug==0.20.3`, `embroider>=0.2,<0.3`
66
+ Prefer a system-wide ONNX Runtime (conda, distro package, self-built)?
67
+ Skip both extras and point `ORT_DYLIB_PATH` at your onnxruntime 1.29
68
+ library instead — but never rely on bare OS-loader discovery: a stale
69
+ system DLL (e.g. in `System32`) fails the load instead of being used.
70
+
71
+ Core dependencies are deliberately few: `ladybug==0.21.2`, `embroider>=0.3,<0.4`
67
72
  (the shared embedding engine — github.com/opticsWolf/embroider, also used by
68
73
  bobine), `onnxruntime==1.29.0` (one pinned ORT binary shared by bobine +
69
74
  embroider; `ORT_DYLIB_PATH`-overridable), `mordant`, `mcp>=2.0`, `pydantic`,
@@ -84,6 +89,7 @@ okf traverse savanna --relationship LINKS_TO --direction BOTH
84
89
  okf diff # drift: graph vs bundle dir
85
90
  okf doctor # health score + findings
86
91
  okf lint kb/ # pre-import gate: frontmatter + links, no model
92
+ okf produce --from sqlite --source shop.db --output kb/ # SQLite → bundle (lint pre-flighted)
87
93
  ```
88
94
 
89
95
  ### Programmatic usage
@@ -305,7 +311,7 @@ okfgraph/
305
311
  │ ├── cli.py # okf: 5 verbs + maintenance, slim help, okfgraph.toml
306
312
  │ ├── mcp_server.py # okf-mcp: 5 tools, MCP ≥ 2.0, lifespan-managed router
307
313
  │ ├── config.py # okfgraph.toml + env + CLI merge
308
- │ ├── images.py # IngestMode (text-only since 0.7.0), planning helpers
314
+ │ ├── images.py # IngestMode (text + vision-onnx since 0.8.0), planning helpers
309
315
  │ ├── security.py # SSRF/domain guards for remote images
310
316
  │ ├── tools.py # legacy tool definitions (superseded by mcp_server)
311
317
  │ └── components/ # ranking, links, search, lint, import_, export, diff, doctor,
@@ -314,7 +320,7 @@ okfgraph/
314
320
  ├── tests/fixtures/ # conformance corpus: ppr, diff, doctor, obsidian, bundles
315
321
  ├── docs/ # converters, harness-integration, plan-retrieval-roundup, diagnostics…
316
322
  ├── .mcp.json # ready MCP wiring (uv run --project . okf-mcp)
317
- ├── architecture.md # long-form architecture spec (v6.0, as-built for 0.2.12)
323
+ ├── architecture.md # long-form architecture spec (v6.5, as-built for 0.8.x)
318
324
  └── pyproject.toml # slim core deps + pdf/dev extras
319
325
  ```
320
326
 
@@ -322,7 +328,7 @@ okfgraph/
322
328
 
323
329
  ## Requirements
324
330
 
325
- Core (`uv sync`): `ladybug==0.20.3`, `embroider>=0.2,<0.3` (PyPI wheels,
331
+ Core (`uv sync`): `ladybug==0.21.2`, `embroider>=0.3,<0.4` (PyPI wheels,
326
332
  github.com/opticsWolf/embroider), `mordant>=0.9`,
327
333
  `mcp>=2.0`, `pydantic>=2`, `pyyaml>=6`,
328
334
  `numpy>=1.26`, `python-frontmatter>=1`, `fasteners>=0.19`. Python ≥ 3.11.
@@ -347,6 +353,7 @@ Check them before any commercial use.
347
353
  | Text embeddings (fp16) | `opticsWolf/jina-embeddings-v5-text-small-retrieval-onnx-fp16` (ONNX conversion of the above) | embroider | CC BY-NC 4.0 |
348
354
  | Text embeddings (nano) | `jinaai/jina-embeddings-v5-text-nano-retrieval` | embroider | CC BY-NC 4.0 |
349
355
  | Image embeddings (≤0.6.x `--extra omni`, removed 0.7.0) | `jinaai/jina-embeddings-v5-omni-small-retrieval` | sentence-transformers (torch path, removed) | CC BY-NC 4.0 |
356
+ | Image embeddings (vision, 0.8.0+) | `opticsWolf/jina-embeddings-v5-omni-nano-retrieval-onnx` (dynamic-grid ONNX export of `jinaai/jina-embeddings-v5-omni-nano-retrieval`) | embroider (`JinaV5Vision`) | CC BY-NC 4.0 |
350
357
  | PDF layout (`--extra pdf`) | `wybxc/DocLayout-YOLO-DocStructBench-onnx` | bobine | Apache-2.0 |
351
358
  | PDF OCR det/rec (`--extra pdf`) | `SWHL/RapidOCR` (PP-OCRv4) | bobine | Apache-2.0 |
352
359
  | PDF tables (`--extra pdf`) | `opendatalab/PDF-Extract-Kit-1.0` (`models/TabRec/SlanetPlus/slanet-plus.onnx`) | bobine | repo declares AGPL-3.0 |
@@ -139,6 +139,8 @@ def _add_global(parser, mark=True):
139
139
  _add("--cache-dir", default=None, help="HuggingFace model cache directory (default: ~/.cache/huggingface, or from okfgraph.toml)")
140
140
  _add("--device", default=None, choices=["auto", "cpu", "cuda"], help="Inference device: auto (CUDA when present) / cpu / cuda (default: auto, or from okfgraph.toml)")
141
141
  _add("--precision", default=None, choices=["auto", "fp32", "fp16", "int8"], help="Weight precision: auto follows device (CUDA->FP16, CPU->FP32); fp16 on CPU is >40x slower; int8 explicit only, needs a measured artifact (default: auto, or from okfgraph.toml)")
142
+ _add("--image-model", default=None, help="Vision model id for image-content search (default: embroider vision contract; needs a text-nano graph, or from okfgraph.toml). Switching forces a fresh reimport (fail-closed image pin).")
143
+ _add("--image-precision", default=None, choices=["auto", "fp32", "fp16"], help="Vision weight precision: auto follows device (CUDA->FP16, CPU->FP32); explicit fp16 on CPU fails fast (default: auto, or from okfgraph.toml)")
142
144
  _add("--cpu-arena", action="store_true", help="Enable the CPU arena allocator (default off: ~8x lower peak RSS for ~1.4x encode time)")
143
145
  _add("--chunk-size", type=int, default=None, help="Chunk size in words for overlap (default: 512, or from okfgraph.toml)")
144
146
  _add("--chunk-overlap", type=int, default=None, help="Overlap in words between chunks (default: 40, or from okfgraph.toml)")
@@ -175,7 +177,7 @@ def _router(args):
175
177
  # Build CLI args dict (only non-None values override config)
176
178
  cli_dict = {}
177
179
  for attr in ("db", "bundle", "primary", "model", "dim", "max_length", "cache_dir", "device",
178
- "precision", "cpu_arena",
180
+ "precision", "cpu_arena", "image_model", "image_precision",
179
181
  "chunk_size", "chunk_overlap",
180
182
  "no_chunking", "mode", "batch_size",
181
183
  "allow_remote_images", "wal_mode", "allowed_image_domains"):
@@ -216,6 +218,8 @@ def _router(args):
216
218
  device=config.embedding.device,
217
219
  precision=config.embedding.precision,
218
220
  cpu_arena=config.embedding.cpu_arena,
221
+ image_model_id=config.embedding.image_model_id,
222
+ image_precision=config.embedding.image_precision,
219
223
  allow_remote_images=config.import_config.allow_remote_images,
220
224
  allowed_image_domains=allowed_domains,
221
225
  chunk_size=config.import_config.chunk_size,
@@ -630,6 +634,54 @@ def _lint(args):
630
634
  return 0 if report["clean"] else 1
631
635
 
632
636
 
637
+ def _produce(args):
638
+ """Generate a bundle from a data source (bundle-hardening §3).
639
+
640
+ Router-free like lint (no DB, no model cold-boot): produce answers
641
+ "what would this source look like as concepts?" The emitted bundle is
642
+ pre-flighted with lint_bundle against the output root, so the report
643
+ below covers both generation and well-formedness.
644
+ Exit 0 = produced + lint-clean, 1 = produced but lint errors (or the
645
+ source is unreadable), 2 = usage (unknown producer, missing file).
646
+ """
647
+ from okfgraph.components.lint import lint_bundle
648
+ from okfgraph.components.producers import PRODUCERS, producer_for
649
+
650
+ which = getattr(args, "from_", None)
651
+ if which not in PRODUCERS:
652
+ print(f"[ERROR] unknown producer {which!r} "
653
+ f"(available: {sorted(PRODUCERS)})")
654
+ return 2
655
+ source = getattr(args, "source", None)
656
+ out = Path(str(getattr(args, "output", None) or "."))
657
+ try:
658
+ bundle = producer_for(which).produce(
659
+ source,
660
+ out,
661
+ prefix=getattr(args, "prefix", None) or None,
662
+ overwrite=bool(getattr(args, "overwrite", False)),
663
+ )
664
+ except FileNotFoundError as exc:
665
+ print(f"[ERROR] {exc}")
666
+ return 2
667
+ except (ValueError, FileExistsError) as exc:
668
+ print(f"[ERROR] {exc}")
669
+ return 1
670
+ print(f"[OK] {bundle.producer}: {bundle.files} file(s) "
671
+ f"from {bundle.source} → {bundle.root / bundle.prefix}/")
672
+ report = lint_bundle(out)
673
+ n_err, n_warn = len(report["errors"]), len(report["warnings"])
674
+ print(f"lint: {report['files']} file(s): {n_err} error(s), {n_warn} warning(s)")
675
+ for e in report["errors"]:
676
+ print(f" [ERROR] {e['file']} {e['rule']}: {e['message']}")
677
+ for w in report["warnings"]:
678
+ print(f" [warn] {w['file']} {w['rule']}: {w['message']}")
679
+ if report["clean"]:
680
+ print("Bundle is lint-clean (safe to import).")
681
+ return 0
682
+ return 1
683
+
684
+
633
685
  def _diff(args):
634
686
  """Structural diff: snapshot (dir vs dir) or drift (graph vs dir).
635
687
 
@@ -1212,9 +1264,9 @@ def build_parser():
1212
1264
  "configured root; with --bundle, only that tree)")
1213
1265
  p.add_argument("--batch-size", type=int, default=32, help="Batch size for encoding (default: 32)")
1214
1266
  p.add_argument(
1215
- "--mode", default="text", choices=["text"],
1216
- help="Image ingestion mode: text (alt-text/filename caption). "
1217
- "The torch-backed optional/omni routes were removed in 0.7.0.",
1267
+ "--mode", default="text", choices=["text", "optional", "omni"],
1268
+ help="Image ingestion mode: text (captions), optional (vision for "
1269
+ "images lacking alt-text), omni (vision for every image).",
1218
1270
  )
1219
1271
  p.add_argument(
1220
1272
  "--purge", action="store_true", default=False,
@@ -1280,10 +1332,10 @@ def build_parser():
1280
1332
  p.add_argument("--kind", required=True, choices=["md", "pdf", "thoughts"],
1281
1333
  help="What to ingest")
1282
1334
  p.add_argument("--md-file", default=None, help="Markdown file (--kind md)")
1283
- p.add_argument("--pdf-file", default=None, help="PDF file (--kind pdf)")
1335
+ p.add_argument("--pdf-file", default=None, help="File to convert (--kind pdf): PDF or Office (docx/xlsx/pptx, legacy doc/xls/ppt)")
1284
1336
  p.add_argument("--thoughts", default=None, help="Raw reasoning text (--kind thoughts)")
1285
1337
  p.add_argument("--topic", default=None, help="Topic (--kind thoughts)")
1286
- p.add_argument("--concept-id", default=None, help="Explicit concept ID (--kind md/thoughts)")
1338
+ p.add_argument("--concept-id", default=None, help="Explicit concept ID (--kind md/thoughts; slashes are namespaces; thoughts default to thoughts/<topic>/<ts>_<id>)")
1287
1339
  p.add_argument("--title", default=None, help="Title override (--kind md)")
1288
1340
  p.add_argument("--description", default=None, help="Description override (--kind md)")
1289
1341
  p.add_argument("--tags", default=None, help="Comma-separated tags (--kind md/thoughts)")
@@ -1301,8 +1353,8 @@ def build_parser():
1301
1353
  help="ONNX routing mode (--kind pdf, default: auto)",
1302
1354
  )
1303
1355
  p.add_argument(
1304
- "--mode", default="text", choices=["text"],
1305
- help="Image ingestion mode: text (caption-based; optional/omni removed in 0.7.0)",
1356
+ "--mode", default="text", choices=["text", "optional", "omni"],
1357
+ help="Image ingestion mode: text (captions), optional/omni (ONNX vision).",
1306
1358
  )
1307
1359
  p.add_argument(
1308
1360
  "--batch-size", type=int, default=32,
@@ -1367,6 +1419,22 @@ def build_parser():
1367
1419
  help="Bundle directory (default: --bundle, okfgraph.toml, or .)")
1368
1420
  p.add_argument("--json", action="store_true", help="Machine-readable report")
1369
1421
 
1422
+ # produce
1423
+ p = sub.add_parser("produce", help="Generate a bundle from a data source")
1424
+ _add_global(p)
1425
+ _add_logging_flags(p)
1426
+ p.add_argument("--from", dest="from_", required=True,
1427
+ help="Producer name (today: sqlite)")
1428
+ p.add_argument("--source", required=True,
1429
+ help="Source to read (sqlite: path to the .db file)")
1430
+ p.add_argument("--output", default=None,
1431
+ help="Bundle root to write under (default: .); files go "
1432
+ "to <output>/<prefix>/ so lint + import agree on ids")
1433
+ p.add_argument("--prefix", default=None,
1434
+ help="Namespace directory (default: the producer's own)")
1435
+ p.add_argument("--overwrite", action="store_true",
1436
+ help="Rewrite existing producer output files")
1437
+
1370
1438
  # shell
1371
1439
  p = sub.add_parser("shell", help="Interactive REPL")
1372
1440
  _add_global(p)
@@ -1459,6 +1527,7 @@ def main():
1459
1527
  "diff": _diff,
1460
1528
  "doctor": _doctor,
1461
1529
  "lint": _lint,
1530
+ "produce": _produce,
1462
1531
  "reindex": _reindex,
1463
1532
  "deleted-list": _deleted_list,
1464
1533
  "deleted-recover": _deleted_recover,
@@ -219,6 +219,12 @@ class DiffManager:
219
219
  alias = self._alias_for(bundle_dir)
220
220
  return self.compare(old_state, state_of_dir(bundle_dir, alias))
221
221
  if not self.roots:
222
+ if self.primary is None:
223
+ raise ValueError(
224
+ "diff_db_dir needs a bundle root: this router was "
225
+ "opened without bundle_root (file-free mode). Pass an "
226
+ "explicit directory to drift against."
227
+ )
222
228
  return self.compare(old_state, state_of_dir(self.primary))
223
229
  # Union state: parse every present tree, resolve links once
224
230
  # against the combined index (cross-root links are not drift).
@@ -198,6 +198,37 @@ class DoctorManager:
198
198
  "omni route — re-import to re-embed by caption",
199
199
  })
200
200
 
201
+ # Image-route report (0.8.0): counts per embed route plus the
202
+ # pinned image model/precision. Missing tables (pre-image graphs)
203
+ # report as unpinned, never as findings.
204
+ try:
205
+ route_rows = self.conn.execute(
206
+ "MATCH (i:ImageAsset) RETURN i.embed_route AS route, count(*) AS n"
207
+ ).rows_as_dict().get_all()
208
+ except Exception:
209
+ route_rows = []
210
+ if route_rows:
211
+ counts = ", ".join(
212
+ f"{r['route']}={r['n']}" for r in sorted(route_rows, key=lambda r: r["route"] or "")
213
+ )
214
+ try:
215
+ im = self.conn.execute(
216
+ "MATCH (m:MetaText) WHERE m.key = 'embedding_image_model' "
217
+ "RETURN m.value AS v"
218
+ ).rows_as_dict().get_all()
219
+ ip = self.conn.execute(
220
+ "MATCH (m:Meta) WHERE m.key = 'embedding_image_precision' "
221
+ "RETURN m.value AS v"
222
+ ).rows_as_dict().get_all()
223
+ except Exception:
224
+ im, ip = [], []
225
+ model = im[0]["v"] if im else "(unpinned)"
226
+ prec = {16: "fp16", 32: "fp32"}.get(ip[0]["v"], "(unpinned)") if ip else "(unpinned)"
227
+ info.append({
228
+ "rule": "image_routes",
229
+ "message": f"image assets: {counts}; image model={model} precision={prec}",
230
+ })
231
+
201
232
  # ORT runtime report (Phase 3, plan-onnx-only): which wheel provides
202
233
  # the shared binary, and whether both are installed (loader picks one
203
234
  # silently — hard warning, cpu XOR gpu).