okfgraph 0.7.0__tar.gz → 0.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {okfgraph-0.7.0 → okfgraph-0.9.0}/PKG-INFO +19 -11
- {okfgraph-0.7.0 → okfgraph-0.9.0}/README.md +15 -8
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/cli.py +77 -8
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/diff.py +6 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/doctor.py +31 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/embedding.py +276 -2
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/image_assets.py +106 -12
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/import_.py +57 -11
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/ingest.py +21 -4
- okfgraph-0.9.0/okfgraph/components/producers.py +515 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/config.py +31 -4
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/images.py +83 -37
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/mcp_server.py +3 -3
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/router.py +97 -14
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/tools.py +10 -8
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/PKG-INFO +19 -11
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/SOURCES.txt +7 -1
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/requires.txt +3 -2
- {okfgraph-0.7.0 → okfgraph-0.9.0}/pyproject.toml +11 -5
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_config.py +3 -5
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_gpu_integration.py +17 -14
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_images.py +55 -24
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_lazy_encoder.py +8 -3
- okfgraph-0.9.0/tests/test_observations.py +204 -0
- okfgraph-0.9.0/tests/test_produce.py +211 -0
- okfgraph-0.9.0/tests/test_rootless.py +133 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_router.py +2 -2
- okfgraph-0.9.0/tests/test_thought_ids.py +132 -0
- okfgraph-0.9.0/tests/test_vision.py +210 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/LICENSE +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/__init__.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/__init__.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/converters.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/delta.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/export.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/links.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/lint.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/purge.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/ranking.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/roots.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/schema.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/components/search.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/models.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph/security.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/dependency_links.txt +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/entry_points.txt +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/okfgraph.egg-info/top_level.txt +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/setup.cfg +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_chunk_cap.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_chunk_search.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_chunking.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_cli.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_converter.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_delta.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_detach.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_diff.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_directory_hash.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_doctor.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_encode_bucketing.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_error_paths.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_explicit_files.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_export_compliance.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_golden_vectors.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_graph_enrichment.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_image_ingest.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ingest.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ingest_tool.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_integration.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_lint.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_logging.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_max_length.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_mcp_server.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_mcp_stdio.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_model.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_models.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_multiroot.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_obsidian.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_okf_ingest_tool.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ort.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_packaging.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_parity.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_pdf_e2e.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_phase0.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ppr_search.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_precision.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_ranking.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_reconstruction.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_reserved.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_retrieval_conformance.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_roundup_cli.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_router_misc.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_rust_backend.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_rust_e2e.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_sanitize.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_search_browser.py +0 -0
- {okfgraph-0.7.0 → okfgraph-0.9.0}/tests/test_security.py +0 -0
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: okfgraph
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.9.0
|
|
4
4
|
Summary: Ladybug-backed OKF knowledge graph with ONNX + Jina v5 embeddings
|
|
5
5
|
License-Expression: Apache-2.0 OR MIT
|
|
6
6
|
Requires-Python: >=3.11
|
|
7
7
|
Description-Content-Type: text/markdown
|
|
8
8
|
License-File: LICENSE
|
|
9
|
-
Requires-Dist: ladybug==0.
|
|
10
|
-
Requires-Dist: embroider<0.
|
|
9
|
+
Requires-Dist: ladybug==0.21.2
|
|
10
|
+
Requires-Dist: embroider<0.4,>=0.3.0
|
|
11
11
|
Requires-Dist: pydantic>=2.0
|
|
12
12
|
Requires-Dist: python-frontmatter>=1.0
|
|
13
13
|
Requires-Dist: pyyaml>=6.0
|
|
@@ -15,6 +15,7 @@ Requires-Dist: numpy>=1.26
|
|
|
15
15
|
Requires-Dist: mordant>=0.9
|
|
16
16
|
Requires-Dist: mcp>=2.0
|
|
17
17
|
Requires-Dist: fasteners>=0.19
|
|
18
|
+
Requires-Dist: Pillow>=11
|
|
18
19
|
Provides-Extra: dev
|
|
19
20
|
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
20
21
|
Provides-Extra: cpu
|
|
@@ -25,7 +26,7 @@ Provides-Extra: pdf
|
|
|
25
26
|
Requires-Dist: bobine>=0.5; extra == "pdf"
|
|
26
27
|
Dynamic: license-file
|
|
27
28
|
|
|
28
|
-
# OKFgraph
|
|
29
|
+
# OKFgraph
|
|
29
30
|
|
|
30
31
|
[](https://pypi.org/project/okfgraph/)
|
|
31
32
|
[](https://www.python.org/)
|
|
@@ -57,14 +58,14 @@ a swappable `DocumentConverter` seam. What isn't needed isn't installed.
|
|
|
57
58
|
| **Embeddings** | Jina v5 (default `…-text-small-retrieval`, `--model` selects registry: text-nano) via the `embroider` Rust wheel; last-token pooling, Matryoshka truncation (default 512, per-model ladder); device `auto` (CUDA→FP16 mirror weights, CPU→FP32, `--precision` pins incl. explicit `int8`); CPU arena off by default (`--cpu-arena`); token ceiling configurable (`--max-length`); model + precision pinned per graph, never mixed. No torch anywhere (0.7.0+) |
|
|
58
59
|
| **Search** | Hybrid RRF fusion (vector + FTS) at concept and chunk granularity; `rank=none\|hub\|ppr` — including **PPR**: lexical seeds → exact Personalized PageRank, zero model load, deterministic |
|
|
59
60
|
| **Read** | Body / chunks / rebuilt document / graph context, with optional **token budgets** (`max_tokens`): self first, then PPR-ranked neighbours, index-first for context |
|
|
60
|
-
| **Storage** | LadybugDB `==0.
|
|
61
|
+
| **Storage** | LadybugDB `==0.21.2` (pinned — 0.20.x segfaulted 2nd in-process vector-index builds): graph + vector + FTS in one file |
|
|
61
62
|
| **Links** | Path links (`](doc.md)`) + `[[wikilinks]]` resolved by name (`uid` → `aliases` → `title` → filename stem); ambiguous names never resolve; broken links tracked and repairable |
|
|
62
|
-
| **Import** | Single files, whole bundles (delta-aware: only changed files re-embed, `--purge` drops deleted concepts), markdown / PDF / raw thoughts; mordant lint on the way in |
|
|
63
|
+
| **Import** | Single files, whole bundles (delta-aware: only changed files re-embed, `--purge` drops deleted concepts), markdown / PDF+Office / raw thoughts; mordant lint on the way in |
|
|
63
64
|
| **PDF** | `DocumentConverter` seam with `BobineConverter` default (routing `auto\|surgical\|always\|never`); bring your own converter, no code changes |
|
|
64
65
|
| **Diff** | `okf diff`: structural snapshot (dir vs dir, no model load) and drift (graph vs dir) modes; CI exit codes + `--json` |
|
|
65
66
|
| **Doctor** | `okf doctor`: 0–100 health score (broken/orphan/stale/duplicate-title/missing-description), safe `--fix` that never touches `reviewed: true`, `--strict` CI gate |
|
|
66
67
|
| **Export** | OKF round-trip with See Also / Cited By enrichment + index files, or `--flavor obsidian` (`[[Title]]` wikilinks, no index files, edge-lossless re-import) |
|
|
67
|
-
| **Images** | Caption-based (`mode=text`
|
|
68
|
+
| **Images** | Caption-based (`mode=text`) or image-content (`mode=optional`/`omni` via the ONNX vision model, needs a text-nano graph; 0.8.0+) content-hash dedup, `okf-asset://` protocol |
|
|
68
69
|
| **MCP** | 5 tools (`search`, `read`, `traverse`, `ingest`, `export_bundle`), MCP ≥ 2.0 (`MCPServer` + `ToolAnnotations`), stdio transport |
|
|
69
70
|
| **CLI** | Same 5 verbs plus maintenance (`init`, `import`, `diff`, `doctor`, `shell`, `reindex`, `broken-links`, `deleted-*`, …), slim per-command help, `okfgraph.toml` config |
|
|
70
71
|
| **Skills** | 3 harness-neutral skills (`okfgraph-mcp`, `okfgraph-cli`, `okfgraph-ingest`), `.mcp.json` wiring included |
|
|
@@ -90,7 +91,12 @@ uv sync --extra pdf # bobine PDF converter
|
|
|
90
91
|
uv sync --extra dev # pytest
|
|
91
92
|
```
|
|
92
93
|
|
|
93
|
-
|
|
94
|
+
Prefer a system-wide ONNX Runtime (conda, distro package, self-built)?
|
|
95
|
+
Skip both extras and point `ORT_DYLIB_PATH` at your onnxruntime 1.29
|
|
96
|
+
library instead — but never rely on bare OS-loader discovery: a stale
|
|
97
|
+
system DLL (e.g. in `System32`) fails the load instead of being used.
|
|
98
|
+
|
|
99
|
+
Core dependencies are deliberately few: `ladybug==0.21.2`, `embroider>=0.3,<0.4`
|
|
94
100
|
(the shared embedding engine — github.com/opticsWolf/embroider, also used by
|
|
95
101
|
bobine), `onnxruntime==1.29.0` (one pinned ORT binary shared by bobine +
|
|
96
102
|
embroider; `ORT_DYLIB_PATH`-overridable), `mordant`, `mcp>=2.0`, `pydantic`,
|
|
@@ -111,6 +117,7 @@ okf traverse savanna --relationship LINKS_TO --direction BOTH
|
|
|
111
117
|
okf diff # drift: graph vs bundle dir
|
|
112
118
|
okf doctor # health score + findings
|
|
113
119
|
okf lint kb/ # pre-import gate: frontmatter + links, no model
|
|
120
|
+
okf produce --from sqlite --source shop.db --output kb/ # SQLite → bundle (lint pre-flighted)
|
|
114
121
|
```
|
|
115
122
|
|
|
116
123
|
### Programmatic usage
|
|
@@ -332,7 +339,7 @@ okfgraph/
|
|
|
332
339
|
│ ├── cli.py # okf: 5 verbs + maintenance, slim help, okfgraph.toml
|
|
333
340
|
│ ├── mcp_server.py # okf-mcp: 5 tools, MCP ≥ 2.0, lifespan-managed router
|
|
334
341
|
│ ├── config.py # okfgraph.toml + env + CLI merge
|
|
335
|
-
│ ├── images.py # IngestMode (text-
|
|
342
|
+
│ ├── images.py # IngestMode (text + vision-onnx since 0.8.0), planning helpers
|
|
336
343
|
│ ├── security.py # SSRF/domain guards for remote images
|
|
337
344
|
│ ├── tools.py # legacy tool definitions (superseded by mcp_server)
|
|
338
345
|
│ └── components/ # ranking, links, search, lint, import_, export, diff, doctor,
|
|
@@ -341,7 +348,7 @@ okfgraph/
|
|
|
341
348
|
├── tests/fixtures/ # conformance corpus: ppr, diff, doctor, obsidian, bundles
|
|
342
349
|
├── docs/ # converters, harness-integration, plan-retrieval-roundup, diagnostics…
|
|
343
350
|
├── .mcp.json # ready MCP wiring (uv run --project . okf-mcp)
|
|
344
|
-
├── architecture.md # long-form architecture spec (v6.
|
|
351
|
+
├── architecture.md # long-form architecture spec (v6.5, as-built for 0.8.x)
|
|
345
352
|
└── pyproject.toml # slim core deps + pdf/dev extras
|
|
346
353
|
```
|
|
347
354
|
|
|
@@ -349,7 +356,7 @@ okfgraph/
|
|
|
349
356
|
|
|
350
357
|
## Requirements
|
|
351
358
|
|
|
352
|
-
Core (`uv sync`): `ladybug==0.
|
|
359
|
+
Core (`uv sync`): `ladybug==0.21.2`, `embroider>=0.3,<0.4` (PyPI wheels,
|
|
353
360
|
github.com/opticsWolf/embroider), `mordant>=0.9`,
|
|
354
361
|
`mcp>=2.0`, `pydantic>=2`, `pyyaml>=6`,
|
|
355
362
|
`numpy>=1.26`, `python-frontmatter>=1`, `fasteners>=0.19`. Python ≥ 3.11.
|
|
@@ -374,6 +381,7 @@ Check them before any commercial use.
|
|
|
374
381
|
| Text embeddings (fp16) | `opticsWolf/jina-embeddings-v5-text-small-retrieval-onnx-fp16` (ONNX conversion of the above) | embroider | CC BY-NC 4.0 |
|
|
375
382
|
| Text embeddings (nano) | `jinaai/jina-embeddings-v5-text-nano-retrieval` | embroider | CC BY-NC 4.0 |
|
|
376
383
|
| Image embeddings (≤0.6.x `--extra omni`, removed 0.7.0) | `jinaai/jina-embeddings-v5-omni-small-retrieval` | sentence-transformers (torch path, removed) | CC BY-NC 4.0 |
|
|
384
|
+
| Image embeddings (vision, 0.8.0+) | `opticsWolf/jina-embeddings-v5-omni-nano-retrieval-onnx` (dynamic-grid ONNX export of `jinaai/jina-embeddings-v5-omni-nano-retrieval`) | embroider (`JinaV5Vision`) | CC BY-NC 4.0 |
|
|
377
385
|
| PDF layout (`--extra pdf`) | `wybxc/DocLayout-YOLO-DocStructBench-onnx` | bobine | Apache-2.0 |
|
|
378
386
|
| PDF OCR det/rec (`--extra pdf`) | `SWHL/RapidOCR` (PP-OCRv4) | bobine | Apache-2.0 |
|
|
379
387
|
| PDF tables (`--extra pdf`) | `opendatalab/PDF-Extract-Kit-1.0` (`models/TabRec/SlanetPlus/slanet-plus.onnx`) | bobine | repo declares AGPL-3.0 |
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# OKFgraph
|
|
1
|
+
# OKFgraph
|
|
2
2
|
|
|
3
3
|
[](https://pypi.org/project/okfgraph/)
|
|
4
4
|
[](https://www.python.org/)
|
|
@@ -30,14 +30,14 @@ a swappable `DocumentConverter` seam. What isn't needed isn't installed.
|
|
|
30
30
|
| **Embeddings** | Jina v5 (default `…-text-small-retrieval`, `--model` selects registry: text-nano) via the `embroider` Rust wheel; last-token pooling, Matryoshka truncation (default 512, per-model ladder); device `auto` (CUDA→FP16 mirror weights, CPU→FP32, `--precision` pins incl. explicit `int8`); CPU arena off by default (`--cpu-arena`); token ceiling configurable (`--max-length`); model + precision pinned per graph, never mixed. No torch anywhere (0.7.0+) |
|
|
31
31
|
| **Search** | Hybrid RRF fusion (vector + FTS) at concept and chunk granularity; `rank=none\|hub\|ppr` — including **PPR**: lexical seeds → exact Personalized PageRank, zero model load, deterministic |
|
|
32
32
|
| **Read** | Body / chunks / rebuilt document / graph context, with optional **token budgets** (`max_tokens`): self first, then PPR-ranked neighbours, index-first for context |
|
|
33
|
-
| **Storage** | LadybugDB `==0.
|
|
33
|
+
| **Storage** | LadybugDB `==0.21.2` (pinned — 0.20.x segfaulted 2nd in-process vector-index builds): graph + vector + FTS in one file |
|
|
34
34
|
| **Links** | Path links (`](doc.md)`) + `[[wikilinks]]` resolved by name (`uid` → `aliases` → `title` → filename stem); ambiguous names never resolve; broken links tracked and repairable |
|
|
35
|
-
| **Import** | Single files, whole bundles (delta-aware: only changed files re-embed, `--purge` drops deleted concepts), markdown / PDF / raw thoughts; mordant lint on the way in |
|
|
35
|
+
| **Import** | Single files, whole bundles (delta-aware: only changed files re-embed, `--purge` drops deleted concepts), markdown / PDF+Office / raw thoughts; mordant lint on the way in |
|
|
36
36
|
| **PDF** | `DocumentConverter` seam with `BobineConverter` default (routing `auto\|surgical\|always\|never`); bring your own converter, no code changes |
|
|
37
37
|
| **Diff** | `okf diff`: structural snapshot (dir vs dir, no model load) and drift (graph vs dir) modes; CI exit codes + `--json` |
|
|
38
38
|
| **Doctor** | `okf doctor`: 0–100 health score (broken/orphan/stale/duplicate-title/missing-description), safe `--fix` that never touches `reviewed: true`, `--strict` CI gate |
|
|
39
39
|
| **Export** | OKF round-trip with See Also / Cited By enrichment + index files, or `--flavor obsidian` (`[[Title]]` wikilinks, no index files, edge-lossless re-import) |
|
|
40
|
-
| **Images** | Caption-based (`mode=text`
|
|
40
|
+
| **Images** | Caption-based (`mode=text`) or image-content (`mode=optional`/`omni` via the ONNX vision model, needs a text-nano graph; 0.8.0+) content-hash dedup, `okf-asset://` protocol |
|
|
41
41
|
| **MCP** | 5 tools (`search`, `read`, `traverse`, `ingest`, `export_bundle`), MCP ≥ 2.0 (`MCPServer` + `ToolAnnotations`), stdio transport |
|
|
42
42
|
| **CLI** | Same 5 verbs plus maintenance (`init`, `import`, `diff`, `doctor`, `shell`, `reindex`, `broken-links`, `deleted-*`, …), slim per-command help, `okfgraph.toml` config |
|
|
43
43
|
| **Skills** | 3 harness-neutral skills (`okfgraph-mcp`, `okfgraph-cli`, `okfgraph-ingest`), `.mcp.json` wiring included |
|
|
@@ -63,7 +63,12 @@ uv sync --extra pdf # bobine PDF converter
|
|
|
63
63
|
uv sync --extra dev # pytest
|
|
64
64
|
```
|
|
65
65
|
|
|
66
|
-
|
|
66
|
+
Prefer a system-wide ONNX Runtime (conda, distro package, self-built)?
|
|
67
|
+
Skip both extras and point `ORT_DYLIB_PATH` at your onnxruntime 1.29
|
|
68
|
+
library instead — but never rely on bare OS-loader discovery: a stale
|
|
69
|
+
system DLL (e.g. in `System32`) fails the load instead of being used.
|
|
70
|
+
|
|
71
|
+
Core dependencies are deliberately few: `ladybug==0.21.2`, `embroider>=0.3,<0.4`
|
|
67
72
|
(the shared embedding engine — github.com/opticsWolf/embroider, also used by
|
|
68
73
|
bobine), `onnxruntime==1.29.0` (one pinned ORT binary shared by bobine +
|
|
69
74
|
embroider; `ORT_DYLIB_PATH`-overridable), `mordant`, `mcp>=2.0`, `pydantic`,
|
|
@@ -84,6 +89,7 @@ okf traverse savanna --relationship LINKS_TO --direction BOTH
|
|
|
84
89
|
okf diff # drift: graph vs bundle dir
|
|
85
90
|
okf doctor # health score + findings
|
|
86
91
|
okf lint kb/ # pre-import gate: frontmatter + links, no model
|
|
92
|
+
okf produce --from sqlite --source shop.db --output kb/ # SQLite → bundle (lint pre-flighted)
|
|
87
93
|
```
|
|
88
94
|
|
|
89
95
|
### Programmatic usage
|
|
@@ -305,7 +311,7 @@ okfgraph/
|
|
|
305
311
|
│ ├── cli.py # okf: 5 verbs + maintenance, slim help, okfgraph.toml
|
|
306
312
|
│ ├── mcp_server.py # okf-mcp: 5 tools, MCP ≥ 2.0, lifespan-managed router
|
|
307
313
|
│ ├── config.py # okfgraph.toml + env + CLI merge
|
|
308
|
-
│ ├── images.py # IngestMode (text-
|
|
314
|
+
│ ├── images.py # IngestMode (text + vision-onnx since 0.8.0), planning helpers
|
|
309
315
|
│ ├── security.py # SSRF/domain guards for remote images
|
|
310
316
|
│ ├── tools.py # legacy tool definitions (superseded by mcp_server)
|
|
311
317
|
│ └── components/ # ranking, links, search, lint, import_, export, diff, doctor,
|
|
@@ -314,7 +320,7 @@ okfgraph/
|
|
|
314
320
|
├── tests/fixtures/ # conformance corpus: ppr, diff, doctor, obsidian, bundles
|
|
315
321
|
├── docs/ # converters, harness-integration, plan-retrieval-roundup, diagnostics…
|
|
316
322
|
├── .mcp.json # ready MCP wiring (uv run --project . okf-mcp)
|
|
317
|
-
├── architecture.md # long-form architecture spec (v6.
|
|
323
|
+
├── architecture.md # long-form architecture spec (v6.5, as-built for 0.8.x)
|
|
318
324
|
└── pyproject.toml # slim core deps + pdf/dev extras
|
|
319
325
|
```
|
|
320
326
|
|
|
@@ -322,7 +328,7 @@ okfgraph/
|
|
|
322
328
|
|
|
323
329
|
## Requirements
|
|
324
330
|
|
|
325
|
-
Core (`uv sync`): `ladybug==0.
|
|
331
|
+
Core (`uv sync`): `ladybug==0.21.2`, `embroider>=0.3,<0.4` (PyPI wheels,
|
|
326
332
|
github.com/opticsWolf/embroider), `mordant>=0.9`,
|
|
327
333
|
`mcp>=2.0`, `pydantic>=2`, `pyyaml>=6`,
|
|
328
334
|
`numpy>=1.26`, `python-frontmatter>=1`, `fasteners>=0.19`. Python ≥ 3.11.
|
|
@@ -347,6 +353,7 @@ Check them before any commercial use.
|
|
|
347
353
|
| Text embeddings (fp16) | `opticsWolf/jina-embeddings-v5-text-small-retrieval-onnx-fp16` (ONNX conversion of the above) | embroider | CC BY-NC 4.0 |
|
|
348
354
|
| Text embeddings (nano) | `jinaai/jina-embeddings-v5-text-nano-retrieval` | embroider | CC BY-NC 4.0 |
|
|
349
355
|
| Image embeddings (≤0.6.x `--extra omni`, removed 0.7.0) | `jinaai/jina-embeddings-v5-omni-small-retrieval` | sentence-transformers (torch path, removed) | CC BY-NC 4.0 |
|
|
356
|
+
| Image embeddings (vision, 0.8.0+) | `opticsWolf/jina-embeddings-v5-omni-nano-retrieval-onnx` (dynamic-grid ONNX export of `jinaai/jina-embeddings-v5-omni-nano-retrieval`) | embroider (`JinaV5Vision`) | CC BY-NC 4.0 |
|
|
350
357
|
| PDF layout (`--extra pdf`) | `wybxc/DocLayout-YOLO-DocStructBench-onnx` | bobine | Apache-2.0 |
|
|
351
358
|
| PDF OCR det/rec (`--extra pdf`) | `SWHL/RapidOCR` (PP-OCRv4) | bobine | Apache-2.0 |
|
|
352
359
|
| PDF tables (`--extra pdf`) | `opendatalab/PDF-Extract-Kit-1.0` (`models/TabRec/SlanetPlus/slanet-plus.onnx`) | bobine | repo declares AGPL-3.0 |
|
|
@@ -139,6 +139,8 @@ def _add_global(parser, mark=True):
|
|
|
139
139
|
_add("--cache-dir", default=None, help="HuggingFace model cache directory (default: ~/.cache/huggingface, or from okfgraph.toml)")
|
|
140
140
|
_add("--device", default=None, choices=["auto", "cpu", "cuda"], help="Inference device: auto (CUDA when present) / cpu / cuda (default: auto, or from okfgraph.toml)")
|
|
141
141
|
_add("--precision", default=None, choices=["auto", "fp32", "fp16", "int8"], help="Weight precision: auto follows device (CUDA->FP16, CPU->FP32); fp16 on CPU is >40x slower; int8 explicit only, needs a measured artifact (default: auto, or from okfgraph.toml)")
|
|
142
|
+
_add("--image-model", default=None, help="Vision model id for image-content search (default: embroider vision contract; needs a text-nano graph, or from okfgraph.toml). Switching forces a fresh reimport (fail-closed image pin).")
|
|
143
|
+
_add("--image-precision", default=None, choices=["auto", "fp32", "fp16"], help="Vision weight precision: auto follows device (CUDA->FP16, CPU->FP32); explicit fp16 on CPU fails fast (default: auto, or from okfgraph.toml)")
|
|
142
144
|
_add("--cpu-arena", action="store_true", help="Enable the CPU arena allocator (default off: ~8x lower peak RSS for ~1.4x encode time)")
|
|
143
145
|
_add("--chunk-size", type=int, default=None, help="Chunk size in words for overlap (default: 512, or from okfgraph.toml)")
|
|
144
146
|
_add("--chunk-overlap", type=int, default=None, help="Overlap in words between chunks (default: 40, or from okfgraph.toml)")
|
|
@@ -175,7 +177,7 @@ def _router(args):
|
|
|
175
177
|
# Build CLI args dict (only non-None values override config)
|
|
176
178
|
cli_dict = {}
|
|
177
179
|
for attr in ("db", "bundle", "primary", "model", "dim", "max_length", "cache_dir", "device",
|
|
178
|
-
"precision", "cpu_arena",
|
|
180
|
+
"precision", "cpu_arena", "image_model", "image_precision",
|
|
179
181
|
"chunk_size", "chunk_overlap",
|
|
180
182
|
"no_chunking", "mode", "batch_size",
|
|
181
183
|
"allow_remote_images", "wal_mode", "allowed_image_domains"):
|
|
@@ -216,6 +218,8 @@ def _router(args):
|
|
|
216
218
|
device=config.embedding.device,
|
|
217
219
|
precision=config.embedding.precision,
|
|
218
220
|
cpu_arena=config.embedding.cpu_arena,
|
|
221
|
+
image_model_id=config.embedding.image_model_id,
|
|
222
|
+
image_precision=config.embedding.image_precision,
|
|
219
223
|
allow_remote_images=config.import_config.allow_remote_images,
|
|
220
224
|
allowed_image_domains=allowed_domains,
|
|
221
225
|
chunk_size=config.import_config.chunk_size,
|
|
@@ -630,6 +634,54 @@ def _lint(args):
|
|
|
630
634
|
return 0 if report["clean"] else 1
|
|
631
635
|
|
|
632
636
|
|
|
637
|
+
def _produce(args):
|
|
638
|
+
"""Generate a bundle from a data source (bundle-hardening §3).
|
|
639
|
+
|
|
640
|
+
Router-free like lint (no DB, no model cold-boot): produce answers
|
|
641
|
+
"what would this source look like as concepts?" The emitted bundle is
|
|
642
|
+
pre-flighted with lint_bundle against the output root, so the report
|
|
643
|
+
below covers both generation and well-formedness.
|
|
644
|
+
Exit 0 = produced + lint-clean, 1 = produced but lint errors (or the
|
|
645
|
+
source is unreadable), 2 = usage (unknown producer, missing file).
|
|
646
|
+
"""
|
|
647
|
+
from okfgraph.components.lint import lint_bundle
|
|
648
|
+
from okfgraph.components.producers import PRODUCERS, producer_for
|
|
649
|
+
|
|
650
|
+
which = getattr(args, "from_", None)
|
|
651
|
+
if which not in PRODUCERS:
|
|
652
|
+
print(f"[ERROR] unknown producer {which!r} "
|
|
653
|
+
f"(available: {sorted(PRODUCERS)})")
|
|
654
|
+
return 2
|
|
655
|
+
source = getattr(args, "source", None)
|
|
656
|
+
out = Path(str(getattr(args, "output", None) or "."))
|
|
657
|
+
try:
|
|
658
|
+
bundle = producer_for(which).produce(
|
|
659
|
+
source,
|
|
660
|
+
out,
|
|
661
|
+
prefix=getattr(args, "prefix", None) or None,
|
|
662
|
+
overwrite=bool(getattr(args, "overwrite", False)),
|
|
663
|
+
)
|
|
664
|
+
except FileNotFoundError as exc:
|
|
665
|
+
print(f"[ERROR] {exc}")
|
|
666
|
+
return 2
|
|
667
|
+
except (ValueError, FileExistsError) as exc:
|
|
668
|
+
print(f"[ERROR] {exc}")
|
|
669
|
+
return 1
|
|
670
|
+
print(f"[OK] {bundle.producer}: {bundle.files} file(s) "
|
|
671
|
+
f"from {bundle.source} → {bundle.root / bundle.prefix}/")
|
|
672
|
+
report = lint_bundle(out)
|
|
673
|
+
n_err, n_warn = len(report["errors"]), len(report["warnings"])
|
|
674
|
+
print(f"lint: {report['files']} file(s): {n_err} error(s), {n_warn} warning(s)")
|
|
675
|
+
for e in report["errors"]:
|
|
676
|
+
print(f" [ERROR] {e['file']} {e['rule']}: {e['message']}")
|
|
677
|
+
for w in report["warnings"]:
|
|
678
|
+
print(f" [warn] {w['file']} {w['rule']}: {w['message']}")
|
|
679
|
+
if report["clean"]:
|
|
680
|
+
print("Bundle is lint-clean (safe to import).")
|
|
681
|
+
return 0
|
|
682
|
+
return 1
|
|
683
|
+
|
|
684
|
+
|
|
633
685
|
def _diff(args):
|
|
634
686
|
"""Structural diff: snapshot (dir vs dir) or drift (graph vs dir).
|
|
635
687
|
|
|
@@ -1212,9 +1264,9 @@ def build_parser():
|
|
|
1212
1264
|
"configured root; with --bundle, only that tree)")
|
|
1213
1265
|
p.add_argument("--batch-size", type=int, default=32, help="Batch size for encoding (default: 32)")
|
|
1214
1266
|
p.add_argument(
|
|
1215
|
-
"--mode", default="text", choices=["text"],
|
|
1216
|
-
help="Image ingestion mode: text (
|
|
1217
|
-
"
|
|
1267
|
+
"--mode", default="text", choices=["text", "optional", "omni"],
|
|
1268
|
+
help="Image ingestion mode: text (captions), optional (vision for "
|
|
1269
|
+
"images lacking alt-text), omni (vision for every image).",
|
|
1218
1270
|
)
|
|
1219
1271
|
p.add_argument(
|
|
1220
1272
|
"--purge", action="store_true", default=False,
|
|
@@ -1280,10 +1332,10 @@ def build_parser():
|
|
|
1280
1332
|
p.add_argument("--kind", required=True, choices=["md", "pdf", "thoughts"],
|
|
1281
1333
|
help="What to ingest")
|
|
1282
1334
|
p.add_argument("--md-file", default=None, help="Markdown file (--kind md)")
|
|
1283
|
-
p.add_argument("--pdf-file", default=None, help="
|
|
1335
|
+
p.add_argument("--pdf-file", default=None, help="File to convert (--kind pdf): PDF or Office (docx/xlsx/pptx, legacy doc/xls/ppt)")
|
|
1284
1336
|
p.add_argument("--thoughts", default=None, help="Raw reasoning text (--kind thoughts)")
|
|
1285
1337
|
p.add_argument("--topic", default=None, help="Topic (--kind thoughts)")
|
|
1286
|
-
p.add_argument("--concept-id", default=None, help="Explicit concept ID (--kind md/thoughts)")
|
|
1338
|
+
p.add_argument("--concept-id", default=None, help="Explicit concept ID (--kind md/thoughts; slashes are namespaces; thoughts default to thoughts/<topic>/<ts>_<id>)")
|
|
1287
1339
|
p.add_argument("--title", default=None, help="Title override (--kind md)")
|
|
1288
1340
|
p.add_argument("--description", default=None, help="Description override (--kind md)")
|
|
1289
1341
|
p.add_argument("--tags", default=None, help="Comma-separated tags (--kind md/thoughts)")
|
|
@@ -1301,8 +1353,8 @@ def build_parser():
|
|
|
1301
1353
|
help="ONNX routing mode (--kind pdf, default: auto)",
|
|
1302
1354
|
)
|
|
1303
1355
|
p.add_argument(
|
|
1304
|
-
"--mode", default="text", choices=["text"],
|
|
1305
|
-
help="Image ingestion mode: text (
|
|
1356
|
+
"--mode", default="text", choices=["text", "optional", "omni"],
|
|
1357
|
+
help="Image ingestion mode: text (captions), optional/omni (ONNX vision).",
|
|
1306
1358
|
)
|
|
1307
1359
|
p.add_argument(
|
|
1308
1360
|
"--batch-size", type=int, default=32,
|
|
@@ -1367,6 +1419,22 @@ def build_parser():
|
|
|
1367
1419
|
help="Bundle directory (default: --bundle, okfgraph.toml, or .)")
|
|
1368
1420
|
p.add_argument("--json", action="store_true", help="Machine-readable report")
|
|
1369
1421
|
|
|
1422
|
+
# produce
|
|
1423
|
+
p = sub.add_parser("produce", help="Generate a bundle from a data source")
|
|
1424
|
+
_add_global(p)
|
|
1425
|
+
_add_logging_flags(p)
|
|
1426
|
+
p.add_argument("--from", dest="from_", required=True,
|
|
1427
|
+
help="Producer name (today: sqlite)")
|
|
1428
|
+
p.add_argument("--source", required=True,
|
|
1429
|
+
help="Source to read (sqlite: path to the .db file)")
|
|
1430
|
+
p.add_argument("--output", default=None,
|
|
1431
|
+
help="Bundle root to write under (default: .); files go "
|
|
1432
|
+
"to <output>/<prefix>/ so lint + import agree on ids")
|
|
1433
|
+
p.add_argument("--prefix", default=None,
|
|
1434
|
+
help="Namespace directory (default: the producer's own)")
|
|
1435
|
+
p.add_argument("--overwrite", action="store_true",
|
|
1436
|
+
help="Rewrite existing producer output files")
|
|
1437
|
+
|
|
1370
1438
|
# shell
|
|
1371
1439
|
p = sub.add_parser("shell", help="Interactive REPL")
|
|
1372
1440
|
_add_global(p)
|
|
@@ -1459,6 +1527,7 @@ def main():
|
|
|
1459
1527
|
"diff": _diff,
|
|
1460
1528
|
"doctor": _doctor,
|
|
1461
1529
|
"lint": _lint,
|
|
1530
|
+
"produce": _produce,
|
|
1462
1531
|
"reindex": _reindex,
|
|
1463
1532
|
"deleted-list": _deleted_list,
|
|
1464
1533
|
"deleted-recover": _deleted_recover,
|
|
@@ -219,6 +219,12 @@ class DiffManager:
|
|
|
219
219
|
alias = self._alias_for(bundle_dir)
|
|
220
220
|
return self.compare(old_state, state_of_dir(bundle_dir, alias))
|
|
221
221
|
if not self.roots:
|
|
222
|
+
if self.primary is None:
|
|
223
|
+
raise ValueError(
|
|
224
|
+
"diff_db_dir needs a bundle root: this router was "
|
|
225
|
+
"opened without bundle_root (file-free mode). Pass an "
|
|
226
|
+
"explicit directory to drift against."
|
|
227
|
+
)
|
|
222
228
|
return self.compare(old_state, state_of_dir(self.primary))
|
|
223
229
|
# Union state: parse every present tree, resolve links once
|
|
224
230
|
# against the combined index (cross-root links are not drift).
|
|
@@ -198,6 +198,37 @@ class DoctorManager:
|
|
|
198
198
|
"omni route — re-import to re-embed by caption",
|
|
199
199
|
})
|
|
200
200
|
|
|
201
|
+
# Image-route report (0.8.0): counts per embed route plus the
|
|
202
|
+
# pinned image model/precision. Missing tables (pre-image graphs)
|
|
203
|
+
# report as unpinned, never as findings.
|
|
204
|
+
try:
|
|
205
|
+
route_rows = self.conn.execute(
|
|
206
|
+
"MATCH (i:ImageAsset) RETURN i.embed_route AS route, count(*) AS n"
|
|
207
|
+
).rows_as_dict().get_all()
|
|
208
|
+
except Exception:
|
|
209
|
+
route_rows = []
|
|
210
|
+
if route_rows:
|
|
211
|
+
counts = ", ".join(
|
|
212
|
+
f"{r['route']}={r['n']}" for r in sorted(route_rows, key=lambda r: r["route"] or "")
|
|
213
|
+
)
|
|
214
|
+
try:
|
|
215
|
+
im = self.conn.execute(
|
|
216
|
+
"MATCH (m:MetaText) WHERE m.key = 'embedding_image_model' "
|
|
217
|
+
"RETURN m.value AS v"
|
|
218
|
+
).rows_as_dict().get_all()
|
|
219
|
+
ip = self.conn.execute(
|
|
220
|
+
"MATCH (m:Meta) WHERE m.key = 'embedding_image_precision' "
|
|
221
|
+
"RETURN m.value AS v"
|
|
222
|
+
).rows_as_dict().get_all()
|
|
223
|
+
except Exception:
|
|
224
|
+
im, ip = [], []
|
|
225
|
+
model = im[0]["v"] if im else "(unpinned)"
|
|
226
|
+
prec = {16: "fp16", 32: "fp32"}.get(ip[0]["v"], "(unpinned)") if ip else "(unpinned)"
|
|
227
|
+
info.append({
|
|
228
|
+
"rule": "image_routes",
|
|
229
|
+
"message": f"image assets: {counts}; image model={model} precision={prec}",
|
|
230
|
+
})
|
|
231
|
+
|
|
201
232
|
# ORT runtime report (Phase 3, plan-onnx-only): which wheel provides
|
|
202
233
|
# the shared binary, and whether both are installed (loader picks one
|
|
203
234
|
# silently — hard warning, cpu XOR gpu).
|