okfgraph 0.2.4__tar.gz → 0.2.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- okfgraph-0.2.7/PKG-INFO +355 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/README.md +4 -3
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/cli.py +6 -3
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/import_.py +34 -20
- okfgraph-0.2.7/okfgraph.egg-info/PKG-INFO +355 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph.egg-info/SOURCES.txt +1 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/pyproject.toml +2 -1
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_chunking.py +17 -16
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_cli.py +8 -0
- okfgraph-0.2.7/tests/test_packaging.py +29 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_router.py +10 -6
- okfgraph-0.2.4/PKG-INFO +0 -25
- okfgraph-0.2.4/okfgraph.egg-info/PKG-INFO +0 -25
- {okfgraph-0.2.4 → okfgraph-0.2.7}/LICENSE +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/__init__.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/__init__.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/converters.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/delta.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/diff.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/doctor.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/embedding.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/export.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/image_assets.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/ingest.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/links.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/lint.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/purge.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/ranking.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/schema.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/components/search.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/config.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/images.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/mcp_server.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/models.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/router.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/security.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph/tools.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph.egg-info/dependency_links.txt +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph.egg-info/entry_points.txt +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph.egg-info/requires.txt +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/okfgraph.egg-info/top_level.txt +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/setup.cfg +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_chunk_search.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_config.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_converter.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_delta.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_diff.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_directory_hash.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_doctor.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_export_compliance.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_gpu_integration.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_graph_enrichment.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_images.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_ingest.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_ingest_tool.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_integration.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_lint.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_logging.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_mcp_server.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_models.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_obsidian.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_okf_ingest_tool.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_parity.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_pdf_e2e.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_ppr_search.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_ranking.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_reconstruction.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_reserved.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_roundup_cli.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_router_misc.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_rust_backend.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_rust_e2e.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_sanitize.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_search_browser.py +0 -0
- {okfgraph-0.2.4 → okfgraph-0.2.7}/tests/test_security.py +0 -0
okfgraph-0.2.7/PKG-INFO
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: okfgraph
|
|
3
|
+
Version: 0.2.7
|
|
4
|
+
Summary: Ladybug-backed OKF knowledge graph with ONNX + Jina v5 embeddings
|
|
5
|
+
License-Expression: Apache-2.0 OR MIT
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Dist: ladybug==0.20.3
|
|
10
|
+
Requires-Dist: okf-embed>=0.1
|
|
11
|
+
Requires-Dist: pydantic>=2.0
|
|
12
|
+
Requires-Dist: python-frontmatter>=1.0
|
|
13
|
+
Requires-Dist: pyyaml>=6.0
|
|
14
|
+
Requires-Dist: numpy>=1.26
|
|
15
|
+
Requires-Dist: onnxruntime==1.29.0
|
|
16
|
+
Requires-Dist: mordant>=0.9
|
|
17
|
+
Requires-Dist: mcp>=2.0
|
|
18
|
+
Requires-Dist: fasteners>=0.19
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=8.0; extra == "dev"
|
|
21
|
+
Provides-Extra: pdf
|
|
22
|
+
Requires-Dist: bobine>=0.5; extra == "pdf"
|
|
23
|
+
Provides-Extra: omni
|
|
24
|
+
Requires-Dist: sentence-transformers>=3.0; extra == "omni"
|
|
25
|
+
Requires-Dist: Pillow>=10.0; extra == "omni"
|
|
26
|
+
Dynamic: license-file
|
|
27
|
+
|
|
28
|
+
# OKFgraph 0.2.7
|
|
29
|
+
|
|
30
|
+
[](https://pypi.org/project/okfgraph/)
|
|
31
|
+
[](https://www.python.org/)
|
|
32
|
+
[](LICENSE)
|
|
33
|
+
[](https://modelcontextprotocol.io/)
|
|
34
|
+
[](https://pypi.org/project/ladybug/)
|
|
35
|
+
|
|
36
|
+
**Ladybug-backed knowledge graph with Rust-driven Jina v5 embeddings, model-free
|
|
37
|
+
graph retrieval, and agent-first MCP + CLI surfaces.**
|
|
38
|
+
|
|
39
|
+
OKFgraph is a Python library, CLI (`okf`), and MCP server (`okf-mcp`) for
|
|
40
|
+
building, querying, and maintaining a knowledge graph from Markdown/OKF
|
|
41
|
+
documents. It combines hybrid semantic search (vector + FTS, RRF-fused),
|
|
42
|
+
chunk-level retrieval, **model-free PPR retrieval that works with no embedding
|
|
43
|
+
model loaded**, structural diffing, scored health checks, and Obsidian-vault
|
|
44
|
+
compatible wikilinks — in a single LadybugDB file.
|
|
45
|
+
|
|
46
|
+
Design stance: **slim dependencies, no legacy fallbacks.** Embeddings run
|
|
47
|
+
through the `okf-embed` Rust wheel (ONNX Runtime, no torch / transformers /
|
|
48
|
+
optimum anywhere). PDF conversion runs through the `bobine` Rust engine behind
|
|
49
|
+
a swappable `DocumentConverter` seam. What isn't needed isn't installed.
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Features
|
|
54
|
+
|
|
55
|
+
| Category | Features |
|
|
56
|
+
|---|---|
|
|
57
|
+
| **Embeddings** | Jina v5 (`jina-embeddings-v5-text-small-retrieval`) via the `okf-embed` Rust wheel; last-token pooling, Matryoshka truncation (32–1024, default 512); omni model (`…-omni-small-retrieval`) lazy-loaded for images only |
|
|
58
|
+
| **Search** | Hybrid RRF fusion (vector + FTS) at concept and chunk granularity; `rank=none\|hub\|ppr` — including **PPR**: lexical seeds → exact Personalized PageRank, zero model load, deterministic |
|
|
59
|
+
| **Read** | Body / chunks / rebuilt document / graph context, with optional **token budgets** (`max_tokens`): self first, then PPR-ranked neighbours, index-first for context |
|
|
60
|
+
| **Storage** | LadybugDB `==0.20.3` (pinned — newer 0.20.x segfaults index builds): graph + vector + FTS in one file |
|
|
61
|
+
| **Links** | Path links (`](doc.md)`) + `[[wikilinks]]` resolved by name (`uid` → `aliases` → `title` → filename stem); ambiguous names never resolve; broken links tracked and repairable |
|
|
62
|
+
| **Import** | Single files, whole bundles (delta-aware: only changed files re-embed, `--purge` drops deleted concepts), markdown / PDF / raw thoughts; mordant lint on the way in |
|
|
63
|
+
| **PDF** | `DocumentConverter` seam with `BobineConverter` default (routing `auto\|surgical\|always\|never`); bring your own converter, no code changes |
|
|
64
|
+
| **Diff** | `okf diff`: structural snapshot (dir vs dir, no model load) and drift (graph vs dir) modes; CI exit codes + `--json` |
|
|
65
|
+
| **Doctor** | `okf doctor`: 0–100 health score (broken/orphan/stale/duplicate-title/missing-description), safe `--fix` that never touches `reviewed: true`, `--strict` CI gate |
|
|
66
|
+
| **Export** | OKF round-trip with See Also / Cited By enrichment + index files, or `--flavor obsidian` (`[[Title]]` wikilinks, no index files, edge-lossless re-import) |
|
|
67
|
+
| **Images** | Modes `text` / `optional` / `omni`, shared text+image vector space, content-hash dedup, `okf-asset://` protocol |
|
|
68
|
+
| **MCP** | 5 tools (`search`, `read`, `traverse`, `ingest`, `export_bundle`), MCP ≥ 2.0 (`MCPServer` + `ToolAnnotations`), stdio transport |
|
|
69
|
+
| **CLI** | Same 5 verbs plus maintenance (`init`, `import`, `diff`, `doctor`, `shell`, `reindex`, `broken-links`, `deleted-*`, …), slim per-command help, `okfgraph.toml` config |
|
|
70
|
+
| **Skills** | 3 harness-neutral skills (`okfgraph-mcp`, `okfgraph-cli`, `okfgraph-ingest`), `.mcp.json` wiring included |
|
|
71
|
+
|
|
72
|
+
---
|
|
73
|
+
|
|
74
|
+
## Installation
|
|
75
|
+
|
|
76
|
+
Requires Python ≥ 3.11.
|
|
77
|
+
|
|
78
|
+
```bash
|
|
79
|
+
pip install "okfgraph[pdf,omni]" # PyPI (okf-embed ships platform wheels; published)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Or from source with `uv` (also builds the `okf-embed` Rust wheel from
|
|
83
|
+
`rust/okf-embed` via `[tool.uv.sources]`):
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
git clone <repo> && cd OKFgraph
|
|
87
|
+
uv sync # core: ladybug, okf-embed, onnxruntime, mordant, mcp, …
|
|
88
|
+
uv sync --extra pdf # bobine PDF converter
|
|
89
|
+
uv sync --extra omni # sentence-transformers + Pillow (image embeddings)
|
|
90
|
+
uv sync --extra dev # pytest
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Core dependencies are deliberately few: `ladybug==0.20.3`, `okf-embed`,
|
|
94
|
+
`onnxruntime==1.29.0` (one pinned ORT binary shared by bobine + okf-embed;
|
|
95
|
+
`ORT_DYLIB_PATH`-overridable), `mordant`, `mcp>=2.0`, `pydantic`, `pyyaml`,
|
|
96
|
+
`numpy`, `python-frontmatter`, `fasteners`. There is no torch, no
|
|
97
|
+
transformers, no optimum, no PDF stack in the core install.
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## Quick start
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
okf init --db kb.db --bundle kb/ # once; persists to okfgraph.toml
|
|
105
|
+
okf import --all --bundle kb/ # delta-aware bulk import
|
|
106
|
+
okf search "honey badger defense" # hybrid semantic search
|
|
107
|
+
okf search --rank ppr "honey badger" # same question, no model load
|
|
108
|
+
okf read savanna --include context --max-tokens 1500
|
|
109
|
+
okf traverse savanna --relationship LINKS_TO --direction BOTH
|
|
110
|
+
okf diff # drift: graph vs bundle dir
|
|
111
|
+
okf doctor # health score + findings
|
|
112
|
+
okf lint kb/ # pre-import gate: frontmatter + links, no model
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
### Programmatic usage
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
from pathlib import Path
|
|
119
|
+
from okfgraph import OKFRouter
|
|
120
|
+
|
|
121
|
+
router = OKFRouter(db_path="kb.db", bundle_root="kb/", embedding_dim=512)
|
|
122
|
+
|
|
123
|
+
# Import: whole bundle (delta-aware), one file, or raw reasoning
|
|
124
|
+
ids = router.import_mgr.import_bundle(Path("kb/"), purge_deleted=False)
|
|
125
|
+
cid = router.import_from_okf(Path("kb/auth.md"), mode="text")
|
|
126
|
+
th = router.ingest_mgr.ingest_thoughts("Session decided X because Y", topic="auth-refactor")
|
|
127
|
+
|
|
128
|
+
# Search: hybrid (default), hub-blended, or model-free PPR
|
|
129
|
+
hits = router.search_hybrid("session auth decisions", limit=5)
|
|
130
|
+
hubbed = router.search_hybrid("session auth decisions", rank="hub")
|
|
131
|
+
cold = router.search_engine.search_with_ppr("auth refactor") # no ONNX load
|
|
132
|
+
|
|
133
|
+
# Read: full shapes, or a token-budgeted section list
|
|
134
|
+
doc = router.get_by_id("auth")
|
|
135
|
+
reading = router.search_engine.read_with_budget("auth", include="context", max_tokens=1500)
|
|
136
|
+
|
|
137
|
+
# Maintain: drift, health, links
|
|
138
|
+
print(router.diff_db_dir(Path("kb/"))["identical"]) # True when in sync
|
|
139
|
+
print(router.diagnose()["score"]) # 0-100
|
|
140
|
+
print(router.doctor_fix()) # safe repairs only
|
|
141
|
+
print(router.repair_links()) # re-point broken links
|
|
142
|
+
|
|
143
|
+
# Export: OKF bundle or Obsidian vault
|
|
144
|
+
router.export_mgr.export_bundle(Path("out-okf/"))
|
|
145
|
+
router.export_mgr.export_bundle(Path("out-vault/"), flavor="obsidian")
|
|
146
|
+
|
|
147
|
+
router.close()
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
### Ingestion kinds
|
|
151
|
+
|
|
152
|
+
| Kind | Entry point | Notes |
|
|
153
|
+
|---|---|---|
|
|
154
|
+
| `md` | `ingest_md(md_path, concept_id?, title?, tags?, mode?)` | mordant-linted; frontmatter wins over overrides |
|
|
155
|
+
| `pdf` | `ingest_pdf(pdf_path, auto_import?, routing_mode?, …)` | bobine converter; `md_path` is transient when auto-importing — verify by searching, not by reading the path |
|
|
156
|
+
| `thoughts` | `ingest_thoughts(text, topic, tags?)` | cheapest, highest value: persist session reasoning with a stable topic scheme (`auth-refactor`, `api-design`) |
|
|
157
|
+
|
|
158
|
+
Frontmatter that matters: `title`, `type`, `tags`, `aliases: [...]` (wikilink
|
|
159
|
+
names), `id:` (stable identity, preserved as `uid`, written back on export),
|
|
160
|
+
`reviewed: true` (immune to `doctor --fix`). Credentials in `resource:`
|
|
161
|
+
URIs are stripped to `***@` at parse time (graphs imported before 0.2.3
|
|
162
|
+
keep old values until re-import). Rule of thumb: thoughts > md >
|
|
163
|
+
pdf — reasoning you already hold beats re-extracting it from files.
|
|
164
|
+
|
|
165
|
+
---
|
|
166
|
+
|
|
167
|
+
## CLI reference
|
|
168
|
+
|
|
169
|
+
Five verbs mirror the MCP tools; maintenance commands cover the rest. Global
|
|
170
|
+
flags (`--db`, `--bundle`, `--dim`, …) are documented once in `okf --help`,
|
|
171
|
+
accepted everywhere, and usually live in `okfgraph.toml`.
|
|
172
|
+
|
|
173
|
+
| Command | Description |
|
|
174
|
+
|---|---|
|
|
175
|
+
| `okf search QUERY [--target concepts\|chunks\|images] [--rank none\|hub\|ppr] [--expand] [--hub-rerank]` | Concepts (RRF hybrid), chunks (passages), images; `--rank ppr` is model-free |
|
|
176
|
+
| `okf read ID [--include body\|chunks\|document\|context] [--max-tokens N]` | Full shapes uncapped; budgeted section list with `--max-tokens` |
|
|
177
|
+
| `okf traverse [ID] [--relationship CONTAINS\|LINKS_TO\|PART_OF\|INCLUDES_ASSET] [--direction …] [--target ID]` | Relationships, directory listing (empty ID = root), shortest path via `--target` |
|
|
178
|
+
| `okf ingest --kind md\|pdf\|thoughts …` | `--md-file`, `--pdf-file` (`--routing-mode`, `--auto-import`), `--thoughts --topic` |
|
|
179
|
+
| `okf export --all\|--concept-id ID --output DIR [--flavor okf\|obsidian]` | Bundle export; obsidian = `[[Title]]` links, no index files |
|
|
180
|
+
| `okf diff [OLD] [NEW] [--json]` | Snapshot (two dirs, no model) or drift (graph vs dir); exit 0 identical / 1 different |
|
|
181
|
+
| `okf lint [DIR] [--json]` | Pre-import gate (no DB, no model); exit 0 clean / 1 errors / 2 bad dir |
|
|
182
|
+
| `okf doctor [--fix] [--strict] [--stale-days N] [--json]` | Score + findings; `--fix` repairs safely, `--strict` exits 1 on any finding |
|
|
183
|
+
| `okf import [--all] [--purge] [--mode text\|optional\|omni]` | Bulk/single import, delta-aware |
|
|
184
|
+
| `okf init`, `okf model-info`, `okf shell`, `okf reindex`, `okf broken-links`, `okf repair-links`, `okf deleted-*` | Setup, cache inspection, REPL, index rebuild, link + soft-delete maintenance |
|
|
185
|
+
|
|
186
|
+
Every CLI call cold-boots the router (model load ~30s when the embedder is
|
|
187
|
+
needed) — batch reads, and reach for `--rank ppr` for topic queries in cold
|
|
188
|
+
sessions.
|
|
189
|
+
|
|
190
|
+
---
|
|
191
|
+
|
|
192
|
+
## MCP server
|
|
193
|
+
|
|
194
|
+
5 tools, MCP ≥ 2.0, stdio transport. Wire once per project with a **stable**
|
|
195
|
+
`--db-path` + `--bundle` (a temp dir means amnesia every session); first boot
|
|
196
|
+
creates the schema; pre-approve the `ingest` / `export_bundle` write tools for
|
|
197
|
+
knowledge workflows. `.mcp.json` ships a ready config (`uv run --project .
|
|
198
|
+
okf-mcp`).
|
|
199
|
+
|
|
200
|
+
```bash
|
|
201
|
+
okf-mcp --db-path ./kb.db --bundle-root ./kb --embedding-dim 1024
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
| Tool | Routing |
|
|
205
|
+
|---|---|
|
|
206
|
+
| `search(query, target?, rank?, …)` | `concepts` = open questions; `chunks` = exact passages (`expand`, `hub_rerank`); `images` = assets; `rank=ppr` = model-free topic search |
|
|
207
|
+
| `read(concept_id, include?, max_tokens?)` | Full shapes, or a budgeted section list |
|
|
208
|
+
| `traverse(start_id, relationship?, direction?, target?, …)` | Walk edges, list dirs, connect two concepts |
|
|
209
|
+
| `ingest(kind, …)` | `md` / `pdf` / `thoughts` with per-kind params |
|
|
210
|
+
| `export_bundle(output_dir, …, flavor?)` | `okf` or `obsidian` |
|
|
211
|
+
|
|
212
|
+
Verify wiring with any `search` — an empty graph returns `[]`, which still
|
|
213
|
+
proves the plumbing works.
|
|
214
|
+
|
|
215
|
+
### Skills
|
|
216
|
+
|
|
217
|
+
Harness-neutral skill sources live in `skills/` (Claude Code auto-loads
|
|
218
|
+
`.claude/skills/<name>/SKILL.md`; other harnesses have their own dirs — see
|
|
219
|
+
`docs/harness-integration.md`):
|
|
220
|
+
|
|
221
|
+
- **`okfgraph-mcp`** — finding knowledge via MCP tools (pick this *or* CLI).
|
|
222
|
+
- **`okfgraph-cli`** — same graph through `okf` shell commands.
|
|
223
|
+
- **`okfgraph-ingest`** — feeding discipline: what to store, which kind,
|
|
224
|
+
converter modes, wikilink/alias conventions. Install alongside either
|
|
225
|
+
finder when the agent should persist knowledge.
|
|
226
|
+
|
|
227
|
+
---
|
|
228
|
+
|
|
229
|
+
## Architecture
|
|
230
|
+
|
|
231
|
+
```
|
|
232
|
+
┌──────────── MCP (5 tools) / CLI (5 verbs + maintenance)
|
|
233
|
+
│ │
|
|
234
|
+
OKFRouter (facade: owns conn, encoder, lock; thin proxies)
|
|
235
|
+
│
|
|
236
|
+
┌────────────┼──────────────────────────────────────────┐
|
|
237
|
+
│ │ │
|
|
238
|
+
Import ──► Embed ──► Search ◄── ranking ──► Export ──► Doctor/Diff
|
|
239
|
+
│ │ │ (seeds+PPR) │ │
|
|
240
|
+
│ │ │ links.py │
|
|
241
|
+
Delta ──► Schema ──► ImageAssets ──► Purge ──► Ingest ──► Converters
|
|
242
|
+
│ (bobine seam)
|
|
243
|
+
└──────────────── LadybugDB (graph + vector + FTS, one file) ──┘
|
|
244
|
+
▲
|
|
245
|
+
Rust okf-embed (Jina v5, ORT) — tokenize, last-token
|
|
246
|
+
pool, L2-norm, Matryoshka truncate; numerics pinned
|
|
247
|
+
by tests/test_parity.py
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Components live in `okfgraph/components/` (one concern each, dependencies
|
|
251
|
+
injected): `search`, `ranking` (pure seeds + exact PPR), `links` (pure
|
|
252
|
+
extraction + name index), `import_` (batch/single upsert, delta-aware),
|
|
253
|
+
`export` (okf/obsidian flavors), `diff` (structural compare), `doctor`
|
|
254
|
+
(scored scan + safe fixes), `embedding` (Rust bridge + chunking),
|
|
255
|
+
`converters` (`DocumentConverter` protocol + `BobineConverter`),
|
|
256
|
+
`image_assets`, `delta`, `purge`, `schema`, `ingest`.
|
|
257
|
+
|
|
258
|
+
Key design decisions:
|
|
259
|
+
|
|
260
|
+
- **Rust-only embeddings, fail-fast** — no Python fallback; a mid-run stack
|
|
261
|
+
switch would silently mix vector spaces in one index.
|
|
262
|
+
- **Last-token pooling** (not mean) — required by Jina v5; mean pooling
|
|
263
|
+
breaks alignment with omni image embeddings.
|
|
264
|
+
- **Single pinned ORT** (`onnxruntime==1.29.0`, `ORT_DYLIB_PATH`-overridable)
|
|
265
|
+
shared by bobine + okf-embed.
|
|
266
|
+
- **Bobine is a plugin, not a dependency** — `DocumentConverter.convert()`
|
|
267
|
+
is the seam; provider owns its options (`routing_mode`,
|
|
268
|
+
`extract_images`); missing bobine → clear `RuntimeError`, never a silent
|
|
269
|
+
fallback path.
|
|
270
|
+
- **Deterministic by construction** — sorted traversal order, fixed PPR
|
|
271
|
+
constants and accumulation order, golden fixtures under `tests/fixtures/`
|
|
272
|
+
(PPR scores, diff reports, doctor scores, vault round-trips).
|
|
273
|
+
- **Bundle is source of truth, graph is the index** — edit markdown,
|
|
274
|
+
re-import; verify ingests by searching, since empty `[]` proves wiring
|
|
275
|
+
while errors prove broken setup.
|
|
276
|
+
- **MCP-first surface** — CLI mirrors the same 5 verbs; maintenance
|
|
277
|
+
(`diff`, `doctor`) is CLI-only to keep the agent tool surface at 5.
|
|
278
|
+
|
|
279
|
+
---
|
|
280
|
+
|
|
281
|
+
## API reference (`OKFRouter` proxies)
|
|
282
|
+
|
|
283
|
+
| Method | Description |
|
|
284
|
+
|---|---|
|
|
285
|
+
| `import_from_okf(file_path, mode?)` / `import_mgr.import_bundle(dir?, batch_size?, mode?, purge_deleted?)` | Single / bulk import |
|
|
286
|
+
| `ingest_mgr.ingest_md / ingest_pdf / ingest_thoughts` | Kind-based ingestion with lint + chunk + embed + link |
|
|
287
|
+
| `search_hybrid(query, …, rank="none"\|"hub"\|"ppr")` | RRF hybrid; `hub` blends authority, `ppr` is model-free |
|
|
288
|
+
| `search_engine.search_with_ppr / read_with_budget / search_chunks / traverse / find_path / get_chunks / get_by_id / list_directory` | Model-free PPR, budgeted reads, retrieval + navigation |
|
|
289
|
+
| `embed_engine.reconstruct_document(id)` / `count_tokens(text)` | ~98% chunk→markdown rebuild; Rust counter with chars/4 fallback |
|
|
290
|
+
| `export_mgr.export_bundle / export_to_okf(..., flavor="okf"\|"obsidian")` | Filtered export, both flavors |
|
|
291
|
+
| `diff_mgr.diff_dirs(old, new)` / `diff_db_dir(bundle_dir)` | Snapshot / drift structural reports |
|
|
292
|
+
| `diagnose(stale_days?)` / `doctor_fix()` | Health report / safe repairs |
|
|
293
|
+
| `list_broken_links()` / `repair_links(skip_sources?)` | Unresolved refs; exact-id + unique-name repair |
|
|
294
|
+
| `model_info(...)` / `default_cache_dir()` | Cache inspection without model load |
|
|
295
|
+
|
|
296
|
+
---
|
|
297
|
+
|
|
298
|
+
## Testing
|
|
299
|
+
|
|
300
|
+
```bash
|
|
301
|
+
uv run --project . pytest tests/ -q # full suite (model loads; takes a while)
|
|
302
|
+
uv run --project . pytest tests/test_ranking.py tests/test_mcp_server.py -q # fast subset
|
|
303
|
+
cd rust/okf-embed && cargo test --locked # Rust unit tests (pure, no model)
|
|
304
|
+
```
|
|
305
|
+
|
|
306
|
+
Golden fixtures under `tests/fixtures/` (`ppr_graph`, `diff_a`/`diff_b`,
|
|
307
|
+
`doctor_bundle`, `obsidian_vault`, `ppr_bundle`) lock deterministic behavior:
|
|
308
|
+
PPR scores byte-stable across runs, diff reports exact, doctor score pinned
|
|
309
|
+
(83 on the fixture bundle), obsidian export→import edge-identical. Live suites
|
|
310
|
+
reuse one class-scoped router each so the ONNX model loads once per class.
|
|
311
|
+
|
|
312
|
+
---
|
|
313
|
+
|
|
314
|
+
## Project structure
|
|
315
|
+
|
|
316
|
+
```text
|
|
317
|
+
okfgraph/
|
|
318
|
+
├── okfgraph/
|
|
319
|
+
│ ├── __init__.py # ConceptModel, OKFRouter, cli_main
|
|
320
|
+
│ ├── models.py # ConceptModel / ChunkModel / ImageAssetModel (extra frontmatter allowed)
|
|
321
|
+
│ ├── router.py # OKFRouter facade (owns resources, thin proxies)
|
|
322
|
+
│ ├── cli.py # okf: 5 verbs + maintenance, slim help, okfgraph.toml
|
|
323
|
+
│ ├── mcp_server.py # okf-mcp: 5 tools, MCP ≥ 2.0, lifespan-managed router
|
|
324
|
+
│ ├── config.py # okfgraph.toml + env + CLI merge
|
|
325
|
+
│ ├── images.py # IngestMode (text|optional|omni), planning helpers
|
|
326
|
+
│ ├── security.py # SSRF/domain guards for remote images
|
|
327
|
+
│ ├── tools.py # legacy tool definitions (superseded by mcp_server)
|
|
328
|
+
│ └── components/ # ranking, links, search, lint, import_, export, diff, doctor,
|
|
329
|
+
│ # embedding, converters, image_assets, delta, purge, schema, ingest
|
|
330
|
+
├── rust/okf-embed/ # Jina v5 Rust loader (ort, CUDA-opportunistic) + Python wheel
|
|
331
|
+
├── skills/ # okfgraph-mcp, okfgraph-cli, okfgraph-ingest (harness-neutral source)
|
|
332
|
+
├── tests/fixtures/ # conformance corpus: ppr, diff, doctor, obsidian, bundles
|
|
333
|
+
├── docs/ # converters, harness-integration, plan-retrieval-roundup, diagnostics…
|
|
334
|
+
├── .mcp.json # ready MCP wiring (uv run --project . okf-mcp)
|
|
335
|
+
├── architecture.md # long-form architecture spec
|
|
336
|
+
└── pyproject.toml # slim core deps + pdf/omni/dev extras
|
|
337
|
+
```
|
|
338
|
+
|
|
339
|
+
---
|
|
340
|
+
|
|
341
|
+
## Requirements
|
|
342
|
+
|
|
343
|
+
Core (`uv sync`): `ladybug==0.20.3`, `okf-embed` (built from `rust/okf-embed`),
|
|
344
|
+
`onnxruntime==1.29.0`, `mordant>=0.9`, `mcp>=2.0`, `pydantic>=2`, `pyyaml>=6`,
|
|
345
|
+
`numpy>=1.26`, `python-frontmatter>=1`, `fasteners>=0.19`. Python ≥ 3.11.
|
|
346
|
+
|
|
347
|
+
- `--extra pdf`: `bobine>=0.5` (default PDF converter).
|
|
348
|
+
- `--extra omni`: `sentence-transformers>=3`, `Pillow>=10` (image embeddings).
|
|
349
|
+
- `--extra dev`: `pytest>=8`.
|
|
350
|
+
|
|
351
|
+
---
|
|
352
|
+
|
|
353
|
+
## License
|
|
354
|
+
|
|
355
|
+
See LICENSE for details (Apache-2.0 OR MIT).
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# OKFgraph 0.2.
|
|
1
|
+
# OKFgraph 0.2.7
|
|
2
2
|
|
|
3
3
|
[](https://pypi.org/project/okfgraph/)
|
|
4
4
|
[](https://www.python.org/)
|
|
@@ -49,7 +49,7 @@ a swappable `DocumentConverter` seam. What isn't needed isn't installed.
|
|
|
49
49
|
Requires Python ≥ 3.11.
|
|
50
50
|
|
|
51
51
|
```bash
|
|
52
|
-
pip install "okfgraph[pdf,omni]" # PyPI
|
|
52
|
+
pip install "okfgraph[pdf,omni]" # PyPI (okf-embed ships platform wheels; published)
|
|
53
53
|
```
|
|
54
54
|
|
|
55
55
|
Or from source with `uv` (also builds the `okf-embed` Rust wheel from
|
|
@@ -273,6 +273,7 @@ Key design decisions:
|
|
|
273
273
|
```bash
|
|
274
274
|
uv run --project . pytest tests/ -q # full suite (model loads; takes a while)
|
|
275
275
|
uv run --project . pytest tests/test_ranking.py tests/test_mcp_server.py -q # fast subset
|
|
276
|
+
cd rust/okf-embed && cargo test --locked # Rust unit tests (pure, no model)
|
|
276
277
|
```
|
|
277
278
|
|
|
278
279
|
Golden fixtures under `tests/fixtures/` (`ppr_graph`, `diff_a`/`diff_b`,
|
|
@@ -297,7 +298,7 @@ okfgraph/
|
|
|
297
298
|
│ ├── images.py # IngestMode (text|optional|omni), planning helpers
|
|
298
299
|
│ ├── security.py # SSRF/domain guards for remote images
|
|
299
300
|
│ ├── tools.py # legacy tool definitions (superseded by mcp_server)
|
|
300
|
-
│ └── components/ # ranking, links, search, import_, export, diff, doctor,
|
|
301
|
+
│ └── components/ # ranking, links, search, lint, import_, export, diff, doctor,
|
|
301
302
|
│ # embedding, converters, image_assets, delta, purge, schema, ingest
|
|
302
303
|
├── rust/okf-embed/ # Jina v5 Rust loader (ort, CUDA-opportunistic) + Python wheel
|
|
303
304
|
├── skills/ # okfgraph-mcp, okfgraph-cli, okfgraph-ingest (harness-neutral source)
|
|
@@ -695,7 +695,10 @@ def _ingest(args):
|
|
|
695
695
|
"""
|
|
696
696
|
logger = logging.getLogger("cli")
|
|
697
697
|
router = _router(args)
|
|
698
|
-
kind = getattr(args, "kind",
|
|
698
|
+
kind = getattr(args, "kind", None)
|
|
699
|
+
if kind not in ("md", "pdf", "thoughts"):
|
|
700
|
+
print("[ERROR] --kind is required: md|pdf|thoughts")
|
|
701
|
+
return
|
|
699
702
|
tags = args.tags.split(",") if getattr(args, "tags", None) else None
|
|
700
703
|
if kind == "md":
|
|
701
704
|
md_file = getattr(args, "md_file", None)
|
|
@@ -1171,8 +1174,8 @@ def build_parser():
|
|
|
1171
1174
|
p = sub.add_parser("ingest", help="Add content: markdown file, PDF, or thoughts")
|
|
1172
1175
|
_add_global(p)
|
|
1173
1176
|
_add_logging_flags(p)
|
|
1174
|
-
p.add_argument("--kind",
|
|
1175
|
-
help="What to ingest
|
|
1177
|
+
p.add_argument("--kind", required=True, choices=["md", "pdf", "thoughts"],
|
|
1178
|
+
help="What to ingest")
|
|
1176
1179
|
p.add_argument("--md-file", default=None, help="Markdown file (--kind md)")
|
|
1177
1180
|
p.add_argument("--pdf-file", default=None, help="PDF file (--kind pdf)")
|
|
1178
1181
|
p.add_argument("--thoughts", default=None, help="Raw reasoning text (--kind thoughts)")
|
|
@@ -145,6 +145,36 @@ class ImportManager:
|
|
|
145
145
|
self.image_mgr = image_mgr
|
|
146
146
|
self.purge_mgr = purge_mgr
|
|
147
147
|
|
|
148
|
+
def _merge_directory_contains(self, parent_id: str, child_id: str):
|
|
149
|
+
"""Create a Directory→Directory CONTAINS edge without crashing Ladybug.
|
|
150
|
+
|
|
151
|
+
Ladybug 0.20.3 segfaults when one Cypher statement combines an
|
|
152
|
+
existing-node ``MERGE``, a new-node ``MERGE``, and a relationship
|
|
153
|
+
``MERGE`` on a long-lived connection. Separate idempotent statements
|
|
154
|
+
preserve the same graph while avoiding that planner path.
|
|
155
|
+
"""
|
|
156
|
+
self.conn.execute("MERGE (p:Directory {id: $id})", {"id": parent_id})
|
|
157
|
+
self.conn.execute("MERGE (d:Directory {id: $id})", {"id": child_id})
|
|
158
|
+
self.conn.execute("""
|
|
159
|
+
MATCH (p:Directory {id: $parent}), (d:Directory {id: $child})
|
|
160
|
+
MERGE (p)-[:CONTAINS]->(d)
|
|
161
|
+
""", {"parent": parent_id, "child": child_id})
|
|
162
|
+
|
|
163
|
+
def _merge_concept_contains(self, parent_id: str, child_id: str):
|
|
164
|
+
"""Create a Directory→Concept CONTAINS edge without crashing Ladybug.
|
|
165
|
+
|
|
166
|
+
See ``_merge_directory_contains``: a single three-clause ``MERGE`` can
|
|
167
|
+
segfault ladybug 0.20.3 after matching an existing Directory and
|
|
168
|
+
creating the next sibling Concept. The separate statements below are
|
|
169
|
+
semantically equivalent.
|
|
170
|
+
"""
|
|
171
|
+
self.conn.execute("MERGE (d:Directory {id: $id})", {"id": parent_id})
|
|
172
|
+
self.conn.execute("MERGE (c:Concept {id: $id})", {"id": child_id})
|
|
173
|
+
self.conn.execute("""
|
|
174
|
+
MATCH (d:Directory {id: $parent}), (c:Concept {id: $child})
|
|
175
|
+
MERGE (d)-[:CONTAINS]->(c)
|
|
176
|
+
""", {"parent": parent_id, "child": child_id})
|
|
177
|
+
|
|
148
178
|
def _batch_build_directories(self, cids: List[str]):
|
|
149
179
|
"""Build directory hierarchy for a batch of concept IDs.
|
|
150
180
|
|
|
@@ -166,11 +196,7 @@ class ImportManager:
|
|
|
166
196
|
for d in sorted_dirs:
|
|
167
197
|
parent = "/".join(d.split("/")[:-1]) if "/" in d else None
|
|
168
198
|
if parent and parent in dir_paths:
|
|
169
|
-
self.
|
|
170
|
-
MERGE (p:Directory {id: $parent})
|
|
171
|
-
MERGE (d:Directory {id: $child})
|
|
172
|
-
MERGE (p)-[:CONTAINS]->(d)
|
|
173
|
-
""", {"parent": parent, "child": d})
|
|
199
|
+
self._merge_directory_contains(parent, d)
|
|
174
200
|
elif parent:
|
|
175
201
|
# Parent is root (not a directory node)
|
|
176
202
|
self.conn.execute("""
|
|
@@ -186,11 +212,7 @@ class ImportManager:
|
|
|
186
212
|
parts = cid.split("/")
|
|
187
213
|
if len(parts) > 1:
|
|
188
214
|
parent_dir = "/".join(parts[:-1])
|
|
189
|
-
self.
|
|
190
|
-
MERGE (d:Directory {id: $parent})
|
|
191
|
-
MERGE (c:Concept {id: $child})
|
|
192
|
-
MERGE (d)-[:CONTAINS]->(c)
|
|
193
|
-
""", {"parent": parent_dir, "child": cid})
|
|
215
|
+
self._merge_concept_contains(parent_dir, cid)
|
|
194
216
|
|
|
195
217
|
|
|
196
218
|
def _load_link_index(self):
|
|
@@ -867,17 +889,9 @@ class ImportManager:
|
|
|
867
889
|
parent = "/".join(path_parts[:i])
|
|
868
890
|
child = "/".join(path_parts[: i + 1])
|
|
869
891
|
if i == len(path_parts) - 1:
|
|
870
|
-
self.
|
|
871
|
-
MERGE (d:Directory {id: $parent})
|
|
872
|
-
MERGE (c:Concept {id: $child})
|
|
873
|
-
MERGE (d)-[:CONTAINS]->(c)
|
|
874
|
-
""", {"parent": parent, "child": child})
|
|
892
|
+
self._merge_concept_contains(parent, child)
|
|
875
893
|
else:
|
|
876
|
-
self.
|
|
877
|
-
MERGE (p:Directory {id: $parent})
|
|
878
|
-
MERGE (d:Directory {id: $child})
|
|
879
|
-
MERGE (p)-[:CONTAINS]->(d)
|
|
880
|
-
""", {"parent": parent, "child": child})
|
|
894
|
+
self._merge_directory_contains(parent, child)
|
|
881
895
|
|
|
882
896
|
# NOTE: link extraction intentionally lives outside _insert_concept
|
|
883
897
|
# (callers run _extract_links_for_concept after the node exists), so
|