openkb 0.1.4.dev0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. openkb-0.3.0/.claude-plugin/marketplace.json +31 -0
  2. {openkb-0.1.4.dev0 → openkb-0.3.0}/.env.example +1 -0
  3. {openkb-0.1.4.dev0 → openkb-0.3.0}/.github/workflows/publish.yml +14 -17
  4. {openkb-0.1.4.dev0 → openkb-0.3.0}/.gitignore +1 -0
  5. {openkb-0.1.4.dev0 → openkb-0.3.0}/PKG-INFO +126 -21
  6. {openkb-0.1.4.dev0 → openkb-0.3.0}/README.md +123 -19
  7. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/agent/chat.py +89 -5
  8. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/agent/compiler.py +357 -29
  9. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/agent/linter.py +1 -1
  10. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/agent/query.py +44 -2
  11. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/agent/tools.py +50 -6
  12. openkb-0.3.0/openkb/cli.py +1958 -0
  13. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/converter.py +1 -1
  14. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/lint.py +83 -7
  15. openkb-0.3.0/openkb/prompts/__init__.py +21 -0
  16. openkb-0.3.0/openkb/prompts/skill_create.md +214 -0
  17. openkb-0.3.0/openkb/skill/__init__.py +101 -0
  18. openkb-0.3.0/openkb/skill/creator.py +209 -0
  19. openkb-0.3.0/openkb/skill/evaluator.py +490 -0
  20. openkb-0.3.0/openkb/skill/generator.py +81 -0
  21. openkb-0.3.0/openkb/skill/marketplace.py +120 -0
  22. openkb-0.3.0/openkb/skill/tools.py +100 -0
  23. openkb-0.3.0/openkb/skill/validator.py +265 -0
  24. openkb-0.3.0/openkb/skill/workspace.py +197 -0
  25. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/state.py +23 -0
  26. openkb-0.3.0/openkb/url_ingest.py +280 -0
  27. {openkb-0.1.4.dev0 → openkb-0.3.0}/pyproject.toml +7 -3
  28. openkb-0.3.0/skills/openkb/SKILL.md +175 -0
  29. openkb-0.3.0/skills/openkb/references/commands.md +69 -0
  30. openkb-0.3.0/skills/openkb/references/wiki-schema.md +119 -0
  31. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_add_command.py +4 -6
  32. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_agent_tools.py +0 -2
  33. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_chat_slash_commands.py +1 -1
  34. openkb-0.3.0/tests/test_cli.py +367 -0
  35. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_compiler.py +10 -10
  36. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_config.py +0 -2
  37. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_converter.py +1 -4
  38. openkb-0.3.0/tests/test_feedback.py +241 -0
  39. openkb-0.3.0/tests/test_generator.py +60 -0
  40. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_images.py +0 -3
  41. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_indexer.py +0 -2
  42. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_lint.py +111 -3
  43. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_linter.py +0 -1
  44. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_list_status.py +19 -1
  45. openkb-0.3.0/tests/test_marketplace.py +187 -0
  46. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_query.py +0 -1
  47. openkb-0.3.0/tests/test_remove.py +1118 -0
  48. openkb-0.3.0/tests/test_skill_chat_slash.py +105 -0
  49. openkb-0.3.0/tests/test_skill_cli.py +396 -0
  50. openkb-0.3.0/tests/test_skill_creator.py +109 -0
  51. openkb-0.3.0/tests/test_skill_evaluator.py +411 -0
  52. openkb-0.3.0/tests/test_skill_name_validation.py +37 -0
  53. openkb-0.3.0/tests/test_skill_tools.py +67 -0
  54. openkb-0.3.0/tests/test_skill_validator.py +410 -0
  55. openkb-0.3.0/tests/test_skill_workspace.py +171 -0
  56. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_state.py +0 -2
  57. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_tree_renderer.py +0 -1
  58. openkb-0.3.0/tests/test_url_ingest.py +621 -0
  59. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_watcher.py +1 -2
  60. openkb-0.3.0/tests/test_write_kb_file.py +79 -0
  61. openkb-0.3.0/uv.lock +3652 -0
  62. openkb-0.1.4.dev0/openkb/cli.py +0 -758
  63. openkb-0.1.4.dev0/tests/test_cli.py +0 -160
  64. {openkb-0.1.4.dev0 → openkb-0.3.0}/LICENSE +0 -0
  65. {openkb-0.1.4.dev0 → openkb-0.3.0}/config.yaml.example +0 -0
  66. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/__init__.py +0 -0
  67. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/__main__.py +0 -0
  68. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/agent/__init__.py +0 -0
  69. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/agent/_markdown.py +0 -0
  70. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/agent/chat_session.py +0 -0
  71. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/config.py +0 -0
  72. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/images.py +0 -0
  73. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/indexer.py +0 -0
  74. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/log.py +0 -0
  75. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/schema.py +0 -0
  76. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/tree_renderer.py +0 -0
  77. {openkb-0.1.4.dev0 → openkb-0.3.0}/openkb/watcher.py +0 -0
  78. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/conftest.py +0 -0
  79. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_chat_session.py +0 -0
  80. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_lint_cli.py +0 -0
  81. {openkb-0.1.4.dev0 → openkb-0.3.0}/tests/test_markdown_renderer.py +0 -0
@@ -0,0 +1,31 @@
1
+ {
2
+ "name": "vectify",
3
+ "owner": {
4
+ "name": "Ray",
5
+ "email": "ray@vectify.ai"
6
+ },
7
+ "metadata": {
8
+ "description": "Skills for navigating an OpenKB-compiled knowledge base from agent CLIs (Claude Code, Codex, Gemini CLI).",
9
+ "version": "0.1.4"
10
+ },
11
+ "plugins": [
12
+ {
13
+ "name": "openkb",
14
+ "description": "Navigate an OpenKB-compiled wiki: discover documents and concepts via openkb CLI commands, read concept and summary pages directly, and follow wikilinks across the knowledge graph.",
15
+ "source": "./",
16
+ "strict": false,
17
+ "version": "0.1.4",
18
+ "author": {
19
+ "name": "Ray",
20
+ "email": "ray@vectify.ai"
21
+ },
22
+ "homepage": "https://github.com/VectifyAI/OpenKB",
23
+ "repository": "https://github.com/VectifyAI/OpenKB",
24
+ "license": "Apache-2.0",
25
+ "keywords": ["knowledge-base", "wiki", "openkb", "rag", "agent-skill"],
26
+ "skills": [
27
+ "./skills/openkb"
28
+ ]
29
+ }
30
+ ]
31
+ }
@@ -2,4 +2,5 @@
2
2
  # OpenAI: LLM_API_KEY=sk-...
3
3
  # Anthropic: LLM_API_KEY=sk-ant-...
4
4
  # Gemini: LLM_API_KEY=AIza...
5
+ # DeepSeek: LLM_API_KEY=sk-...
5
6
  LLM_API_KEY=your-key-here
@@ -1,16 +1,20 @@
1
1
  name: Publish to PyPI
2
2
 
3
3
  # Release flow:
4
- # 1. Bump `version` in pyproject.toml on main.
5
- # 2. `git tag -a vX.Y.Z -m "Release X.Y.Z" && git push origin vX.Y.Z`
6
- # The tag MUST match pyproject.toml's version exactly (with the `v` prefix).
7
- # 3. This workflow builds the package, publishes to PyPI via OIDC trusted
8
- # publishing, and creates a GitHub Release with auto-generated notes.
4
+ # 1. `git tag -a vX.Y.Z -m "Release X.Y.Z" && git push origin vX.Y.Z`
5
+ # 2. This workflow builds the package (hatch-vcs derives the version from
6
+ # the tag automatically pyproject.toml has no static version field),
7
+ # publishes to PyPI via OIDC trusted publishing, and creates a GitHub
8
+ # Release with auto-generated notes.
9
+ #
10
+ # Tag must follow PEP 440: `v0.1.4`, `v0.2.0rc1`, `v0.1.4.dev0`. The
11
+ # leading `v` is stripped by hatch-vcs when computing the package version.
9
12
  #
10
13
  # Do not run `python -m build && twine upload` locally — that bypasses the
11
- # version check and leaves no GitHub Release. PyPI rejects duplicate
12
- # version uploads, so if the workflow fails after PyPI publish succeeded,
13
- # manually create the missing GitHub Release with `gh release create vX.Y.Z`.
14
+ # GitHub Release creation and produces a release without an attached
15
+ # changelog. PyPI rejects duplicate version uploads, so if the workflow
16
+ # fails after PyPI publish succeeded, manually create the missing GitHub
17
+ # Release with `gh release create vX.Y.Z`.
14
18
 
15
19
  on:
16
20
  push:
@@ -26,20 +30,13 @@ jobs:
26
30
  contents: write # Create GitHub Release
27
31
  steps:
28
32
  - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.2.2
33
+ with:
34
+ fetch-depth: 0 # hatch-vcs needs full history + tags
29
35
 
30
36
  - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
31
37
  with:
32
38
  python-version: "3.12"
33
39
 
34
- - name: Verify tag matches pyproject.toml version
35
- run: |
36
- tag="${GITHUB_REF_NAME#v}"
37
- pkg_version=$(python -c "import tomllib,pathlib; print(tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version'])")
38
- if [ "$tag" != "$pkg_version" ]; then
39
- echo "::error::Tag v$tag does not match pyproject.toml version $pkg_version"
40
- exit 1
41
- fi
42
-
43
40
  - name: Install build tools
44
41
  run: pip install build
45
42
 
@@ -13,6 +13,7 @@ venv/
13
13
  raw/
14
14
  wiki/
15
15
  .openkb/
16
+ output/
16
17
 
17
18
  # Local only
18
19
  docs/
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: openkb
3
- Version: 0.1.4.dev0
3
+ Version: 0.3.0
4
4
  Summary: OpenKB: Open LLM Knowledge Base, powered by PageIndex
5
5
  Project-URL: Repository, https://github.com/VectifyAI/OpenKB
6
6
  Project-URL: Homepage, https://github.com/VectifyAI/OpenKB
@@ -22,13 +22,14 @@ Requires-Python: >=3.10
22
22
  Requires-Dist: click>=8.0
23
23
  Requires-Dist: json-repair
24
24
  Requires-Dist: litellm
25
- Requires-Dist: markitdown[all]
25
+ Requires-Dist: markitdown[docx,pptx,xls,xlsx]>=0.1.5
26
26
  Requires-Dist: openai-agents
27
27
  Requires-Dist: pageindex==0.3.0.dev1
28
28
  Requires-Dist: prompt-toolkit>=3.0
29
29
  Requires-Dist: python-dotenv
30
30
  Requires-Dist: pyyaml
31
31
  Requires-Dist: rich>=13.0
32
+ Requires-Dist: trafilatura>=2.0
32
33
  Requires-Dist: watchdog>=3.0
33
34
  Provides-Extra: dev
34
35
  Requires-Dist: pytest; extra == 'dev'
@@ -59,17 +60,7 @@ The idea is based on a [concept](https://x.com/karpathy/status/20398056595256445
59
60
 
60
61
  Traditional RAG rediscovers knowledge from scratch on every query. Nothing accumulates. OpenKB compiles knowledge once into a persistent wiki, then keeps it current. Cross-references already exist. Contradictions are flagged. Synthesis reflects everything consumed.
61
62
 
62
- ### Features
63
-
64
- - **Broad format support** — PDF, Word, Markdown, PowerPoint, HTML, Excel, text, and more via markitdown
65
- - **Scale to long documents** — Long and complex documents are handled via [PageIndex](https://github.com/VectifyAI/PageIndex) tree indexing, enabling accurate, vectorless long-context retrieval
66
- - **Native multi-modality** — Retrieves and understands figures, tables, and images, not just text
67
- - **Compiled Wiki** — LLM manages and compiles your documents into summaries, concept pages, and cross-links, all kept in sync
68
- - **Query** — Ask questions (one-off) against your wiki. The LLM navigates your compiled knowledge to answer
69
- - **Interactive Chat** — Multi-turn conversations with persisted sessions you can resume across runs
70
- - **Lint** — Health checks find contradictions, gaps, orphans, and stale content
71
- - **Watch mode** — Drop files into `raw/`, wiki updates automatically
72
- - **Obsidian compatible** — Wiki is plain `.md` files with `[[wikilinks]]`. Open in Obsidian for graph view and browsing
63
+ OpenKB has two layers: a **wiki foundation** that compiles and maintains your knowledge, and **generators** (query / chat / Skill Factory) that turn it into useful output. See [Usage](#️-usage) for the full command list.
73
64
 
74
65
  # 🚀 Getting Started
75
66
 
@@ -109,13 +100,17 @@ openkb init
109
100
 
110
101
  # 3. Add documents
111
102
  openkb add paper.pdf
112
- openkb add ~/papers/ # Add a whole directory
103
+ openkb add ~/papers/ # Add a whole directory
104
+ openkb add https://arxiv.org/pdf/2509.11420 # Or fetch from a URL
113
105
 
114
106
  # 4. Ask a question
115
107
  openkb query "What are the main findings?"
116
108
 
117
109
  # 5. Or chat interactively
118
110
  openkb chat
111
+
112
+ # 6. Or distill your wiki into a redistributable skill
113
+ openkb skill new my-expert "Reason like an expert on <topic-from-your-docs>"
119
114
  ```
120
115
 
121
116
  ### Set up your LLM
@@ -145,7 +140,7 @@ raw/ You drop files here
145
140
  │ Wiki Compilation (using LLM)
146
141
  │ │
147
142
  ▼ ▼
148
- wiki/
143
+ wiki/ │ ← the foundation
149
144
  ├── index.md Knowledge base overview
150
145
  ├── log.md Operations timeline
151
146
  ├── AGENTS.md Wiki schema (LLM instructions)
@@ -154,6 +149,13 @@ wiki/
154
149
  ├── concepts/ Cross-document synthesis ← the good stuff
155
150
  ├── explorations/ Saved query results
156
151
  └── reports/ Lint reports
152
+
153
+ ┌──────────────────────┼──────────────────────┐
154
+ ▼ ▼ ▼
155
+ query / chat Skill Factory (future)
156
+ (LLM answers from openkb skill new ppt / podcast /
157
+ the wiki) → output/skills/ report / …
158
+ + marketplace.json
157
159
  ```
158
160
 
159
161
  ### Short vs. Long Document Handling
@@ -180,26 +182,43 @@ A single source might touch 10-15 wiki pages. Knowledge accumulates: each docume
180
182
 
181
183
  # ⚙️ Usage
182
184
 
183
- ### Commands
185
+ OpenKB commands fall into two layers: the **wiki foundation** (compile + manage your knowledge) and **generators** (turn that wiki into useful output).
186
+
187
+ ## 🧱 Wiki Foundation — compile and maintain
184
188
 
185
189
  | Command | Description |
186
190
  |---|---|
187
191
  | `openkb init` | Initialize a new knowledge base (interactive) |
188
- | <code>openkb&nbsp;add&nbsp;&lt;file_or_dir&gt;</code> | Add documents and compile to wiki |
189
- | <code>openkb&nbsp;query&nbsp;"question"</code> | Ask a question over the knowledge base (use `--save` to save the answer to `wiki/explorations/`) |
190
- | `openkb chat` | Start an interactive multi-turn chat (use `--resume`, `--list`, `--delete` to manage sessions) |
192
+ | <code>openkb&nbsp;add&nbsp;&lt;file_or_dir_or_URL&gt;</code> | Add documents and compile to wiki. URL ingest auto-detects PDF (saved as `.pdf` → PageIndex / markitdown) vs HTML (trafilatura main-content extract → `.md`) |
193
+ | <code>openkb&nbsp;remove&nbsp;&lt;doc&gt;</code> | Remove a document and clean up its wiki pages, images, registry, and PageIndex state (use `--dry-run` to preview, `--keep-raw` / `--keep-empty-concepts` to retain artifacts) |
191
194
  | `openkb watch` | Watch `raw/` and auto-compile new files |
192
195
  | `openkb lint` | Run structural + knowledge health checks |
193
196
  | `openkb list` | List indexed documents and concepts |
194
197
  | `openkb status` | Show knowledge base stats |
198
+ | <code>openkb&nbsp;feedback&nbsp;["msg"]</code> | File feedback by opening a prefilled GitHub issue (use `--type bug/feature/question` to tag the issue) |
195
199
 
196
200
  <!-- | `openkb lint --fix` | Auto-fix what it can | -->
197
201
 
198
- ### Interactive Chat
202
+ ## Generators — turn the wiki into output
203
+
204
+ A "generator" reads from the compiled wiki and produces something usable: an answer, a conversation, a skill folder. The wiki is the substrate; generators are the surfaces.
205
+
206
+ | Command | Output |
207
+ |---|---|
208
+ | <code>openkb&nbsp;query&nbsp;"question"</code> | A grounded answer with citations (use `--save` to persist to `wiki/explorations/`) |
209
+ | `openkb chat` | Interactive multi-turn session over the wiki (use `--resume`, `--list`, `--delete` to manage sessions) |
210
+ | <code>openkb&nbsp;skill&nbsp;new&nbsp;&lt;name&gt;&nbsp;"&lt;intent&gt;"</code> | A redistributable Anthropic Skill at `<kb>/output/skills/<name>/` + auto-updated `marketplace.json` |
211
+ | <code>openkb&nbsp;skill&nbsp;validate&nbsp;[name]</code> | Structural lint of compiled skills (frontmatter, file sizes, wikilinks, scripts/ stdlib check with `--strict`). Auto-runs at end of `skill new` |
212
+ | <code>openkb&nbsp;skill&nbsp;eval&nbsp;&lt;name&gt;</code> | Trigger-accuracy evaluation — does the `description:` field actually fire? LLM generates eval prompts; grader LLM scores activation. `--save` persists the eval set |
213
+ | <code>openkb&nbsp;skill&nbsp;history&nbsp;&lt;name&gt;</code> / <code>openkb&nbsp;skill&nbsp;rollback&nbsp;&lt;name&gt;</code> | Iteration workspace — every overwrite saves the previous version to `output/skills/<name>-workspace/iteration-N/` with a structural diff. Rollback restores any iteration |
214
+
215
+ ### Query & Chat — ask the wiki
199
216
 
200
- `openkb chat` opens an interactive chat session over your wiki knowledge base. Unlike the one-shot `openkb query`, each turn carries the conversation history, so you can dig into a topic without re-typing context.
217
+ `openkb query "..."` answers a single question. `openkb chat` is interactive each turn carries history, so you can dig into a topic without re-typing context. Both use the same underlying wiki and the same retrieval primitives (PageIndex for long docs, direct concept reads for short).
201
218
 
202
219
  ```bash
220
+ openkb query "What does the literature say about attention scaling?"
221
+
203
222
  openkb chat # start a new session
204
223
  openkb chat --resume # resume the most recent session
205
224
  openkb chat --resume 20260411 # resume by id (unique prefix works)
@@ -213,11 +232,70 @@ Inside a chat, type `/` to access slash commands (Tab to complete):
213
232
  - `/status` — show knowledge base status
214
233
  - `/list` — list all documents
215
234
  - `/add <path>` — add a document or directory without leaving the chat
235
+ - `/skill new <name> "<intent>"` — compile a skill from this chat (see below)
216
236
  - `/save [name]` — export the transcript to `wiki/explorations/`
217
237
  - `/clear` — start a fresh session (the current one stays on disk)
218
238
  - `/lint` — run knowledge base lint
219
239
  - `/exit` — exit (Ctrl-D also works)
220
240
 
241
+ ### 🛠 Skill Factory — *Drop in a book. Out comes a digital expert.*
242
+
243
+ The newest generator. `openkb skill new` distills any subset of your wiki into an [Anthropic Skill](https://docs.claude.com/en/docs/build-with-claude/skills) — a portable folder that **Claude Code, Codex CLI, Gemini CLI, and Cursor** all install and load natively. Drop in a book's worth of papers; out comes a specialist that other agents can call on.
244
+
245
+ ```bash
246
+ openkb skill new karpathy-thinking \
247
+ "Reason about transformers and attention in Karpathy's style"
248
+ ```
249
+
250
+ This produces:
251
+
252
+ ```
253
+ <kb>/output/skills/karpathy-thinking/
254
+ ├── SKILL.md # YAML frontmatter + when-to-use + approach
255
+ ├── references/ # depth material the agent loads on demand
256
+ │ ├── methodology.md
257
+ │ └── key-quotes.md
258
+ └── (scripts/) # optional, only if intent implies computation
259
+ ```
260
+
261
+ …plus an auto-updated `<kb>/.claude-plugin/marketplace.json` so the whole KB is one-line installable.
262
+
263
+ **Install locally:**
264
+
265
+ ```bash
266
+ cp -r output/skills/karpathy-thinking ~/.claude/skills/
267
+ ```
268
+
269
+ **Share with others** — push your KB to GitHub, then anyone runs:
270
+
271
+ ```bash
272
+ npx skills@latest add <your-org>/<your-repo>
273
+ ```
274
+
275
+ **Iterate from chat** — compilation is one-shot, but follow-up edits aren't. Inside `openkb chat`, you can refine without re-running the whole pipeline:
276
+
277
+ ```
278
+ /skill new karpathy-thinking "Reason about transformers like Karpathy"
279
+ [generation streams]
280
+ > description is too generic, make it about transformer implementations specifically
281
+ [agent edits SKILL.md frontmatter in place]
282
+ ```
283
+
284
+ **Quality gates** — structural validation, trigger-accuracy + body-coverage evaluation, and full history/rollback:
285
+
286
+ ```bash
287
+ # Lint structure (auto-runs at end of `skill new`)
288
+ openkb skill validate karpathy-thinking
289
+ openkb skill validate --strict # treat warnings as failures
290
+
291
+ # Does the description actually fire when it should?
292
+ openkb skill eval karpathy-thinking --save
293
+
294
+ # History + rollback if a new iteration regresses
295
+ openkb skill history karpathy-thinking
296
+ openkb skill rollback karpathy-thinking --to 2
297
+ ```
298
+
221
299
  ### Configuration
222
300
 
223
301
  Settings are initialized by `openkb init`, and stored in `.openkb/config.yaml`:
@@ -272,6 +350,33 @@ OpenKB's wiki is a directory of Markdown files with `[[wikilinks]]`. Obsidian re
272
350
  3. Use graph view to see knowledge connections
273
351
  4. Use Obsidian Web Clipper to add web articles to `raw/`
274
352
 
353
+ ### Using with Claude Code / Codex / Gemini CLI
354
+
355
+ OpenKB ships a `SKILL.md` so any agent CLI can read your compiled wiki — no extra runtime, no MCP setup, just install the skill once.
356
+
357
+ **Claude Code**:
358
+
359
+ ```
360
+ /plugin marketplace add VectifyAI/OpenKB
361
+ /plugin install openkb@vectify
362
+ ```
363
+
364
+ **Gemini CLI**:
365
+
366
+ ```bash
367
+ gemini skills install https://github.com/VectifyAI/OpenKB.git --path skills/openkb --consent
368
+ ```
369
+
370
+ **OpenAI Codex CLI** (no marketplace command yet — manual symlink):
371
+
372
+ ```bash
373
+ git clone https://github.com/VectifyAI/OpenKB.git ~/openkb-src
374
+ mkdir -p ~/.agents/skills
375
+ ln -s ~/openkb-src/skills/openkb ~/.agents/skills/openkb
376
+ ```
377
+
378
+ The skill is read-only — it won't run `openkb add`, `remove`, or `lint --fix` without you asking. See [`skills/openkb/SKILL.md`](skills/openkb/SKILL.md) for the full instruction set.
379
+
275
380
  # 🧭 Learn More
276
381
 
277
382
  ### Compared to Karpathy's Approach
@@ -22,17 +22,7 @@ The idea is based on a [concept](https://x.com/karpathy/status/20398056595256445
22
22
 
23
23
  Traditional RAG rediscovers knowledge from scratch on every query. Nothing accumulates. OpenKB compiles knowledge once into a persistent wiki, then keeps it current. Cross-references already exist. Contradictions are flagged. Synthesis reflects everything consumed.
24
24
 
25
- ### Features
26
-
27
- - **Broad format support** — PDF, Word, Markdown, PowerPoint, HTML, Excel, text, and more via markitdown
28
- - **Scale to long documents** — Long and complex documents are handled via [PageIndex](https://github.com/VectifyAI/PageIndex) tree indexing, enabling accurate, vectorless long-context retrieval
29
- - **Native multi-modality** — Retrieves and understands figures, tables, and images, not just text
30
- - **Compiled Wiki** — LLM manages and compiles your documents into summaries, concept pages, and cross-links, all kept in sync
31
- - **Query** — Ask questions (one-off) against your wiki. The LLM navigates your compiled knowledge to answer
32
- - **Interactive Chat** — Multi-turn conversations with persisted sessions you can resume across runs
33
- - **Lint** — Health checks find contradictions, gaps, orphans, and stale content
34
- - **Watch mode** — Drop files into `raw/`, wiki updates automatically
35
- - **Obsidian compatible** — Wiki is plain `.md` files with `[[wikilinks]]`. Open in Obsidian for graph view and browsing
25
+ OpenKB has two layers: a **wiki foundation** that compiles and maintains your knowledge, and **generators** (query / chat / Skill Factory) that turn it into useful output. See [Usage](#️-usage) for the full command list.
36
26
 
37
27
  # 🚀 Getting Started
38
28
 
@@ -72,13 +62,17 @@ openkb init
72
62
 
73
63
  # 3. Add documents
74
64
  openkb add paper.pdf
75
- openkb add ~/papers/ # Add a whole directory
65
+ openkb add ~/papers/ # Add a whole directory
66
+ openkb add https://arxiv.org/pdf/2509.11420 # Or fetch from a URL
76
67
 
77
68
  # 4. Ask a question
78
69
  openkb query "What are the main findings?"
79
70
 
80
71
  # 5. Or chat interactively
81
72
  openkb chat
73
+
74
+ # 6. Or distill your wiki into a redistributable skill
75
+ openkb skill new my-expert "Reason like an expert on <topic-from-your-docs>"
82
76
  ```
83
77
 
84
78
  ### Set up your LLM
@@ -108,7 +102,7 @@ raw/ You drop files here
108
102
  │ Wiki Compilation (using LLM)
109
103
  │ │
110
104
  ▼ ▼
111
- wiki/
105
+ wiki/ │ ← the foundation
112
106
  ├── index.md Knowledge base overview
113
107
  ├── log.md Operations timeline
114
108
  ├── AGENTS.md Wiki schema (LLM instructions)
@@ -117,6 +111,13 @@ wiki/
117
111
  ├── concepts/ Cross-document synthesis ← the good stuff
118
112
  ├── explorations/ Saved query results
119
113
  └── reports/ Lint reports
114
+
115
+ ┌──────────────────────┼──────────────────────┐
116
+ ▼ ▼ ▼
117
+ query / chat Skill Factory (future)
118
+ (LLM answers from openkb skill new ppt / podcast /
119
+ the wiki) → output/skills/ report / …
120
+ + marketplace.json
120
121
  ```
121
122
 
122
123
  ### Short vs. Long Document Handling
@@ -143,26 +144,43 @@ A single source might touch 10-15 wiki pages. Knowledge accumulates: each docume
143
144
 
144
145
  # ⚙️ Usage
145
146
 
146
- ### Commands
147
+ OpenKB commands fall into two layers: the **wiki foundation** (compile + manage your knowledge) and **generators** (turn that wiki into useful output).
148
+
149
+ ## 🧱 Wiki Foundation — compile and maintain
147
150
 
148
151
  | Command | Description |
149
152
  |---|---|
150
153
  | `openkb init` | Initialize a new knowledge base (interactive) |
151
- | <code>openkb&nbsp;add&nbsp;&lt;file_or_dir&gt;</code> | Add documents and compile to wiki |
152
- | <code>openkb&nbsp;query&nbsp;"question"</code> | Ask a question over the knowledge base (use `--save` to save the answer to `wiki/explorations/`) |
153
- | `openkb chat` | Start an interactive multi-turn chat (use `--resume`, `--list`, `--delete` to manage sessions) |
154
+ | <code>openkb&nbsp;add&nbsp;&lt;file_or_dir_or_URL&gt;</code> | Add documents and compile to wiki. URL ingest auto-detects PDF (saved as `.pdf` → PageIndex / markitdown) vs HTML (trafilatura main-content extract → `.md`) |
155
+ | <code>openkb&nbsp;remove&nbsp;&lt;doc&gt;</code> | Remove a document and clean up its wiki pages, images, registry, and PageIndex state (use `--dry-run` to preview, `--keep-raw` / `--keep-empty-concepts` to retain artifacts) |
154
156
  | `openkb watch` | Watch `raw/` and auto-compile new files |
155
157
  | `openkb lint` | Run structural + knowledge health checks |
156
158
  | `openkb list` | List indexed documents and concepts |
157
159
  | `openkb status` | Show knowledge base stats |
160
+ | <code>openkb&nbsp;feedback&nbsp;["msg"]</code> | File feedback by opening a prefilled GitHub issue (use `--type bug/feature/question` to tag the issue) |
158
161
 
159
162
  <!-- | `openkb lint --fix` | Auto-fix what it can | -->
160
163
 
161
- ### Interactive Chat
164
+ ## Generators — turn the wiki into output
162
165
 
163
- `openkb chat` opens an interactive chat session over your wiki knowledge base. Unlike the one-shot `openkb query`, each turn carries the conversation history, so you can dig into a topic without re-typing context.
166
+ A "generator" reads from the compiled wiki and produces something usable: an answer, a conversation, a skill folder. The wiki is the substrate; generators are the surfaces.
167
+
168
+ | Command | Output |
169
+ |---|---|
170
+ | <code>openkb&nbsp;query&nbsp;"question"</code> | A grounded answer with citations (use `--save` to persist to `wiki/explorations/`) |
171
+ | `openkb chat` | Interactive multi-turn session over the wiki (use `--resume`, `--list`, `--delete` to manage sessions) |
172
+ | <code>openkb&nbsp;skill&nbsp;new&nbsp;&lt;name&gt;&nbsp;"&lt;intent&gt;"</code> | A redistributable Anthropic Skill at `<kb>/output/skills/<name>/` + auto-updated `marketplace.json` |
173
+ | <code>openkb&nbsp;skill&nbsp;validate&nbsp;[name]</code> | Structural lint of compiled skills (frontmatter, file sizes, wikilinks, scripts/ stdlib check with `--strict`). Auto-runs at end of `skill new` |
174
+ | <code>openkb&nbsp;skill&nbsp;eval&nbsp;&lt;name&gt;</code> | Trigger-accuracy evaluation — does the `description:` field actually fire? LLM generates eval prompts; grader LLM scores activation. `--save` persists the eval set |
175
+ | <code>openkb&nbsp;skill&nbsp;history&nbsp;&lt;name&gt;</code> / <code>openkb&nbsp;skill&nbsp;rollback&nbsp;&lt;name&gt;</code> | Iteration workspace — every overwrite saves the previous version to `output/skills/<name>-workspace/iteration-N/` with a structural diff. Rollback restores any iteration |
176
+
177
+ ### Query & Chat — ask the wiki
178
+
179
+ `openkb query "..."` answers a single question. `openkb chat` is interactive — each turn carries history, so you can dig into a topic without re-typing context. Both use the same underlying wiki and the same retrieval primitives (PageIndex for long docs, direct concept reads for short).
164
180
 
165
181
  ```bash
182
+ openkb query "What does the literature say about attention scaling?"
183
+
166
184
  openkb chat # start a new session
167
185
  openkb chat --resume # resume the most recent session
168
186
  openkb chat --resume 20260411 # resume by id (unique prefix works)
@@ -176,11 +194,70 @@ Inside a chat, type `/` to access slash commands (Tab to complete):
176
194
  - `/status` — show knowledge base status
177
195
  - `/list` — list all documents
178
196
  - `/add <path>` — add a document or directory without leaving the chat
197
+ - `/skill new <name> "<intent>"` — compile a skill from this chat (see below)
179
198
  - `/save [name]` — export the transcript to `wiki/explorations/`
180
199
  - `/clear` — start a fresh session (the current one stays on disk)
181
200
  - `/lint` — run knowledge base lint
182
201
  - `/exit` — exit (Ctrl-D also works)
183
202
 
203
+ ### 🛠 Skill Factory — *Drop in a book. Out comes a digital expert.*
204
+
205
+ The newest generator. `openkb skill new` distills any subset of your wiki into an [Anthropic Skill](https://docs.claude.com/en/docs/build-with-claude/skills) — a portable folder that **Claude Code, Codex CLI, Gemini CLI, and Cursor** all install and load natively. Drop in a book's worth of papers; out comes a specialist that other agents can call on.
206
+
207
+ ```bash
208
+ openkb skill new karpathy-thinking \
209
+ "Reason about transformers and attention in Karpathy's style"
210
+ ```
211
+
212
+ This produces:
213
+
214
+ ```
215
+ <kb>/output/skills/karpathy-thinking/
216
+ ├── SKILL.md # YAML frontmatter + when-to-use + approach
217
+ ├── references/ # depth material the agent loads on demand
218
+ │ ├── methodology.md
219
+ │ └── key-quotes.md
220
+ └── (scripts/) # optional, only if intent implies computation
221
+ ```
222
+
223
+ …plus an auto-updated `<kb>/.claude-plugin/marketplace.json` so the whole KB is one-line installable.
224
+
225
+ **Install locally:**
226
+
227
+ ```bash
228
+ cp -r output/skills/karpathy-thinking ~/.claude/skills/
229
+ ```
230
+
231
+ **Share with others** — push your KB to GitHub, then anyone runs:
232
+
233
+ ```bash
234
+ npx skills@latest add <your-org>/<your-repo>
235
+ ```
236
+
237
+ **Iterate from chat** — compilation is one-shot, but follow-up edits aren't. Inside `openkb chat`, you can refine without re-running the whole pipeline:
238
+
239
+ ```
240
+ /skill new karpathy-thinking "Reason about transformers like Karpathy"
241
+ [generation streams]
242
+ > description is too generic, make it about transformer implementations specifically
243
+ [agent edits SKILL.md frontmatter in place]
244
+ ```
245
+
246
+ **Quality gates** — structural validation, trigger-accuracy + body-coverage evaluation, and full history/rollback:
247
+
248
+ ```bash
249
+ # Lint structure (auto-runs at end of `skill new`)
250
+ openkb skill validate karpathy-thinking
251
+ openkb skill validate --strict # treat warnings as failures
252
+
253
+ # Does the description actually fire when it should?
254
+ openkb skill eval karpathy-thinking --save
255
+
256
+ # History + rollback if a new iteration regresses
257
+ openkb skill history karpathy-thinking
258
+ openkb skill rollback karpathy-thinking --to 2
259
+ ```
260
+
184
261
  ### Configuration
185
262
 
186
263
  Settings are initialized by `openkb init`, and stored in `.openkb/config.yaml`:
@@ -235,6 +312,33 @@ OpenKB's wiki is a directory of Markdown files with `[[wikilinks]]`. Obsidian re
235
312
  3. Use graph view to see knowledge connections
236
313
  4. Use Obsidian Web Clipper to add web articles to `raw/`
237
314
 
315
+ ### Using with Claude Code / Codex / Gemini CLI
316
+
317
+ OpenKB ships a `SKILL.md` so any agent CLI can read your compiled wiki — no extra runtime, no MCP setup, just install the skill once.
318
+
319
+ **Claude Code**:
320
+
321
+ ```
322
+ /plugin marketplace add VectifyAI/OpenKB
323
+ /plugin install openkb@vectify
324
+ ```
325
+
326
+ **Gemini CLI**:
327
+
328
+ ```bash
329
+ gemini skills install https://github.com/VectifyAI/OpenKB.git --path skills/openkb --consent
330
+ ```
331
+
332
+ **OpenAI Codex CLI** (no marketplace command yet — manual symlink):
333
+
334
+ ```bash
335
+ git clone https://github.com/VectifyAI/OpenKB.git ~/openkb-src
336
+ mkdir -p ~/.agents/skills
337
+ ln -s ~/openkb-src/skills/openkb ~/.agents/skills/openkb
338
+ ```
339
+
340
+ The skill is read-only — it won't run `openkb add`, `remove`, or `lint --fix` without you asking. See [`skills/openkb/SKILL.md`](skills/openkb/SKILL.md) for the full instruction set.
341
+
238
342
  # 🧭 Learn More
239
343
 
240
344
  ### Compared to Karpathy's Approach