openkb 0.1.4.dev0__tar.gz → 0.2.0__tar.gz

openkb-0.2.0/.claude-plugin/marketplace.json

@@ -0,0 +1,31 @@
+{
+  "name": "vectify",
+  "owner": {
+    "name": "Ray",
+    "email": "ray@vectify.ai"
+  },
+  "metadata": {
+    "description": "Skills for navigating an OpenKB-compiled knowledge base from agent CLIs (Claude Code, Codex, Gemini CLI).",
+    "version": "0.1.4"
+  },
+  "plugins": [
+    {
+      "name": "openkb",
+      "description": "Navigate an OpenKB-compiled wiki: discover documents and concepts via openkb CLI commands, read concept and summary pages directly, and follow wikilinks across the knowledge graph.",
+      "source": "./",
+      "strict": false,
+      "version": "0.1.4",
+      "author": {
+        "name": "Ray",
+        "email": "ray@vectify.ai"
+      },
+      "homepage": "https://github.com/VectifyAI/OpenKB",
+      "repository": "https://github.com/VectifyAI/OpenKB",
+      "license": "Apache-2.0",
+      "keywords": ["knowledge-base", "wiki", "openkb", "rag", "agent-skill"],
+      "skills": [
+        "./skills/openkb"
+      ]
+    }
+  ]
+}

{openkb-0.1.4.dev0 → openkb-0.2.0}/.github/workflows/publish.yml

@@ -1,16 +1,20 @@
 name: Publish to PyPI
 
 # Release flow:
-#   1. Bump `version` in pyproject.toml on main.
-#   2. `git tag -a vX.Y.Z -m "Release X.Y.Z" && git push origin vX.Y.Z`
-#      The tag MUST match pyproject.toml's version exactly (with the `v` prefix).
-#   3. This workflow builds the package, publishes to PyPI via OIDC trusted
-#      publishing, and creates a GitHub Release with auto-generated notes.
+#   1. `git tag -a vX.Y.Z -m "Release X.Y.Z" && git push origin vX.Y.Z`
+#   2. This workflow builds the package (hatch-vcs derives the version from
+#      the tag automatically — pyproject.toml has no static version field),
+#      publishes to PyPI via OIDC trusted publishing, and creates a GitHub
+#      Release with auto-generated notes.
+#
+# Tag must follow PEP 440: `v0.1.4`, `v0.2.0rc1`, `v0.1.4.dev0`. The
+# leading `v` is stripped by hatch-vcs when computing the package version.
 #
 # Do not run `python -m build && twine upload` locally — that bypasses the
-# version check and leaves no GitHub Release. PyPI rejects duplicate
-# version uploads, so if the workflow fails after PyPI publish succeeded,
-# manually create the missing GitHub Release with `gh release create vX.Y.Z`.
+# GitHub Release creation and produces a release without an attached
+# changelog. PyPI rejects duplicate version uploads, so if the workflow
+# fails after PyPI publish succeeded, manually create the missing GitHub
+# Release with `gh release create vX.Y.Z`.
 
 on:
   push:
@@ -26,20 +30,13 @@ jobs:
       contents: write   # Create GitHub Release
     steps:
       - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332  # v4.2.2
+        with:
+          fetch-depth: 0   # hatch-vcs needs full history + tags
 
       - uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3  # v5.2.0
         with:
           python-version: "3.12"
 
-      - name: Verify tag matches pyproject.toml version
-        run: |
-          tag="${GITHUB_REF_NAME#v}"
-          pkg_version=$(python -c "import tomllib,pathlib; print(tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version'])")
-          if [ "$tag" != "$pkg_version" ]; then
-            echo "::error::Tag v$tag does not match pyproject.toml version $pkg_version"
-            exit 1
-          fi
-
       - name: Install build tools
         run: pip install build
 

{openkb-0.1.4.dev0 → openkb-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: openkb
-Version: 0.1.4.dev0
+Version: 0.2.0
 Summary: OpenKB: Open LLM Knowledge Base, powered by PageIndex
 Project-URL: Repository, https://github.com/VectifyAI/OpenKB
 Project-URL: Homepage, https://github.com/VectifyAI/OpenKB
@@ -29,6 +29,7 @@ Requires-Dist: prompt-toolkit>=3.0
 Requires-Dist: python-dotenv
 Requires-Dist: pyyaml
 Requires-Dist: rich>=13.0
+Requires-Dist: trafilatura>=2.0
 Requires-Dist: watchdog>=3.0
 Provides-Extra: dev
 Requires-Dist: pytest; extra == 'dev'
@@ -109,7 +110,8 @@ openkb init
 
 # 3. Add documents
 openkb add paper.pdf
-openkb add ~/papers/  # Add a whole directory
+openkb add ~/papers/                            # Add a whole directory
+openkb add https://arxiv.org/pdf/2509.11420     # Or fetch from a URL
 
 # 4. Ask a question
 openkb query "What are the main findings?"
@@ -185,13 +187,15 @@ A single source might touch 10-15 wiki pages. Knowledge accumulates: each docume
 | Command | Description |
 |---|---|
 | `openkb init` | Initialize a new knowledge base (interactive) |
-| <code>openkb&nbsp;add&nbsp;&lt;file_or_dir&gt;</code> | Add documents and compile to wiki |
+| <code>openkb&nbsp;add&nbsp;&lt;file_or_dir_or_URL&gt;</code> | Add documents and compile to wiki. URL ingest auto-detects PDF (saved as `.pdf` → PageIndex / markitdown) vs HTML (trafilatura main-content extract → `.md`) |
+| <code>openkb&nbsp;remove&nbsp;&lt;doc&gt;</code> | Remove a document and clean up its wiki pages, images, registry, and PageIndex state (use `--dry-run` to preview, `--keep-raw` / `--keep-empty-concepts` to retain artifacts) |
 | <code>openkb&nbsp;query&nbsp;"question"</code> | Ask a question over the knowledge base (use `--save` to save the answer to `wiki/explorations/`) |
 | `openkb chat` | Start an interactive multi-turn chat (use `--resume`, `--list`, `--delete` to manage sessions) |
 | `openkb watch` | Watch `raw/` and auto-compile new files |
 | `openkb lint` | Run structural + knowledge health checks |
 | `openkb list` | List indexed documents and concepts |
 | `openkb status` | Show knowledge base stats |
+| <code>openkb&nbsp;feedback&nbsp;["msg"]</code> | File feedback by opening a prefilled GitHub issue (use `--type bug/feature/question` to tag the issue) |
 
 <!-- | `openkb lint --fix` | Auto-fix what it can | -->
 
@@ -272,6 +276,33 @@ OpenKB's wiki is a directory of Markdown files with `[[wikilinks]]`. Obsidian re
 3. Use graph view to see knowledge connections
 4. Use Obsidian Web Clipper to add web articles to `raw/`
 
+### Using with Claude Code / Codex / Gemini CLI
+
+OpenKB ships a `SKILL.md` so any agent CLI can read your compiled wiki — no extra runtime, no MCP setup, just install the skill once.
+
+**Claude Code**:
+
+```
+/plugin marketplace add VectifyAI/OpenKB
+/plugin install openkb@vectify
+```
+
+**Gemini CLI**:
+
+```bash
+gemini skills install https://github.com/VectifyAI/OpenKB.git --path skills/openkb --consent
+```
+
+**OpenAI Codex CLI** (no marketplace command yet — manual symlink):
+
+```bash
+git clone https://github.com/VectifyAI/OpenKB.git ~/openkb-src
+mkdir -p ~/.agents/skills
+ln -s ~/openkb-src/skills/openkb ~/.agents/skills/openkb
+```
+
+The skill is read-only — it won't run `openkb add`, `remove`, or `lint --fix` without you asking. See [`skills/openkb/SKILL.md`](skills/openkb/SKILL.md) for the full instruction set.
+
 # 🧭 Learn More
 
 ### Compared to Karpathy's Approach

{openkb-0.1.4.dev0 → openkb-0.2.0}/README.md

@@ -72,7 +72,8 @@ openkb init
 
 # 3. Add documents
 openkb add paper.pdf
-openkb add ~/papers/  # Add a whole directory
+openkb add ~/papers/                            # Add a whole directory
+openkb add https://arxiv.org/pdf/2509.11420     # Or fetch from a URL
 
 # 4. Ask a question
 openkb query "What are the main findings?"
@@ -148,13 +149,15 @@ A single source might touch 10-15 wiki pages. Knowledge accumulates: each docume
 | Command | Description |
 |---|---|
 | `openkb init` | Initialize a new knowledge base (interactive) |
-| <code>openkb&nbsp;add&nbsp;&lt;file_or_dir&gt;</code> | Add documents and compile to wiki |
+| <code>openkb&nbsp;add&nbsp;&lt;file_or_dir_or_URL&gt;</code> | Add documents and compile to wiki. URL ingest auto-detects PDF (saved as `.pdf` → PageIndex / markitdown) vs HTML (trafilatura main-content extract → `.md`) |
+| <code>openkb&nbsp;remove&nbsp;&lt;doc&gt;</code> | Remove a document and clean up its wiki pages, images, registry, and PageIndex state (use `--dry-run` to preview, `--keep-raw` / `--keep-empty-concepts` to retain artifacts) |
 | <code>openkb&nbsp;query&nbsp;"question"</code> | Ask a question over the knowledge base (use `--save` to save the answer to `wiki/explorations/`) |
 | `openkb chat` | Start an interactive multi-turn chat (use `--resume`, `--list`, `--delete` to manage sessions) |
 | `openkb watch` | Watch `raw/` and auto-compile new files |
 | `openkb lint` | Run structural + knowledge health checks |
 | `openkb list` | List indexed documents and concepts |
 | `openkb status` | Show knowledge base stats |
+| <code>openkb&nbsp;feedback&nbsp;["msg"]</code> | File feedback by opening a prefilled GitHub issue (use `--type bug/feature/question` to tag the issue) |
 
 <!-- | `openkb lint --fix` | Auto-fix what it can | -->
 
@@ -235,6 +238,33 @@ OpenKB's wiki is a directory of Markdown files with `[[wikilinks]]`. Obsidian re
 3. Use graph view to see knowledge connections
 4. Use Obsidian Web Clipper to add web articles to `raw/`
 
+### Using with Claude Code / Codex / Gemini CLI
+
+OpenKB ships a `SKILL.md` so any agent CLI can read your compiled wiki — no extra runtime, no MCP setup, just install the skill once.
+
+**Claude Code**:
+
+```
+/plugin marketplace add VectifyAI/OpenKB
+/plugin install openkb@vectify
+```
+
+**Gemini CLI**:
+
+```bash
+gemini skills install https://github.com/VectifyAI/OpenKB.git --path skills/openkb --consent
+```
+
+**OpenAI Codex CLI** (no marketplace command yet — manual symlink):
+
+```bash
+git clone https://github.com/VectifyAI/OpenKB.git ~/openkb-src
+mkdir -p ~/.agents/skills
+ln -s ~/openkb-src/skills/openkb ~/.agents/skills/openkb
+```
+
+The skill is read-only — it won't run `openkb add`, `remove`, or `lint --fix` without you asking. See [`skills/openkb/SKILL.md`](skills/openkb/SKILL.md) for the full instruction set.
+
 # 🧭 Learn More
 
 ### Compared to Karpathy's Approach

{openkb-0.1.4.dev0 → openkb-0.2.0}/openkb/agent/chat.py

@@ -338,7 +338,6 @@ async def _run_turn(
     need_blank_before_text = False
 
     if use_color and not raw:
-        from rich.console import Console
         from rich.live import Live
 
         console = _make_rich_console()

{openkb-0.1.4.dev0 → openkb-0.2.0}/openkb/agent/compiler.py

@@ -445,6 +445,28 @@ def _insert_section_entry(lines: list[str], heading: str, entry: str) -> bool:
     return True
 
 
+def _remove_section_entry(lines: list[str], heading: str, link: str) -> bool:
+    """Remove the first entry whose line starts with ``- {link}`` in the named
+    section. Returns True if an entry was removed.
+
+    Matching is intentionally strict (prefix-only, matching the canonical
+    bullet form written by ``_insert_section_entry`` and friends). An earlier
+    substring fallback could wrongly delete sibling bullets whose brief text
+    referenced the removed link.
+    """
+    bounds = _get_section_bounds(lines, heading)
+    if bounds is None:
+        return False
+
+    start, end = bounds
+    entry_prefix = f"- {link}"
+    for i in range(start, end):
+        if lines[i].startswith(entry_prefix):
+            del lines[i]
+            return True
+    return False
+
+
 
 def _write_summary(wiki_dir: Path, doc_name: str, summary: str,
                     doc_type: str = "short") -> None:
@@ -562,6 +584,44 @@ def _prepend_source_to_frontmatter(text: str, source_file: str) -> str:
     return "\n".join(fm_lines) + body
 
 
+def _remove_source_from_frontmatter(text: str, source_file: str) -> tuple[str, bool]:
+    """Remove ``source_file`` from the inline ``sources:`` list in YAML frontmatter.
+
+    Returns ``(rewritten_text, sources_now_empty)``. ``sources_now_empty`` is
+    True when ``source_file`` was the only remaining item in the list (callers
+    can use this to decide whether to delete the page entirely).
+
+    If the frontmatter is missing, malformed, has no ``sources:`` line, or
+    the source is not present in the list, returns ``(text, False)``.
+    """
+    if not text.startswith("---"):
+        return text, False
+
+    fm_end = text.find("---", 3)
+    if fm_end == -1:
+        return text, False
+
+    fm_block = text[:fm_end]
+    body = text[fm_end:]
+    fm_lines = fm_block.split("\n")
+
+    for i, line in enumerate(fm_lines):
+        if not line.lstrip().startswith("sources:"):
+            continue
+        lb = line.find("[")
+        rb = line.rfind("]")
+        if lb == -1 or rb == -1 or rb < lb:
+            return text, False
+        items = [s.strip() for s in line[lb + 1:rb].split(",") if s.strip()]
+        if source_file not in items:
+            return text, False
+        items.remove(source_file)
+        fm_lines[i] = f"sources: [{', '.join(items)}]"
+        return "\n".join(fm_lines) + body, len(items) == 0
+
+    return text, False
+
+
 def _add_related_link(wiki_dir: Path, concept_slug: str, doc_name: str, source_file: str) -> None:
     """Add a cross-reference link to an existing concept page (no LLM call)."""
     concepts_dir = wiki_dir / "concepts"
@@ -631,6 +691,123 @@ def _backlink_concepts(wiki_dir: Path, doc_name: str, concept_slugs: list[str])
         _insert_section_entry(lines, "## Related Documents", f"- {link}")
         path.write_text("\n".join(lines), encoding="utf-8")
 
+
+def remove_doc_from_concept_pages(
+    wiki_dir: Path,
+    doc_name: str,
+    *,
+    keep_empty: bool = False,
+) -> dict[str, list[str]]:
+    """Update or delete concept pages affected by removing a document.
+
+    For each ``concepts/*.md`` whose frontmatter ``sources:`` lists
+    ``summaries/{doc_name}``:
+
+    - Remove that source from the frontmatter list.
+    - Remove any ``- [[summaries/{doc_name}]]`` entries from the
+      ``## Related Documents`` section.
+    - Remove any standalone ``See also: [[summaries/{doc_name}]]`` lines
+      (left by ``_add_related_link``).
+    - If the ``sources:`` list becomes empty AND ``keep_empty`` is False,
+      delete the concept page entirely.
+
+    Args:
+        wiki_dir: Path to the wiki root directory.
+        doc_name: The summary slug being removed (e.g.
+            ``"attention-is-all-you-need"``).
+        keep_empty: When True, retains concept pages whose only source
+            was the removed doc — leaves their frontmatter with an empty
+            ``sources: []`` list. Useful when the doc is being replaced
+            by a newer version that will repopulate the source on the
+            next ``openkb add``.
+
+    Returns:
+        ``{"modified": [slugs...], "deleted": [slugs...]}`` — concept
+        slugs whose pages were edited vs. deleted.
+    """
+    concepts_dir = wiki_dir / "concepts"
+    if not concepts_dir.is_dir():
+        return {"modified": [], "deleted": []}
+
+    source_file = f"summaries/{doc_name}.md"
+    bare_source = f"summaries/{doc_name}"
+    link = f"[[{bare_source}]]"
+
+    modified: list[str] = []
+    deleted: list[str] = []
+
+    for path in sorted(concepts_dir.glob("*.md")):
+        text = path.read_text(encoding="utf-8")
+        # Cheap filter: skip pages that don't reference the doc at all.
+        if source_file not in text and bare_source not in text:
+            continue
+
+        new_text, sources_empty = _remove_source_from_frontmatter(text, source_file)
+
+        # Drop the doc's entry from the "## Related Documents" section.
+        if link in new_text:
+            lines = new_text.split("\n")
+            while _remove_section_entry(lines, "## Related Documents", link):
+                pass
+            new_text = "\n".join(lines)
+
+        # Drop standalone "See also: [[summaries/{doc_name}]]" lines.
+        # The dominant form (written by ``_add_related_link``) is a
+        # paragraph: preceded by a blank line and trailed by either a
+        # newline or end-of-string. The first regex matches that shape
+        # exactly, preserving one trailing newline so paragraph spacing
+        # in surrounding content survives.
+        new_text = re.sub(
+            rf"\n\n[ \t]*See also:[ \t]*\[\[{re.escape(bare_source)}\]\][ \t]*(\n|\Z)",
+            r"\1",
+            new_text,
+        )
+        # Fallback for hand-edited inline "See also:" lines that lack the
+        # paragraph-break separator above. Bounded to a single line via
+        # `[ \t]` and an optional trailing newline.
+        new_text = re.sub(
+            rf"^[ \t]*See also:[ \t]*\[\[{re.escape(bare_source)}\]\][ \t]*\n?",
+            "",
+            new_text,
+            flags=re.MULTILINE,
+        )
+
+        if sources_empty and not keep_empty:
+            path.unlink()
+            deleted.append(path.stem)
+        elif new_text != text:
+            path.write_text(new_text, encoding="utf-8")
+            modified.append(path.stem)
+
+    return {"modified": modified, "deleted": deleted}
+
+
+def remove_doc_from_index(wiki_dir: Path, doc_name: str, concept_slugs_deleted: list[str]) -> None:
+    """Remove the document's entry from ``index.md`` along with any concept
+    entries for concepts that were deleted as a side effect.
+
+    No-op when ``index.md`` doesn't exist. Section headings are kept even
+    when their last entry is removed — adding a new doc later repopulates
+    them.
+    """
+    index_path = wiki_dir / "index.md"
+    if not index_path.exists():
+        return
+
+    lines = index_path.read_text(encoding="utf-8").split("\n")
+
+    doc_link = f"[[summaries/{doc_name}]]"
+    while _remove_section_entry(lines, "## Documents", doc_link):
+        pass
+
+    for slug in concept_slugs_deleted:
+        concept_link = f"[[concepts/{slug}]]"
+        while _remove_section_entry(lines, "## Concepts", concept_link):
+            pass
+
+    index_path.write_text("\n".join(lines), encoding="utf-8")
+
+
 def _update_index(
     wiki_dir: Path, doc_name: str, concept_names: list[str],
     doc_brief: str = "", concept_briefs: dict[str, str] | None = None,

{openkb-0.1.4.dev0 → openkb-0.2.0}/openkb/agent/linter.py

@@ -8,7 +8,7 @@ from agents import Agent, Runner, function_tool
 from openkb.agent.tools import list_wiki_files, read_wiki_file
 
 MAX_TURNS = 50
-from openkb.schema import SCHEMA_MD, get_agents_md
+from openkb.schema import get_agents_md
 
 _LINTER_INSTRUCTIONS_TEMPLATE = """\
 You are OpenKB's semantic lint agent. Your job is to audit the wiki

{openkb-0.1.4.dev0 → openkb-0.2.0}/openkb/agent/query.py

@@ -113,7 +113,7 @@ async def run_query(
         The agent's final answer as a string.
     """
     import sys
-    from agents import RawResponsesStreamEvent, RunItemStreamEvent, ItemHelpers
+    from agents import RawResponsesStreamEvent, RunItemStreamEvent
     from openai.types.responses import ResponseTextDeltaEvent
     from openkb.config import load_config
 

{openkb-0.1.4.dev0 → openkb-0.2.0}/openkb/agent/tools.py

@@ -6,6 +6,7 @@ tested in isolation without requiring the openai-agents runtime.
 """
 from __future__ import annotations
 
+import contextlib
 import json as _json
 from pathlib import Path
 
@@ -71,7 +72,9 @@ def parse_pages(pages: str) -> list[int]:
             segments = part.split("-")
             # Re-join to handle leading negatives: segments[0] may be empty
             # if part starts with "-".  We just try to parse start/end.
-            try:
+            # Silently skip malformed segments — parse_pages is a tolerant
+            # parser by design (user-supplied page specs may contain typos).
+            with contextlib.suppress(ValueError):
                 if len(segments) == 2:
                     start, end = int(segments[0]), int(segments[1])
                     result.update(range(start, end + 1))
@@ -79,13 +82,9 @@ def parse_pages(pages: str) -> list[int]:
                     # e.g. "-1" split gives ['', '1']
                     result.add(-int(segments[1]))
                 # More complex cases (e.g. negative range) are ignored.
-            except ValueError:
-                pass
         else:
-            try:
+            with contextlib.suppress(ValueError):
                 result.add(int(part))
-            except ValueError:
-                pass
     return sorted(n for n in result if n > 0)