llm-wiki-tool 0.1.0__tar.gz

llm_wiki_tool-0.1.0/PKG-INFO

@@ -0,0 +1,134 @@
+Metadata-Version: 2.4
+Name: llm-wiki-tool
+Version: 0.1.0
+Summary: CLI that keeps small, capped, LLM-maintained wikis for coding agents
+Author: Leonardo Marziali
+Project-URL: Homepage, https://github.com/leonardo-marziali/llm-wiki
+Project-URL: Changelog, https://github.com/leonardo-marziali/llm-wiki/blob/main/CHANGELOG.md
+Project-URL: Issues, https://github.com/leonardo-marziali/llm-wiki/issues
+Keywords: wiki,llm,claude,knowledge-base,coding-agents
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: typer~=0.27
+Requires-Dist: pydantic~=2.14
+Requires-Dist: pyyaml~=6.0
+Requires-Dist: rapidfuzz~=3.14
+Requires-Dist: platformdirs~=4.13
+Requires-Dist: markitdown[docx,pdf,pptx,xlsx]~=0.1.9
+Requires-Dist: python-pptx~=1.0
+Requires-Dist: openpyxl~=3.1
+Requires-Dist: pypdf~=6.21
+Requires-Dist: pillow~=13.0
+Requires-Dist: imagehash~=4.3
+Provides-Extra: docling
+Requires-Dist: docling~=2.142; extra == "docling"
+Provides-Extra: dev
+Requires-Dist: pytest~=9.1; extra == "dev"
+Requires-Dist: python-docx~=1.2; extra == "dev"
+Requires-Dist: pre-commit~=4.7; extra == "dev"
+
+# llm-wiki
+
+`wiki` keeps small, capped, LLM-maintained wikis that coding agents read. You
+drop source files (documents, slides, sheets, PDFs, images, notes) into a wiki;
+`wiki ingest` has headless Claude Code plan and write cited pages on a branch,
+and you accept or reject the result at one gate. Agents read the wiki through
+the read-only `wiki-lookup` skill, at no LLM cost.
+
+## Requirements
+
+- Python 3.12 or later
+- git, with `user.name` and `user.email` configured
+- [Claude Code](https://code.claude.com) logged in with a subscription
+  (`claude /login`); `wiki` never uses an API key
+
+## Install
+
+```sh
+pipx install llm-wiki-tool      # or: pipx install 'llm-wiki-tool[docling]'
+wiki doctor                     # checks claude, its flags, login, isolation and image input
+```
+
+The package is `llm-wiki-tool` on PyPI; the command it installs is `wiki`.
+Releases and their notes are in [CHANGELOG.md](CHANGELOG.md).
+
+## Quick start
+
+```sh
+wiki init ~/wikis/payments --description "Payments team conventions and architecture"
+cp design.docx diagram.png ~/wikis/payments/inbox/
+wiki ingest --wiki payments     # plan, write on a branch, then accept or reject
+wiki search "refund flow" --wiki payments
+wiki status --all
+wiki skill install              # the wiki-lookup skill, for every repo
+```
+
+A wiki is either its own git repo or a folder inside an existing repo. Wikis are
+found through `--wiki NAME|PATH`, the current directory, the registry
+(`$LLM_WIKI_HOME/wikis.toml`, written by `wiki init`) or `LLM_WIKI_PATH`, a list
+of wiki directories separated by the platform path separator. Running
+`wiki init` on an existing or cloned wiki only registers it.
+
+Claude Code asks once before it runs the skill. Once you approve, the skill's
+own `allowed-tools` cover `wiki status`, `wiki search` and reading pages. To
+skip the prompt, for example in headless `claude -p` runs, add
+`"Skill(wiki-lookup)"` to `permissions.allow` in your Claude Code settings.
+
+To make one repo prefer a wiki, run
+`wiki skill install --project <repo> --pointer --wiki <name>`. It writes a short
+marked block into `<repo>/CLAUDE.md`.
+
+## Commands
+
+| Command              | What it does                                                     |
+| -------------------- | ---------------------------------------------------------------- |
+| `wiki init`          | Create or register a wiki                                        |
+| `wiki ingest`        | Ingest the inbox, paths or `--note` text, then one gate          |
+| `wiki search`        | Ranked page search for agents (`--format json`)                  |
+| `wiki status`        | Usage against caps, stale pages, pending and quarantined         |
+| `wiki lint`          | Deterministic checks of pages, links, citations and caps         |
+| `wiki lint --deep`   | Also a sampled LLM pass; writes `.wiki/reports/lint-<date>.md`   |
+| `wiki lint --fix`    | Repair the findings on a branch, then one gate (`--yes`)         |
+| `wiki consolidate`   | Split, merge, promote and archive pages, then one gate (`--yes`) |
+| `wiki skill install` | Install the read-only `wiki-lookup` skill                        |
+| `wiki doctor`        | Check `claude`, its flags, login, isolation and read denial      |
+
+## Caps and consolidate
+
+Every wiki has hard caps (500 pages, 100 per section, 100 sources); `wiki.toml`
+may only lower them, under `[caps]`, and each `warn_*` level defaults to 80% of
+its `max_*`. Near a cap, `wiki status` prints a `near cap:` line. At the page
+cap, ingest still updates existing pages but refuses new ones: the source is
+kept in `raw/` as failed, and the message ends with "run `wiki consolidate` or
+start a new wiki". `wiki consolidate` then frees room on a branch, behind the
+same gate: it retargets links to retired pages, merges duplicate titles (one
+page becomes a `superseded` stub; a to-be twin of an as-is page or a retired
+page is left to you), splits pages over 400 lines while the cap leaves room for
+one new page, and promotes to-be pages whose fact is now built (at most 3 a
+run). Retired pages that nothing links to any more move to
+`<section>/_archive/`, and their ids are never reused for topic pages (a source
+page's id follows its raw path, so a raw file re-added at the same path gets its
+`src-…` id back); the summary names each retired page that still waits and the
+files that link it. At the source cap, ingest refuses new sources instead: start
+a new wiki or delete obsolete raw sources. Maintenance never adds evidence or
+refreshes `last_verified`; stale pages and changed sources stay yours to
+re-check. The next `wiki ingest` retries the failed source.
+
+A page with a secret or injection finding gets no maintenance job, and its text
+never enters a plan or `--deep` prompt; when `wiki/open-questions.md` has such a
+finding, no edit may change it. Edit sessions work in the wiki root with file
+tools; `claude` denies them reading the flagged files and `.wiki/quarantine/`
+(`Read(./…)` rules passed with `--disallowed-tools`, which also keep those files
+out of Grep), and `wiki doctor` probes that the denial holds (`read-denial`).
+
+## Upgrading existing wikis
+
+`wiki init` writes a wiki's `AGENTS.md` (its rules for the model) once, and the
+tool never rewrites it. Wikis created by an older version keep their old rules.
+To pick up new rules, compare your `AGENTS.md` with the packaged template
+(`llm_wiki/templates/AGENTS.md` in the installed package) and copy the changes
+by hand, keeping your wiki's own name, description and sections table.
+
+Wikis created before `wiki lint --deep` existed lack `.wiki/reports/` in their
+`.gitignore`. Add that line so the deep reports stay out of git; the commands
+work either way.

llm_wiki_tool-0.1.0/README.md

@@ -0,0 +1,105 @@
+# llm-wiki
+
+`wiki` keeps small, capped, LLM-maintained wikis that coding agents read. You
+drop source files (documents, slides, sheets, PDFs, images, notes) into a wiki;
+`wiki ingest` has headless Claude Code plan and write cited pages on a branch,
+and you accept or reject the result at one gate. Agents read the wiki through
+the read-only `wiki-lookup` skill, at no LLM cost.
+
+## Requirements
+
+- Python 3.12 or later
+- git, with `user.name` and `user.email` configured
+- [Claude Code](https://code.claude.com) logged in with a subscription
+  (`claude /login`); `wiki` never uses an API key
+
+## Install
+
+```sh
+pipx install llm-wiki-tool      # or: pipx install 'llm-wiki-tool[docling]'
+wiki doctor                     # checks claude, its flags, login, isolation and image input
+```
+
+The package is `llm-wiki-tool` on PyPI; the command it installs is `wiki`.
+Releases and their notes are in [CHANGELOG.md](CHANGELOG.md).
+
+## Quick start
+
+```sh
+wiki init ~/wikis/payments --description "Payments team conventions and architecture"
+cp design.docx diagram.png ~/wikis/payments/inbox/
+wiki ingest --wiki payments     # plan, write on a branch, then accept or reject
+wiki search "refund flow" --wiki payments
+wiki status --all
+wiki skill install              # the wiki-lookup skill, for every repo
+```
+
+A wiki is either its own git repo or a folder inside an existing repo. Wikis are
+found through `--wiki NAME|PATH`, the current directory, the registry
+(`$LLM_WIKI_HOME/wikis.toml`, written by `wiki init`) or `LLM_WIKI_PATH`, a list
+of wiki directories separated by the platform path separator. Running
+`wiki init` on an existing or cloned wiki only registers it.
+
+Claude Code asks once before it runs the skill. Once you approve, the skill's
+own `allowed-tools` cover `wiki status`, `wiki search` and reading pages. To
+skip the prompt, for example in headless `claude -p` runs, add
+`"Skill(wiki-lookup)"` to `permissions.allow` in your Claude Code settings.
+
+To make one repo prefer a wiki, run
+`wiki skill install --project <repo> --pointer --wiki <name>`. It writes a short
+marked block into `<repo>/CLAUDE.md`.
+
+## Commands
+
+| Command              | What it does                                                     |
+| -------------------- | ---------------------------------------------------------------- |
+| `wiki init`          | Create or register a wiki                                        |
+| `wiki ingest`        | Ingest the inbox, paths or `--note` text, then one gate          |
+| `wiki search`        | Ranked page search for agents (`--format json`)                  |
+| `wiki status`        | Usage against caps, stale pages, pending and quarantined         |
+| `wiki lint`          | Deterministic checks of pages, links, citations and caps         |
+| `wiki lint --deep`   | Also a sampled LLM pass; writes `.wiki/reports/lint-<date>.md`   |
+| `wiki lint --fix`    | Repair the findings on a branch, then one gate (`--yes`)         |
+| `wiki consolidate`   | Split, merge, promote and archive pages, then one gate (`--yes`) |
+| `wiki skill install` | Install the read-only `wiki-lookup` skill                        |
+| `wiki doctor`        | Check `claude`, its flags, login, isolation and read denial      |
+
+## Caps and consolidate
+
+Every wiki has hard caps (500 pages, 100 per section, 100 sources); `wiki.toml`
+may only lower them, under `[caps]`, and each `warn_*` level defaults to 80% of
+its `max_*`. Near a cap, `wiki status` prints a `near cap:` line. At the page
+cap, ingest still updates existing pages but refuses new ones: the source is
+kept in `raw/` as failed, and the message ends with "run `wiki consolidate` or
+start a new wiki". `wiki consolidate` then frees room on a branch, behind the
+same gate: it retargets links to retired pages, merges duplicate titles (one
+page becomes a `superseded` stub; a to-be twin of an as-is page or a retired
+page is left to you), splits pages over 400 lines while the cap leaves room for
+one new page, and promotes to-be pages whose fact is now built (at most 3 a
+run). Retired pages that nothing links to any more move to
+`<section>/_archive/`, and their ids are never reused for topic pages (a source
+page's id follows its raw path, so a raw file re-added at the same path gets its
+`src-…` id back); the summary names each retired page that still waits and the
+files that link it. At the source cap, ingest refuses new sources instead: start
+a new wiki or delete obsolete raw sources. Maintenance never adds evidence or
+refreshes `last_verified`; stale pages and changed sources stay yours to
+re-check. The next `wiki ingest` retries the failed source.
+
+A page with a secret or injection finding gets no maintenance job, and its text
+never enters a plan or `--deep` prompt; when `wiki/open-questions.md` has such a
+finding, no edit may change it. Edit sessions work in the wiki root with file
+tools; `claude` denies them reading the flagged files and `.wiki/quarantine/`
+(`Read(./…)` rules passed with `--disallowed-tools`, which also keep those files
+out of Grep), and `wiki doctor` probes that the denial holds (`read-denial`).
+
+## Upgrading existing wikis
+
+`wiki init` writes a wiki's `AGENTS.md` (its rules for the model) once, and the
+tool never rewrites it. Wikis created by an older version keep their old rules.
+To pick up new rules, compare your `AGENTS.md` with the packaged template
+(`llm_wiki/templates/AGENTS.md` in the installed package) and copy the changes
+by hand, keeping your wiki's own name, description and sections table.
+
+Wikis created before `wiki lint --deep` existed lack `.wiki/reports/` in their
+`.gitignore`. Add that line so the deep reports stay out of git; the commands
+work either way.

llm_wiki_tool-0.1.0/pyproject.toml

@@ -0,0 +1,95 @@
+[build-system]
+requires = ["setuptools>=69"]
+build-backend = "setuptools.build_meta"
+
+[project]
+# The import package is llm_wiki and the command is `wiki`. PyPI refuses
+# "llm-wiki" and "llm-wiki-cli" as too close to "llmwiki" and "llmwiki-cli"
+name = "llm-wiki-tool"
+dynamic = ["version"]
+description = "CLI that keeps small, capped, LLM-maintained wikis for coding agents"
+readme = "README.md"
+authors = [{ name = "Leonardo Marziali" }]
+keywords = ["wiki", "llm", "claude", "knowledge-base", "coding-agents"]
+requires-python = ">=3.12"
+dependencies = [
+    "typer~=0.27",
+    "pydantic~=2.14",
+    "pyyaml~=6.0",
+    "rapidfuzz~=3.14",
+    "platformdirs~=4.13",
+    "markitdown[docx,pptx,xlsx,pdf]~=0.1.9",
+    "python-pptx~=1.0",
+    "openpyxl~=3.1",
+    "pypdf~=6.21",
+    "pillow~=13.0",
+    "imagehash~=4.3",
+]
+
+[project.optional-dependencies]
+docling = ["docling~=2.142"]
+dev = ["pytest~=9.1", "python-docx~=1.2", "pre-commit~=4.7"]
+
+[project.urls]
+Homepage = "https://github.com/leonardo-marziali/llm-wiki"
+Changelog = "https://github.com/leonardo-marziali/llm-wiki/blob/main/CHANGELOG.md"
+Issues = "https://github.com/leonardo-marziali/llm-wiki/issues"
+
+[project.scripts]
+wiki = "llm_wiki.cli:main"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.dynamic]
+version = {attr = "llm_wiki.__version__"}
+
+[tool.setuptools.package-data]
+llm_wiki = ["templates/*", "templates/**/*", "prompts/*"]
+
+[tool.semantic_release]
+# .github/workflows/release.yml runs it on every push to main: it bumps
+# __version__, updates CHANGELOG.md, commits, tags and creates the release
+version_variables = ["src/llm_wiki/__init__.py:__version__"]
+tag_format = "v{version}"
+commit_parser = "conventional"
+commit_message = "chore(release): {version} [skip ci]\n\nAutomatically generated by python-semantic-release"
+# Stay on 0.x until 1.0.0 is cut on purpose
+allow_zero_version = true
+major_on_zero = false
+
+[tool.semantic_release.branches.main]
+match = "main"
+
+[tool.semantic_release.changelog]
+mode = "update"
+exclude_commit_patterns = ['chore\(release\): ']
+
+[tool.semantic_release.changelog.default_templates]
+changelog_file = "CHANGELOG.md"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.pyright]
+include = ["src", "tests"]
+pythonVersion = "3.12"
+typeCheckingMode = "standard"
+venvPath = "."
+venv = ".venv"
+# pytest puts tests/ on sys.path, so tests can `from conftest import ...`
+extraPaths = ["tests"]
+
+[tool.ruff]
+target-version = "py312"
+line-length = 88
+
+[tool.ruff.lint]
+# Ruff >= 0.16 defaults already cover F, B, UP, SIM, RUF, PL*, FURB, PT and more
+extend-select = ["E", "W", "I", "UP", "B", "SIM", "RUF"]
+# ‹› delimit redaction marks; – and — appear in user-facing prose on purpose
+allowed-confusables = ["‹", "›", "–"]
+
+[tool.ruff.lint.flake8-bugbear]
+# Typer reads parameter specs from defaults; they are never mutated
+extend-immutable-calls = ["typer.Argument", "typer.Option"]

llm_wiki_tool-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

llm_wiki_tool-0.1.0/src/llm_wiki/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

llm_wiki_tool-0.1.0/src/llm_wiki/archive.py

@@ -0,0 +1,136 @@
+"""Deterministic archiving for ``wiki consolidate`` (Ruling 5): a deprecated or
+superseded page that no live page links to moves to its folder's ``_archive/``
+by ``git mv``. No model is involved.
+"""
+
+from collections import defaultdict
+from contextlib import suppress
+from pathlib import Path
+
+from llm_wiki import gitops
+from llm_wiki.index import GENERATED_MARK
+from llm_wiki.lint import is_flagged, link_targets
+from llm_wiki.maint import RETIRED, live_linkers
+from llm_wiki.manifest import Manifest
+from llm_wiki.pages import id_for, iter_pages
+
+ARCHIVE = "_archive"
+_INDEXES = {"index.md", "_index.md"}
+_ALWAYS_LINKED = {"overview", "glossary"}  # the rebuilt index.md links them
+
+
+def _link_text(path: Path) -> str:
+    """The text whose links stay: an index's generated part is rebuilt without
+    the archived pages, its hand-written header is kept.
+    """
+    text = path.read_text(encoding="utf-8", errors="replace")
+    return text.split(GENERATED_MARK, 1)[0] if path.name in _INDEXES else text
+
+
+def _linked_from(root: Path) -> dict[Path, set[Path]]:
+    """Each link target -> the wiki files outside ``_archive/`` that link it."""
+    wiki = root / "wiki"
+    out: dict[Path, set[Path]] = defaultdict(set)
+    for f in sorted(wiki.rglob("*.md")):
+        if not f.is_file() or ARCHIVE in f.relative_to(wiki).parts[:-1]:
+            continue
+        here = f.resolve()
+        for target in link_targets(f.parent, _link_text(f)) - {here}:
+            out[target].add(here)
+    return out
+
+
+def _select(root: Path) -> tuple[dict[str, str], dict[str, list[str]]]:
+    """(id -> wiki-root-relative path of every page ``archive_unlinked`` moves,
+    id -> the root-relative files that hold back each page that waits).
+
+    A candidate is retired and has no live linker (``maint.live_linkers``, R10:
+    the predicate relink jobs use). It still waits while any file that stays
+    links it (a retired page that is not archived, open-questions.md, an index
+    header): archiving it would break that link. A flagged page stays in sight
+    of lint (Ruling 4), and index.md always links the overview and glossary.
+    """
+    root = Path(root)
+    picked: dict[Path, tuple[str, str]] = {}
+    for p in iter_pages(root):
+        id_ = id_for(p.rel)
+        if (
+            p.fm is not None
+            and p.fm.status in RETIRED
+            and id_ not in _ALWAYS_LINKED
+            and not is_flagged(p)
+            and not live_linkers(root, id_)
+        ):
+            picked[p.path.resolve()] = (id_, f"wiki/{p.rel}")
+    linked_from = _linked_from(root)
+    first = dict(picked)
+    while blocked := [t for t in picked if linked_from[t] - set(picked)]:
+        for t in blocked:
+            del picked[t]
+    top = root.resolve()
+    waiting = {
+        id_: sorted(f.relative_to(top).as_posix() for f in linked_from[t] - set(picked))
+        for t, (id_, _) in first.items()
+        if t not in picked
+    }
+    return dict(sorted(picked.values())), dict(sorted(waiting.items()))
+
+
+def archive_candidates(root: Path) -> list[str]:
+    """The ids ``archive_unlinked`` would archive now."""
+    return list(_select(Path(root))[0])
+
+
+def archive_waiting(root: Path) -> dict[str, list[str]]:
+    """The retired pages without a live linker that still wait, each with the
+    root-relative files that link it and stay.
+    """
+    return _select(Path(root))[1]
+
+
+def _dest(root: Path, rel: str) -> str:
+    """``wiki/<section>/_archive/<slug>.md`` (``wiki/_archive/`` for a top-level
+    page), with ``-2``, ``-3`` … when the name is taken.
+    """
+    folder, name = rel.rsplit("/", 1)
+    stem = name.removesuffix(".md")
+    dest, n = f"{folder}/{ARCHIVE}/{name}", 2
+    while (root / dest).exists():
+        dest = f"{folder}/{ARCHIVE}/{stem}-{n}.md"
+        n += 1
+    return dest
+
+
+def archive_unlinked(root: Path, manifest: Manifest) -> list[str]:
+    """Move every candidate to ``_archive/`` with ``git mv`` and commit the moves;
+    drop the archived ids from every ``pages_touched`` of ``manifest`` (the
+    caller saves it). Returns the archived ids.
+    """
+    root = Path(root)
+    found = _select(root)[0]
+    moved: list[str] = []
+    made: list[Path] = []  # the _archive/ folders this call creates
+    try:
+        for rel in found.values():
+            dest = _dest(root, rel)
+            folder = (root / dest).parent
+            if not folder.is_dir():
+                folder.mkdir()
+                made.append(folder)
+            gitops.move(root, rel, dest)
+            moved += [rel, dest]
+        if moved:
+            message = f"consolidate(archive): {', '.join(found)}"
+            gitops.stage_commit(root, moved, message, force=moved)
+    except BaseException:  # Ctrl-C before the commit: no move is left half-done
+        gitops.restore(root, moved)
+        for folder in made:
+            with suppress(OSError):
+                folder.rmdir()
+        raise
+    gone = set(found)
+    for entry in manifest.entries.values():
+        kept = [i for i in entry.pages_touched if i not in gone]
+        if len(kept) < len(entry.pages_touched):
+            manifest.record(entry.model_copy(update={"pages_touched": kept}))
+    return list(found)

llm_wiki_tool-0.1.0/src/llm_wiki/caps.py

@@ -0,0 +1,21 @@
+"""Hard caps. wiki.toml may only lower them (see config.WikiConfig.effective_caps)."""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Caps:
+    max_sources: int = 100
+    warn_sources: int = 80
+    max_pages: int = 500
+    warn_pages: int = 400
+    max_section_pages: int = 100
+    warn_section_pages: int = 80
+    max_index_lines: int = 99
+    max_page_lines: int = 400
+    max_pages_per_ingest: int = 15
+    lookup_pages: int = 5
+    split_chars: int = 80_000
+
+
+HARD_CAPS = Caps()