docshelf-mcp 0.2.0__py3-none-any.whl

docshelf_mcp/__init__.py

@@ -0,0 +1,20 @@
+"""docshelf-mcp — MCP server for managing AI-friendly document collections.
+
+A document collection (`shelf`) is a git-tracked folder that:
+
+* Holds source documents (PDF or Markdown) organized into categories.
+* Auto-converts PDFs to Markdown and splits oversized files by H2 heading.
+* Generates an `INDEX.md` navigation page with raw GitHub URLs so an LLM
+  can fetch only the section it needs instead of swallowing the whole file.
+
+Use `docshelf_mcp.server:main` (or the `docshelf-mcp` console script) to
+launch the MCP server. Use `docshelf_mcp.core.shelf:Shelf` for direct
+library access.
+"""
+
+from __future__ import annotations
+
+from docshelf_mcp.core.shelf import Shelf
+
+__version__ = "0.1.0"
+__all__ = ["Shelf", "__version__"]

docshelf_mcp/__main__.py

@@ -0,0 +1,6 @@
+"""Allow ``python -m docshelf_mcp``."""
+
+from docshelf_mcp.server import main
+
+if __name__ == "__main__":
+    main()

docshelf_mcp/config.py

@@ -0,0 +1,28 @@
+"""Server-level configuration (env vars).
+
+Right now the only knob is ``DOCSHELF_ROOT`` — the default shelf directory
+used when an MCP tool is called without an explicit ``shelf_path``. Override
+it via env to point at your shelf:
+
+.. code-block:: shell
+
+    export DOCSHELF_ROOT="$HOME/Documents/my-docs"
+
+If unset, tools fall back to ``$PWD``, which is the right behaviour when the
+server is started from a shelf directory.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+__all__ = ["default_shelf_root"]
+
+
+def default_shelf_root() -> Path:
+    """Resolve the default shelf root for tool calls without an explicit path."""
+    env = os.environ.get("DOCSHELF_ROOT", "").strip()
+    if env:
+        return Path(env).expanduser().resolve()
+    return Path.cwd().resolve()

docshelf_mcp/core/__init__.py

@@ -0,0 +1,6 @@
+"""Core building blocks: PDF conversion, splitting, indexing, shelf state."""
+
+from docshelf_mcp.core.shelf import Shelf
+from docshelf_mcp.core.slugify import slugify
+
+__all__ = ["Shelf", "slugify"]

docshelf_mcp/core/converter.py

@@ -0,0 +1,88 @@
+"""PDF → Markdown conversion.
+
+Two engines:
+
+* ``"fast"`` — `pymupdf4llm <https://pypi.org/project/pymupdf4llm/>`__.
+  Pure-Python, no GPU, ~1 second per 100 pages. Good enough for most
+  technical manuals. **Default.**
+* ``"high"`` — `marker-pdf <https://pypi.org/project/marker-pdf/>`__.
+  Higher fidelity (tables, equations), but heavy (PyTorch). Optional —
+  only imported if requested.
+
+Both code paths are deferred imports: importing ``docshelf_mcp`` doesn't
+pull in PyMuPDF or marker until you actually call :func:`pdf_to_markdown`.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Literal
+
+__all__ = ["pdf_to_markdown", "Quality"]
+
+Quality = Literal["fast", "high"]
+
+
+class ConversionError(RuntimeError):
+    """Raised when PDF conversion fails or a required engine is missing."""
+
+
+def _convert_fast(pdf_path: Path) -> str:
+    try:
+        import pymupdf4llm  # type: ignore[import-not-found]
+    except ImportError as exc:
+        raise ConversionError(
+            "pymupdf4llm is required for quality='fast' but is not installed. "
+            "Install it with: pip install pymupdf4llm"
+        ) from exc
+
+    return pymupdf4llm.to_markdown(str(pdf_path))
+
+
+def _convert_high(pdf_path: Path) -> str:
+    try:
+        from marker.converters.pdf import PdfConverter  # type: ignore[import-not-found]
+        from marker.models import create_model_dict  # type: ignore[import-not-found]
+        from marker.output import text_from_rendered  # type: ignore[import-not-found]
+    except ImportError as exc:
+        raise ConversionError(
+            "marker-pdf is required for quality='high' but is not installed. "
+            "Install it with: pip install 'docshelf-mcp[high-quality]' or "
+            "pip install marker-pdf. Note: marker-pdf pulls in PyTorch (~2 GB)."
+        ) from exc
+
+    converter = PdfConverter(artifact_dict=create_model_dict())
+    rendered = converter(str(pdf_path))
+    text, _, _ = text_from_rendered(rendered)
+    return text
+
+
+def pdf_to_markdown(pdf_path: Path | str, quality: Quality = "fast") -> str:
+    """Convert a PDF file to Markdown.
+
+    Args:
+        pdf_path: Path to the source PDF.
+        quality: ``"fast"`` (default, pymupdf4llm) or ``"high"`` (marker-pdf).
+
+    Returns:
+        The extracted Markdown text. NOT cleaned —
+        run :func:`docshelf_mcp.core.splitter.clean_markdown` to remove
+        PDF-extraction artefacts.
+
+    Raises:
+        FileNotFoundError: If ``pdf_path`` doesn't exist.
+        ConversionError: If the requested engine is missing or fails.
+    """
+    pdf_path = Path(pdf_path).expanduser().resolve()
+    if not pdf_path.exists():
+        raise FileNotFoundError(f"PDF not found: {pdf_path}")
+    if pdf_path.suffix.lower() != ".pdf":
+        raise ConversionError(
+            f"Expected a .pdf file, got {pdf_path.suffix or '<no extension>'}"
+        )
+
+    if quality == "fast":
+        return _convert_fast(pdf_path)
+    if quality == "high":
+        return _convert_high(pdf_path)
+    raise ConversionError(f"Unknown quality preset: {quality!r}")

docshelf_mcp/core/indexer.py

@@ -0,0 +1,267 @@
+"""Generate ``INDEX.md`` — the navigation page for a shelf.
+
+The index is the *one* file you attach to your AI chat / Claude project.
+For every document in the shelf, it lists:
+
+* The category bucket (e.g. ``## Motherboards``).
+* A human title + short description.
+* A `raw.githubusercontent.com` URL the model can fetch on demand.
+* For split documents: every section as its own link.
+
+The index is **regenerated** every time the shelf changes — the source of
+truth is the on-disk shelf state, not the previous index.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+__all__ = [
+    "DocumentEntry",
+    "build_index",
+    "scan_shelf",
+    "raw_github_url",
+    "DEFAULT_PREAMBLE",
+]
+
+
+@dataclass
+class DocumentEntry:
+    """One document in a shelf."""
+
+    category: str
+    title: str
+    description: str
+    #: Path relative to the shelf root, e.g. ``"docs/motherboards/asus-x870.md"``.
+    relative_path: str
+    size_bytes: int
+    #: Section files (relative paths) if the document was split. Empty list
+    #: means the document is a single self-contained file.
+    section_paths: list[str] = field(default_factory=list)
+
+
+DEFAULT_PREAMBLE = (
+    "This is the navigation index for the document shelf. "
+    "Each entry below links to a Markdown file on GitHub (raw URL) — "
+    "an AI assistant can fetch only the section it needs instead of "
+    "loading the entire collection. Large documents are split by chapter; "
+    "follow the section links for those."
+)
+
+
+def raw_github_url(remote: str, branch: str, relative_path: str) -> str:
+    """Build a ``raw.githubusercontent.com`` URL.
+
+    Args:
+        remote: GitHub URL of the form ``https://github.com/owner/repo`` or
+            ``git@github.com:owner/repo.git``.
+        branch: Branch name (e.g. ``"main"``).
+        relative_path: Path relative to the repo root, forward-slashed.
+
+    Returns:
+        Empty string if ``remote`` cannot be parsed (so the index still renders).
+    """
+    owner_repo = _parse_owner_repo(remote)
+    if owner_repo is None:
+        return ""
+    owner, repo = owner_repo
+    return f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/{relative_path}"
+
+
+_GITHUB_HTTPS = re.compile(r"github\.com[:/](?P<owner>[^/]+)/(?P<repo>[^/.]+)")
+
+
+def _parse_owner_repo(remote: str) -> tuple[str, str] | None:
+    m = _GITHUB_HTTPS.search(remote)
+    if not m:
+        return None
+    repo = m.group("repo")
+    if repo.endswith(".git"):
+        repo = repo[:-4]
+    return m.group("owner"), repo
+
+
+def scan_shelf(shelf_root: Path) -> list[DocumentEntry]:
+    """Walk ``shelf_root/docs/`` and produce a :class:`DocumentEntry` list.
+
+    Conventions:
+
+    * ``docs/<category>/<doc>.md`` — a regular single-file document.
+    * ``docs/<category>/<doc>/<NNN>-<slug>.md`` — a split document (the
+      sibling directory of ``<doc>.md`` if both exist).
+    * ``docs/<category>/.meta.json`` — optional override file with the
+      shape ``{"<doc>.md": {"title": "...", "description": "..."}}``.
+    """
+    docs_root = shelf_root / "docs"
+    entries: list[DocumentEntry] = []
+    if not docs_root.exists():
+        return entries
+
+    for category_dir in sorted(p for p in docs_root.iterdir() if p.is_dir()):
+        meta = _load_meta(category_dir)
+        for md_file in sorted(category_dir.glob("*.md")):
+            name = md_file.name
+            override = meta.get(name, {})
+            title = override.get("title") or _title_from_filename(md_file.stem)
+            description = override.get("description", "")
+
+            # Look for a sibling directory of the same stem — split sections.
+            split_dir = category_dir / md_file.stem
+            section_paths: list[str] = []
+            if split_dir.is_dir():
+                section_paths = sorted(
+                    str(p.relative_to(shelf_root).as_posix())
+                    for p in split_dir.glob("*.md")
+                )
+
+            entries.append(
+                DocumentEntry(
+                    category=category_dir.name,
+                    title=title,
+                    description=description,
+                    relative_path=str(md_file.relative_to(shelf_root).as_posix()),
+                    size_bytes=md_file.stat().st_size,
+                    section_paths=section_paths,
+                )
+            )
+    return entries
+
+
+def _load_meta(category_dir: Path) -> dict[str, dict]:
+    meta_path = category_dir / ".meta.json"
+    if not meta_path.is_file():
+        return {}
+    try:
+        return json.loads(meta_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+
+
+def _title_from_filename(stem: str) -> str:
+    """Best-effort: ``asus-x870-plus_ug`` → ``Asus X870 Plus Ug``."""
+    parts = re.split(r"[-_]+", stem)
+    return " ".join(p.capitalize() if p.islower() else p for p in parts if p)
+
+
+def _pretty_section(section_relpath: str) -> str:
+    """``docs/foo/bar/003-power-supplies.md`` → ``Power supplies``."""
+    stem = Path(section_relpath).stem
+    stem = re.sub(r"^\d+-", "", stem)
+    text = stem.replace("-", " ").replace("_", " ").strip()
+    return text.capitalize() if text else stem
+
+
+def build_index(
+    shelf_name: str,
+    entries: list[DocumentEntry],
+    *,
+    remote: str = "",
+    branch: str = "main",
+    preamble: str = DEFAULT_PREAMBLE,
+    category_order: list[str] | None = None,
+) -> str:
+    """Render the INDEX.md text.
+
+    Args:
+        shelf_name: Human-readable shelf name, used as the H1.
+        entries: From :func:`scan_shelf`.
+        remote: GitHub remote URL. Empty string disables raw URL links.
+        branch: Branch for raw URLs.
+        preamble: Intro paragraph below the H1.
+        category_order: Optional pinned order for the first N categories.
+            Unknown categories follow in alphabetical order.
+
+    Returns:
+        The full Markdown text of ``INDEX.md``.
+    """
+    lines: list[str] = []
+    lines.append(f"# {shelf_name}")
+    lines.append("")
+    if remote:
+        lines.append(f"Repo: <{remote}>")
+        lines.append("")
+    if preamble:
+        lines.append(preamble)
+        lines.append("")
+
+    by_cat: dict[str, list[DocumentEntry]] = {}
+    for e in entries:
+        by_cat.setdefault(e.category, []).append(e)
+
+    seen: set[str] = set()
+    ordered: list[str] = []
+    for cat in category_order or []:
+        if cat in by_cat:
+            ordered.append(cat)
+            seen.add(cat)
+    for cat in sorted(by_cat.keys()):
+        if cat not in seen:
+            ordered.append(cat)
+
+    if not ordered:
+        lines.append("_No documents yet. Use the `add_document` tool to start your shelf._")
+        lines.append("")
+        return "\n".join(lines)
+
+    for cat in ordered:
+        lines.append(f"## {_humanize_category(cat)}")
+        lines.append("")
+        for entry in sorted(by_cat[cat], key=lambda e: e.title.lower()):
+            _render_entry(lines, entry, remote, branch)
+        lines.append("")
+
+    lines.append("---")
+    lines.append("")
+    lines.append(
+        "*Auto-generated by [docshelf-mcp]"
+        "(https://github.com/ignatenkofi/docshelf-mcp). "
+        "Edit the source files in `docs/` and call `rebuild_index` to regenerate.*"
+    )
+    lines.append("")
+    return "\n".join(lines)
+
+
+def _humanize_category(cat: str) -> str:
+    return cat.replace("-", " ").replace("_", " ").strip().title()
+
+
+def _render_entry(
+    lines: list[str],
+    entry: DocumentEntry,
+    remote: str,
+    branch: str,
+) -> None:
+    url = raw_github_url(remote, branch, entry.relative_path) if remote else ""
+    label = f"`{Path(entry.relative_path).name}`"
+    if url:
+        full_link = f"[{label}]({url})"
+    else:
+        full_link = label
+
+    desc = entry.description.strip()
+    desc_suffix = f" — {desc}" if desc else ""
+
+    if entry.section_paths:
+        lines.append(f"### {entry.title}{desc_suffix}")
+        lines.append("")
+        size_kb = entry.size_bytes // 1024
+        lines.append(
+            f"Full document: {full_link} "
+            f"(~{size_kb} KB — prefer pulling individual sections below)."
+        )
+        lines.append("")
+        for sect in entry.section_paths:
+            sect_label = _pretty_section(sect)
+            if remote:
+                sect_url = raw_github_url(remote, branch, sect)
+                lines.append(f"- [{sect_label}]({sect_url})")
+            else:
+                lines.append(f"- {sect_label} (`{sect}`)")
+        lines.append("")
+        lines.append(f"  *(sections: {len(entry.section_paths)})*")
+        lines.append("")
+    else:
+        lines.append(f"- **{entry.title}**{desc_suffix} — {full_link}")