passageway-connector-mkdocs 0.4.0__py3-none-any.whl

passageway_connector_mkdocs/__main__.py

@@ -0,0 +1,3 @@
+from passageway_connector_mkdocs.connector import main
+
+main()

passageway_connector_mkdocs/connector.py

@@ -0,0 +1,146 @@
+"""passageway connector for websites built with MkDocs: one item per section of the site's search index."""
+
+import hashlib
+import json
+import time
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass
+from urllib.parse import urldefrag, urlparse
+
+import httpx
+from passageway_sdk import Connector, SettingsError, SourceError
+from passageway_sdk.html import clean, to_markdown
+from passageway_sdk.settings import names, weight_of, weights, whole
+from passageway_sdk.web import TooLarge, Web
+
+INDEX = "search/search_index.json"
+
+
+@dataclass(frozen=True)
+class Section:
+    id: str  # the address: the site, the page, and the heading's anchor if it has one
+    page_title: str
+    heading: str | None  # None for the text before the page's first heading
+    text: str
+    where: str  # the page's location below the site, for WEIGHTS and the breadcrumb
+
+
+def _base(url: str) -> str:
+    """The site an address names: the folder of a search index, else the address itself, ending in `/`."""
+    url = urldefrag(url)[0]
+    if url.endswith(INDEX):
+        return url.removesuffix(INDEX)
+    return url if url.endswith("/") else url + "/"
+
+
+def _intro(page_text: str, first: str | None) -> str:
+    """The page entry of MkDocs' own search plugin holds the whole page; keep what comes before its first section."""
+    if first:
+        at = page_text.find(first)
+        if at > 0:
+            return page_text[:at]
+    return page_text
+
+
+def sections(base: str, docs: list[dict]) -> list[Section]:
+    """The sections of a search index, in its order; entries without text are left out, a repeated address counts once."""
+    pages: dict[str, str] = {}  # page location -> page title
+    firsts: dict[str, tuple[str, str]] = {}  # page location -> (title, text) of its first section with text
+    texts = {d["location"]: to_markdown(d["text"]) for d in docs}
+    for d in docs:
+        page, _, anchor = d["location"].partition("#")
+        if not anchor:
+            pages.setdefault(page, clean(d["title"]))
+        elif texts[d["location"]]:
+            firsts.setdefault(page, (clean(d["title"]), texts[d["location"]]))
+    out, seen = [], set()
+    for d in docs:
+        location = d["location"]
+        if location in seen:
+            continue
+        seen.add(location)
+        page, _, anchor = location.partition("#")
+        text = texts[location]
+        if not anchor:
+            # the first section of this page, with its heading, starts where the intro ends
+            first_title, first = firsts.get(page, (None, None))
+            text = _intro(text, first)
+            if first_title and text.rstrip().endswith(first_title):
+                text = text.rstrip()[: -len(first_title)]
+            text = text.strip()
+        if text:
+            title = pages.get(page) or clean(d["title"]) or urlparse(base).netloc
+            out.append(Section(base + location, title, clean(d["title"]) if anchor else None, text, page))
+    return out
+
+
+class MkDocs(Connector):
+    name = "mkdocs connector"
+
+    def __init__(self, env: Mapping[str, str], *, transport: httpx.BaseTransport | None = None,
+                 sleep: Callable[[float], None] = time.sleep):
+        urls = names(env, "URLS")
+        if not urls:
+            raise SettingsError("URLS must name at least one site")
+        for url in urls:
+            if urlparse(url).scheme not in ("http", "https") or not urlparse(url).netloc:
+                raise SettingsError(f"URLS must be http(s) addresses, got {url!r}")
+        self.sites = list(dict.fromkeys(_base(u) for u in urls))
+        self.max_bytes = whole(env, "MAX_BYTES", 50_000_000)
+        self.weights = weights(env, "WEIGHTS")
+        self._web = Web("passageway-connector-mkdocs", max_bytes=self.max_bytes, transport=transport, sleep=sleep)
+
+    # -- the protocol ------------------------------------------------------------------------
+
+    def versions(self) -> dict[str, str]:
+        return {s.id: self._version(s) for site in self.sites for s in self._sections(site)}
+
+    def fetch(self, ids: list[str]) -> Iterator[dict]:
+        wanted = set(ids)
+        for site in self.sites:
+            if any(i.startswith(site) for i in wanted):
+                for section in self._sections(site):
+                    if section.id in wanted:
+                        yield self._item(section)
+
+    # -- the sites ---------------------------------------------------------------------------
+
+    def _sections(self, site: str) -> list[Section]:
+        url = site + INDEX
+        try:
+            text = self._web.get(url, required=True).text
+        except TooLarge:
+            raise SourceError(f"{url}: larger than MAX_BYTES ({self.max_bytes})") from None
+        try:
+            docs = json.loads(text)["docs"]
+            docs = [{"location": str(d["location"]), "title": str(d.get("title") or ""), "text": str(d.get("text") or "")}
+                    for d in docs]
+        except (ValueError, KeyError, TypeError):
+            raise SourceError(f"{url}: not an MkDocs search index") from None
+        return sections(site, docs)
+
+    def _weight(self, section: Section) -> float:
+        return weight_of(self.weights, section.where)
+
+    def _version(self, section: Section) -> str:
+        weight = self._weight(section)  # a changed weight is a changed item, so the index follows it
+        body = f"{section.page_title}\0{section.heading}\0{section.text}" + (f"\0{weight}" if weight != 1.0 else "")
+        return hashlib.sha256(body.encode()).hexdigest()[:16]
+
+    def _item(self, section: Section) -> dict:
+        item = {
+            "id": section.id,
+            "title": section.page_title,
+            "url": section.id,
+            "version": self._version(section),
+            "sections": [{"heading": section.heading, "text": section.text}],
+        }
+        if folders := [p for p in section.where.split("/") if p]:
+            item["path"] = folders
+        if (weight := self._weight(section)) != 1.0:
+            item["weight"] = weight
+        return item
+
+
+def main() -> None:
+    MkDocs.main()

passageway_connector_mkdocs-0.4.0.dist-info/METADATA

@@ -0,0 +1,80 @@
+Metadata-Version: 2.5
+Name: passageway-connector-mkdocs
+Version: 0.4.0
+Summary: passageway connector for websites built with MkDocs, read from their search index
+License-Expression: PolyForm-Noncommercial-1.0.0
+Requires-Python: >=3.12
+Requires-Dist: passageway-sdk[html,web]~=0.4.0
+Description-Content-Type: text/markdown
+
+# passageway-connector-mkdocs
+
+A [passageway](https://github.com/getpassageway/passageway#readme) connector that indexes a website built with [MkDocs](https://www.mkdocs.org/) or [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/): someone else's documentation, when you have the site but not its repository. It reads the site's search index, the file its search box uses, so it needs no crawling, and every hit opens the page at its heading.
+
+## What it does
+
+- **Each section of the search index is one item.** A built MkDocs site publishes `search/search_index.json`, with an entry for every page and every heading in it. Each entry with text becomes an item whose address is the page at that heading (`https://docs.example.org/guide/setup/#linux`), so a hit lands on the passage, not at the top of a long page. The text before a page's first heading is an item of its own at the page's address.
+- **The page title is the item's title, the heading is its section,** and the folders of the address are its breadcrumb (`path`), so a `path:` filter works.
+- **The id is the address, the version a hash of the text,** so an edit re-indexes only the sections it touched.
+- **A site address means its search index.** `https://docs.example.org` is read as `https://docs.example.org/search/search_index.json`; a site under a path (`https://user.github.io/project`) works the same, and the index's own address can be given too.
+- **Both search plugins are understood.** Material for MkDocs writes HTML into the index, which becomes text with code blocks and lists kept. MkDocs' own search plugin writes plain text and repeats the whole page in the page's entry; only the part before the first heading is kept there, so a passage is not indexed twice.
+- **Any failure fails the whole listing.** A site without a search index, an index that is not one, a server that is down: the sync stops with a message that names the address, so a half-read site never deletes what was indexed before.
+
+When the site publishes an `llms-full.txt` (the `mkdocs-llmstxt` plugin writes one), the [llms.txt connector](https://getpassageway.com/connectors/llmstxt/) reads the markdown itself; when you have the repository, the [Markdown connector](https://getpassageway.com/connectors/markdown/) does. This connector is for the site as it is published.
+
+## Install
+
+Add the connector to the environment that runs passageway (see [Install](https://github.com/getpassageway/passageway/blob/main/docs/install.md)):
+
+```bash
+uv tool install --python 3.14 "passageway[mkdocs]"
+```
+
+## An example
+
+A site `https://docs.example.org` whose `search/search_index.json` is:
+
+```json
+{"docs": [
+  {"location": "guide/setup/", "title": "Setup", "text": "<p>How to set it up.</p>"},
+  {"location": "guide/setup/#linux", "title": "Linux", "text": "<p>Use the <code>apt</code> package.</p>"}
+]}
+```
+
+gives, for the second entry, the item:
+
+```json
+{
+  "id": "https://docs.example.org/guide/setup/#linux",
+  "title": "Setup",
+  "url": "https://docs.example.org/guide/setup/#linux",
+  "path": ["guide", "setup"],
+  "sections": [
+    {
+      "heading": "Linux",
+      "text": "Use the apt package."
+    }
+  ]
+}
+```
+
+and is configured as:
+
+```toml
+[[connector]]
+name = "vendor-docs"
+command = ["/path/to/python", "-m", "passageway_connector_mkdocs"]
+env = { URLS = "https://docs.example.org|https://user.github.io/project", WEIGHTS = "changelog=0.5" }
+```
+
+## Settings
+
+All settings are strings in the connector's `env`.
+
+| Setting | Meaning | Default |
+|---|---|---|
+| `URLS` | Required. Site addresses, or the addresses of their `search_index.json`, separated by `\|`. | |
+| `MAX_BYTES` | The largest search index that is read, in bytes. A larger one fails the sync. | `50000000` |
+| `WEIGHTS` | Lowers or raises the weight of the sections whose path starts with a prefix: `changelog=0.5,blog/=0.3` (comma-separated `prefix=weight`, the weight above 0, the first match wins). Release notes repeat the vocabulary of how-to pages and push them down the results; a weight below 1 ranks them lower. The path is the page's address without the site's own. | all sections 1.0 |
+
+Requests carry the user agent `passageway-connector-mkdocs`, follow redirects and verify HTTPS against the operating system's trust store.

passageway_connector_mkdocs-0.4.0.dist-info/RECORD

@@ -0,0 +1,6 @@
+passageway_connector_mkdocs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+passageway_connector_mkdocs/__main__.py,sha256=0vK3RfxGFBCp9nAQUG9GYa1iL8jm87jQ_JRnfwNaVLg,63
+passageway_connector_mkdocs/connector.py,sha256=CLdtcRHBpr5xazO6qN1mcZXp_qk_bKr8H62lWi0YGI0,6096
+passageway_connector_mkdocs-0.4.0.dist-info/METADATA,sha256=wOQ8xoXravPXc1q3aut_n7JhcNwcFKi4Rob-1-KCAZ0,4605
+passageway_connector_mkdocs-0.4.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+passageway_connector_mkdocs-0.4.0.dist-info/RECORD,,

passageway_connector_mkdocs-0.4.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any