passageway-sdk 0.3.0__py3-none-any.whl

passageway_sdk/__init__.py

@@ -0,0 +1,191 @@
+"""Write a passageway connector as three small functions; the protocol plumbing is done here.
+
+Subclass Connector, implement versions() and fetch() (and changed() if the source can answer it cheaply), and call
+YourConnector.main(). The reference is https://passageway.marceltov.de/reference/connector-protocol/.
+"""
+
+import json
+import os
+import sys
+from collections.abc import Iterable, Mapping
+from datetime import datetime
+from typing import NoReturn, TextIO
+
+from passageway_sdk.markdown import first_h1, split_markdown, strip_code
+
+__all__ = ["Connector", "SettingsError", "SourceError", "first_h1", "split_markdown", "strip_code", "validate_item"]
+
+PROTOCOL = "1"
+MODES = "check|list|fetch|changed"
+
+
+class SettingsError(Exception):
+    """The settings are wrong. The connector exits 2 with the message on stderr and prints nothing."""
+
+
+class SourceError(Exception):
+    """The source failed (down, refused, unreadable). The connector exits 1 with the message and prints nothing."""
+
+
+def _check(ok: bool, message: str) -> None:
+    if not ok:
+        raise ValueError(message)
+
+
+def _strings(record: dict, key: str, *, non_empty: bool = False) -> None:
+    values = record.get(key, [])
+    _check(isinstance(values, list) and all(isinstance(v, str) and (v or not non_empty) for v in values),
+           f"'{key}' must be a list of {'non-empty ' if non_empty else ''}strings")
+
+
+def validate_item(record: object) -> None:
+    """Raises ValueError if the record is not a valid item (the same rules the core applies)."""
+    _check(isinstance(record, dict), "a record must be a JSON object")
+    for key in ("title", "url", "version"):
+        _check(isinstance(record.get(key), str), f"'{key}' must be a string")
+    _check(isinstance(record.get("id"), str) and record["id"], "'id' must be a non-empty string")
+    sections = record.get("sections")
+    _check(isinstance(sections, list), "'sections' must be a list")
+    for s in sections:
+        _check(isinstance(s, dict) and isinstance(s.get("text"), str), "a section needs a string 'text'")
+        _check(s.get("heading") is None or isinstance(s["heading"], str), "'heading' must be a string")
+    weight = record.get("weight", 1.0)
+    _check(isinstance(weight, (int, float)) and not isinstance(weight, bool) and 0 <= weight <= 2,
+           "'weight' must be a number from 0 to 2")
+    _strings(record, "path")
+    _strings(record, "labels")
+    _strings(record, "links", non_empty=True)
+    _check(record.get("updated") is None or isinstance(record["updated"], str), "'updated' must be a string")
+
+
+class Connector:
+    """One source. Subclass it; settings arrive in __init__ as the environment."""
+
+    name = "connector"  # prefixes the messages on stderr
+    expected_errors: tuple[type[Exception], ...] = ()  # extra exceptions that mean "the source failed" (exit 1)
+
+    def __init__(self, env: Mapping[str, str]):
+        """Read and validate the settings; raise SettingsError if they are wrong."""
+
+    def versions(self) -> dict[str, str]:
+        """Every item that exists, as id -> version. The version changes when the content changes."""
+        raise NotImplementedError
+
+    def fetch(self, ids: list[str]) -> Iterable[dict]:
+        """The item record for each requested id that still exists."""
+        raise NotImplementedError
+
+    def changed(self, since: datetime) -> Iterable[dict]:
+        """Optional: items changed since `since` (timezone-aware). Print nothing if that is not cheap."""
+        return ()
+
+    def count(self) -> int:
+        return len(self.versions())
+
+    def warn(self, message: str) -> None:
+        """Report something odd (a skipped file, say) on stderr without failing the call."""
+        self.__dict__.setdefault("_warnings", []).append(message)
+
+    # -- the plumbing ------------------------------------------------------------------------
+
+    @classmethod
+    def run(cls, argv: list[str], env: Mapping[str, str], stdin: TextIO, stdout: TextIO, stderr: TextIO,
+            **options) -> int:
+        """Run one mode and return the exit code. Output is written only if everything succeeded.
+
+        A SettingsError gives 2, a SourceError or an expected error 1. Any other exception is a bug in the connector and
+        propagates (the process then exits 1 with a traceback); nothing has been written by then either.
+        """
+        mode = argv[1] if len(argv) > 1 else ""
+        if mode not in MODES.split("|"):
+            print(f"usage: {os.path.basename(argv[0]) if argv else 'connector'} {MODES} (changed takes --since <UTC time>)",
+                  file=stderr)
+            return 2
+        since = None
+        if mode == "changed" and cls.changed is not Connector.changed:  # a connector without `changed` has no use for the time
+            try:
+                since = _since(argv)
+            except ValueError as e:
+                print(f"{cls.name}: {e}", file=stderr)
+                return 2
+        connector = None
+        try:
+            connector = cls(env, **options)
+            lines = connector._lines(mode, stdin, since)
+        except SettingsError as e:
+            _show_warnings(connector, stderr)
+            print(f"{cls.name}: {e}", file=stderr)
+            return 2
+        except (SourceError, *cls.expected_errors) as e:
+            _show_warnings(connector, stderr)
+            print(f"{cls.name}: {e}", file=stderr)
+            return 1
+        _show_warnings(connector, stderr)
+        stdout.write("".join(lines))
+        return 0
+
+    @classmethod
+    def main(cls) -> NoReturn:
+        sys.exit(cls.run(sys.argv, os.environ, sys.stdin, sys.stdout, sys.stderr))
+
+    def _lines(self, mode: str, stdin: TextIO, since: datetime | None) -> list[str]:
+        if mode == "check":
+            count = self.count()
+            if isinstance(count, bool) or not isinstance(count, int) or count < 0:
+                raise SourceError(f"count() must return a whole number from 0, got {count!r}")
+            return [_line({"ok": True, "count": count, "protocol": PROTOCOL})]
+        if mode == "list":
+            return [_line(_listed(i, v)) for i, v in self.versions().items()]
+        if mode == "changed":
+            return [_line(_valid(item)) for item in self.changed(since)]
+        ids = _ids(stdin)
+        lines = []
+        for item in self.fetch(ids):
+            if _valid(item)["id"] not in ids:
+                raise SourceError(f"fetch returned {item['id']!r}, which was not requested")
+            lines.append(_line(item))
+        return lines
+
+
+def _show_warnings(connector: "Connector | None", stderr: TextIO) -> None:
+    for message in getattr(connector, "_warnings", ()):
+        print(f"{connector.name}: warning: {message}", file=stderr)
+
+
+def _line(record: dict) -> str:
+    return json.dumps(record) + "\n"  # ASCII-safe on purpose: a locale without UTF-8 must not crash the output
+
+
+def _valid(item: object) -> dict:
+    try:
+        validate_item(item)
+    except ValueError as e:
+        label = item.get("id") if isinstance(item, dict) else None
+        raise SourceError(f"invalid item{f' {label!r}' if label else ''}: {e}") from None
+    return item
+
+
+def _listed(item_id: object, version: object) -> dict:
+    if not (isinstance(item_id, str) and item_id and isinstance(version, str)):
+        raise SourceError(f"invalid list entry {item_id!r}: an id is a non-empty string and a version a string")
+    return {"id": item_id, "version": version}
+
+
+def _ids(stdin: TextIO) -> list[str]:
+    try:
+        ids = [json.loads(line)["id"] for line in stdin if line.strip()]
+        if not all(isinstance(i, str) and i for i in ids):
+            raise ValueError
+        return ids
+    except (ValueError, KeyError, TypeError):
+        raise SourceError("fetch expects one JSON object per line on stdin, each with a non-empty string 'id'") from None
+
+
+def _since(argv: list[str]) -> datetime:
+    try:
+        value = datetime.fromisoformat(argv[argv.index("--since") + 1].replace("Z", "+00:00"))
+    except (ValueError, IndexError):
+        raise ValueError("changed needs --since <UTC time>, e.g. 2026-01-01T00:00:00Z") from None
+    if value.tzinfo is None:
+        raise ValueError("--since must carry a timezone, e.g. 2026-01-01T00:00:00Z")
+    return value

passageway_sdk/markdown.py

@@ -0,0 +1,78 @@
+"""Markdown helpers for connectors: cut a document into sections at its headings, and see it without code."""
+
+import re
+from collections.abc import Iterator
+
+_FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})(.*)$")
+_HEADING = re.compile(r"^ {0,3}(#{1,6})[ \t]+(.*?)(?:[ \t]+#+)?[ \t]*$")
+_INLINE_CODE = re.compile(r"(?<!`)(`+)(?!`)[^\n]+?(?<!`)\1(?!`)")  # a span closes with a run of exactly the same length
+SEPARATOR = " › "
+
+
+def _lines(text: str) -> Iterator[tuple[str, bool]]:
+    """Each line with whether it is inside a fenced code block (the fence lines themselves count as inside)."""
+    fence = ""  # the opening fence while inside a block
+    for line in text.splitlines():
+        match = _FENCE.match(line)
+        if not fence:
+            # the info string of a backtick fence cannot contain a backtick: "```a`b" is not a fence
+            opens = bool(match) and not (match.group(1)[0] == "`" and "`" in match.group(2))
+            if opens:
+                fence = match.group(1)
+            yield line, opens
+        else:
+            # a closing fence: the same character, at least as long, and nothing after it
+            if match and match.group(1)[0] == fence[0] and len(match.group(1)) >= len(fence) and not match.group(2).strip():
+                fence = ""
+            yield line, True
+
+
+def split_markdown(text: str, *, skip_first_h1: bool = False) -> list[dict]:
+    """Sections for an item: {"heading": "Setup › Linux", "text": "…"}, one per heading level (# to ######).
+
+    The heading is the trail of the parent headings. Text before the first heading has heading None. A `#` inside a fenced
+    code block is not a heading. Sections without text are dropped (their heading stays in the trail of their children).
+    With skip_first_h1 the first level-1 heading is left out of the trail, because it is the title of the item.
+    """
+    sections: list[dict] = []
+    trail: list[tuple[int, str]] = []
+    lines: list[str] = []
+    heading: str | None = None
+    seen_h1 = False
+
+    def flush() -> None:
+        body = "\n".join(lines).strip()
+        if body:
+            sections.append({"heading": heading, "text": body})
+        lines.clear()
+
+    for line, in_code in _lines(text):
+        match = None if in_code else _HEADING.match(line)
+        if not match or not match.group(2).strip():
+            lines.append(line)
+            continue
+        flush()
+        level, title = len(match.group(1)), match.group(2).strip()
+        if skip_first_h1 and level == 1 and not seen_h1:
+            seen_h1 = True
+            trail = []  # the title is above everything: what follows starts again from the top
+        else:
+            trail = [t for t in trail if t[0] < level] + [(level, title)]
+        heading = SEPARATOR.join(t[1] for t in trail) or None
+    flush()
+    return sections
+
+
+def strip_code(text: str) -> str:
+    """The text without fenced code blocks and inline code, e.g. to find the links a document really contains."""
+    outside = "\n".join(line for line, in_code in _lines(text) if not in_code)
+    return _INLINE_CODE.sub("", outside)
+
+
+def first_h1(text: str) -> str | None:
+    """The text of the first level-1 heading outside code, or None: the title of a document that has no other."""
+    for line, in_code in _lines(text):
+        match = None if in_code else _HEADING.match(line)
+        if match and len(match.group(1)) == 1 and match.group(2).strip():
+            return match.group(2).strip()
+    return None

passageway_sdk/settings.py

@@ -0,0 +1,44 @@
+"""Read connector settings from the environment: every value is a string, and a wrong one is a SettingsError."""
+
+from collections.abc import Mapping
+
+from passageway_sdk import SettingsError
+
+
+def names(env: Mapping[str, str], key: str) -> tuple[str, ...]:
+    """A list setting: values separated by `|`, blanks dropped; empty if the setting is missing."""
+    return tuple(p.strip() for p in env.get(key, "").split("|") if p.strip())
+
+
+def whole(env: Mapping[str, str], key: str, default: int, low: int = 1) -> int:
+    """A whole number of at least `low`."""
+    raw = env.get(key, "").strip()
+    try:
+        value = int(raw) if raw else default
+    except ValueError:
+        raise SettingsError(f"{key} must be a whole number, got {raw!r}") from None
+    if value < low:
+        raise SettingsError(f"{key} must be at least {low}, got {value}")
+    return value
+
+
+def flag(env: Mapping[str, str], key: str, default: bool) -> bool:
+    """`true` or `false`, in any case."""
+    raw = env.get(key, "").strip().lower()
+    if not raw:
+        return default
+    if raw not in ("true", "false"):
+        raise SettingsError(f"{key} must be true or false, got {raw!r}")
+    return raw == "true"
+
+
+def seconds(env: Mapping[str, str], key: str, default: float) -> float:
+    """A number of seconds that is not negative."""
+    raw = env.get(key, "").strip()
+    try:
+        value = float(raw) if raw else default
+    except ValueError:
+        raise SettingsError(f"{key} must be a number of seconds, got {raw!r}") from None
+    if value < 0:
+        raise SettingsError(f"{key} must not be negative, got {value}")
+    return value

passageway_sdk/web.py

@@ -0,0 +1,142 @@
+"""A read-mostly web client for connectors and embedders. Needs the extra: `pip install passageway-sdk[web]`."""
+
+import json as jsonlib
+import ssl
+import time
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Any, NamedTuple
+
+from passageway_sdk import SourceError
+
+try:
+    import httpx
+    import truststore
+except ImportError as e:  # the SDK itself has no dependencies; this module is the one that has
+    raise ImportError("passageway_sdk.web needs httpx and truststore: install passageway-sdk[web]") from e
+
+
+class Reply(NamedTuple):
+    status: int
+    text: str  # decoded as UTF-8, undecodable bytes replaced
+    content_type: str
+    headers: Mapping[str, str] = MappingProxyType({})  # names in lower case
+
+    def json(self) -> Any:
+        return jsonlib.loads(self.text)
+
+
+class TooLarge(SourceError):
+    """The reply was larger than the limit."""
+
+
+class HttpError(SourceError):
+    """The server answered with a status the call cannot go on with: a 5xx or 429 that retries did not clear, or any
+    non-200 when the call was `required`."""
+
+    def __init__(self, url: str, status: int, message: str | None = None, attempts: int = 1):
+        super().__init__(message or f"{url}: HTTP {status}")
+        self.url, self.status, self.attempts = url, status, attempts  # attempts > 1: a retry policy gave up
+
+
+class NetworkError(SourceError):
+    """The server could not be reached or went silent (a connection or timeout error)."""
+
+    def __init__(self, url: str, detail: str):
+        super().__init__(f"{url}: {detail}")
+        self.url, self.detail = url, detail
+
+
+@dataclass(frozen=True)
+class Retry:
+    """Wait and try again when the server asks to come back later (seconds)."""
+
+    statuses: tuple[int, ...] = (429, 502, 503, 504)
+    network: bool = False  # also retry when the server cannot be reached or times out
+    budget: float = 900  # the most that is waited in total for one request before giving up
+    max_backoff: float = 60  # the longest wait the client picks itself: 1, 2, 4, … up to this
+    max_after: float = 300  # the longest wait a Retry-After header may ask for
+    say_from: float = 10  # a wait this long is reported to `log`, shorter ones stay quiet
+
+
+class Web:
+    """HTTP with the operating system's trust store (never certifi's bundle), a pause between requests and a size cap.
+
+    A failure to reach the server, or a 5xx or 429 that a `retry` policy did not clear, raises SourceError (HttpError for a
+    status), so a source that is down fails the whole call and never looks empty. Any other status is returned for the
+    connector to judge; `required=True` raises on it too. Turning TLS checking off (`verify=False`) is an explicit switch.
+    """
+
+    def __init__(self, user_agent: str | None = None, *, base_url: str = "", headers: Mapping[str, str] | None = None,
+                 verify: bool = True, follow_redirects: bool = True, pause: float = 0.2, timeout: float = 30.0,
+                 max_bytes: int = 2_000_000, retry: Retry | None = None, log: Callable[[str], None] | None = None, transport: "httpx.BaseTransport | None" = None,
+                 sleep: Callable[[float], None] = time.sleep):
+        all_headers = {**({"User-Agent": user_agent} if user_agent else {}), **(headers or {})}
+        self._http = httpx.Client(base_url=base_url, headers=all_headers, timeout=timeout, follow_redirects=follow_redirects,
+                                  transport=transport, verify=truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT) if verify else False)
+        self._pause, self._max_bytes, self._retry, self._log, self._sleep, self._last = pause, max_bytes, retry, log, sleep, None
+
+    def get(self, url: str, *, params: Mapping[str, str] | None = None, required: bool = False, max_bytes: int | None = None,
+            retry_if: Callable[[Reply], str | None] | None = None) -> Reply:
+        return self.request("GET", url, params=params, required=required, max_bytes=max_bytes, retry_if=retry_if)
+
+    def request(self, method: str, url: str, *, params: Mapping[str, str] | None = None, json: Any = None, required: bool = False,
+                max_bytes: int | None = None, retry_if: Callable[[Reply], str | None] | None = None) -> Reply:
+        """`retry_if(reply)` may name a reason to try again although the status is fine (an API that answers 200 with
+        "I am lagging"); it counts against the same budget as a busy status."""
+        limit, waited, attempt = max_bytes or self._max_bytes, 0.0, 0
+        retry = self._retry
+        if retry_if and not retry:
+            raise ValueError("retry_if needs a retry policy: pass retry=Retry(...) to Web")
+        while True:
+            attempt += 1
+            retry_after = None
+            try:
+                reply, retry_after = self._once(method, url, params, json, limit)
+                reason = f"HTTP {reply.status}"
+                if again := retry_if(reply) if retry_if else None:
+                    reason = again
+                elif not (retry and reply.status in retry.statuses):
+                    if reply.status >= 500 or reply.status == 429 or (required and reply.status != 200):
+                        raise HttpError(url, reply.status)
+                    return reply
+            except NetworkError as e:
+                if not (retry and retry.network):
+                    raise
+                reply, reason = None, e.detail
+            if waited >= retry.budget:
+                message = f"{url}: giving up after {attempt} attempts and {waited:.0f} s of waiting: {reason}"
+                raise HttpError(url, reply.status, message, attempt) if reply else NetworkError(url, message.removeprefix(f"{url}: "))
+            wait = min(self._wait(attempt, retry_after), retry.budget - waited)
+            if wait >= retry.say_from and self._log:
+                self._log(f"{reason} on {method} {url}: waiting {wait:.0f} s ({waited:.0f} s of {retry.budget:.0f} s waited)")
+            self._sleep(wait)
+            waited += wait
+
+    def _wait(self, attempt: int, retry_after: str | None) -> float:
+        backoff = min(float(2 ** (attempt - 1)), self._retry.max_backoff)
+        try:
+            return max(backoff, min(float(retry_after), self._retry.max_after)) if retry_after else backoff
+        except ValueError:
+            return backoff
+
+    def _once(self, method: str, url: str, params, json, limit: int) -> tuple[Reply, str | None]:
+        if self._last is not None and (wait := self._pause - (time.monotonic() - self._last)) > 0:
+            self._sleep(wait)
+        try:
+            with self._http.stream(method, url, params=params, json=json) as response:
+                body = bytearray()
+                for chunk in response.iter_bytes():
+                    body += chunk
+                    if len(body) > limit:
+                        raise TooLarge(f"{url}: larger than {limit} bytes")
+                headers = {k.lower(): v for k, v in response.headers.items()}
+                reply = Reply(response.status_code, body.decode("utf-8", errors="replace"), headers.get("content-type", ""), headers)
+                return reply, headers.get("retry-after")
+        except httpx.TransportError as e:
+            raise NetworkError(url, f"{type(e).__name__}: {e}") from e
+        except httpx.HTTPError as e:
+            raise SourceError(f"{url}: {e}") from e
+        finally:
+            self._last = time.monotonic()

passageway_sdk-0.3.0.dist-info/METADATA

@@ -0,0 +1,27 @@
+Metadata-Version: 2.5
+Name: passageway-sdk
+Version: 0.3.0
+Summary: Write a passageway connector as three small functions: the protocol plumbing is done for you
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.12
+Provides-Extra: web
+Requires-Dist: httpx>=0.28.1; extra == 'web'
+Requires-Dist: truststore>=0.10; extra == 'web'
+Description-Content-Type: text/markdown
+
+# passageway-sdk
+
+Write a [passageway](https://passageway.marceltov.de/) connector as three small functions. No dependencies; `passageway-sdk[web]` adds a polite web client for connectors that read a website.
+
+```python
+from passageway_sdk import Connector
+
+class Notes(Connector):
+    def versions(self): ...      # {id: version} for every item
+    def fetch(self, ids): ...    # an item record per id
+
+Notes.main()
+```
+
+The SDK dispatches the mode, reads the ids, buffers the output (all or nothing), maps errors to exit codes and validates each item. Reference: https://passageway.marceltov.de/reference/sdk/ and the [guide](https://passageway.marceltov.de/guides/write-a-connector/).

passageway_sdk-0.3.0.dist-info/RECORD

@@ -0,0 +1,8 @@
+passageway_sdk/__init__.py,sha256=2i9FveaYIg9RsFWCQ8PjV7--jZ98F5UHBc_BPSc_O2A,8247
+passageway_sdk/markdown.py,sha256=jkcCFLNoZEeR2ahXBDvZWNT5XMjYKfrWFUGLJxrV7b8,3481
+passageway_sdk/settings.py,sha256=YsTnQ1ptbVi4X7N6go_IlBVdhePNkwKb9ApdE9V8TnU,1633
+passageway_sdk/web.py,sha256=ff8rybmOkRTLeGB0hEi75Xrv_WCuZOQWZL5qKkBOtgk,7492
+passageway_sdk-0.3.0.dist-info/METADATA,sha256=ppWfacQFKh_m32Bb0waV7AXBwG--k1IQLM_ioIZPH3s,1072
+passageway_sdk-0.3.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+passageway_sdk-0.3.0.dist-info/licenses/LICENSE,sha256=AhfRnZW5Yfi9VuESZGyPTziynbdP9kNMOtCykeDpAyw,1072
+passageway_sdk-0.3.0.dist-info/RECORD,,

passageway_sdk-0.3.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

passageway_sdk-0.3.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Marcel Bruckner
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.