ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/site.py

@@ -0,0 +1,84 @@
+"""Which site a run belongs to: the key for its memory and its output folder.
+
+Usually that is the host ("hn.algolia.com"). When the start URL begins with a path that
+names a section of the site, that first segment is kept too ("www.google.com/maps"), so
+one big host's separate apps (Maps, Search, Support...) don't share memory and files.
+The host alone still decides what counts as the site's own API versus a third party.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from urllib.parse import urlsplit
+
+from ghost_api.analysis.url_grouping import looks_like_id
+
+# A section name: a short plain word, no dots (files), no encoded data.
+_SECTION_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_-]{0,39}$")
+# Locale prefixes (/en/, /en-us/, /pt_BR/) say which language, not which app.
+_LOCALE_RE = re.compile(r"^[a-z]{2}([-_][A-Za-z]{2})?$")
+_HOST_RE = r"[a-z0-9]([a-z0-9.-]*[a-z0-9])?"
+SITE_RE = re.compile(rf"^{_HOST_RE}(/[A-Za-z][A-Za-z0-9_-]{{0,39}})?$")
+
+
+def site_key(url: str) -> str:
+    """'https://www.google.com/maps/@12.9,77.6' -> 'www.google.com/maps';
+    'https://hn.algolia.com/' -> 'hn.algolia.com'."""
+    parts = urlsplit(url)
+    host = (parts.hostname or "").lower()
+    segments = [s for s in parts.path.split("/") if s]
+    if segments and is_section(segments[0]):
+        return f"{host}/{segments[0]}"
+    return host
+
+
+def is_section(segment: str) -> bool:
+    return bool(
+        _SECTION_RE.match(segment) and not _LOCALE_RE.match(segment) and not looks_like_id(segment)
+    )
+
+
+def site_host(site: str) -> str:
+    """The host part: 'www.google.com/maps' -> 'www.google.com'."""
+    return site.split("/", 1)[0]
+
+
+def is_site(name: str) -> bool:
+    """A safe site name (used in file paths and URLs): host, optionally /section."""
+    return bool(SITE_RE.match(name))
+
+
+# In dashboard URLs a site is one path segment, so its "/" is written as "~".
+def to_url_id(site: str) -> str:
+    return site.replace("/", "~")
+
+
+def from_url_id(url_id: str) -> str:
+    return url_id.replace("~", "/", 1)
+
+
+class SiteAmbiguous(ValueError):
+    def __init__(self, name: str, choices: list[str]) -> None:
+        self.choices = choices
+        super().__init__(f"{name} has several sites: {', '.join(choices)}. Name one of them.")
+
+
+def resolve_site(name: str, known: Iterable[str]) -> str:
+    """What the user typed -> a stored site name.
+
+    Accepts a site ('www.google.com/maps'), a URL, or a bare domain. A bare domain with
+    exactly one stored site under it means that site; with several, SiteAmbiguous.
+    Unknown names are returned as typed (the caller reports "nothing stored")."""
+    name = name.strip().rstrip("/")
+    if "://" in name:
+        name = site_key(name)
+    sites = set(known)
+    if name in sites:
+        return name
+    under = sorted(s for s in sites if s.startswith(name + "/"))
+    if len(under) == 1:
+        return under[0]
+    if under:
+        raise SiteAmbiguous(name, under)
+    return name

ghost_api/specdiff.py

@@ -0,0 +1,122 @@
+"""`ghost-api verify-spec`: compare a published OpenAPI spec with what the site really does.
+
+Documentation drifts from reality. Using the endpoints Ghost API observed and verified for
+a site, this reports:
+
+    undocumented     the site uses it, the spec doesn't mention it
+    not seen         the spec documents it, the site never used it while explored
+                     (it may be unused, removed, or just not reached)
+    field gaps       for endpoints in both: reply fields the spec doesn't list, and spec
+                     fields the real replies never contained
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from ghost_api.analysis.graphql import split_template
+from ghost_api.analysis.models import Endpoint
+
+METHODS = ("get", "post", "put", "patch", "delete", "head", "options")
+
+
+class SpecError(Exception):
+    """The file is not an OpenAPI document Ghost API can read."""
+
+
+@dataclass
+class EndpointCheck:
+    endpoint: str  # "GET /api/items/{}"
+    status: str  # "documented", "undocumented", "not seen"
+    extra_fields: list[str] = field(default_factory=list)  # in replies, not in the spec
+    missing_fields: list[str] = field(default_factory=list)  # in the spec, never in replies
+
+
+def load_spec(path: Path) -> dict[str, Any]:
+    try:
+        text = path.read_text(encoding="utf-8")
+        data = json.loads(text) if path.suffix == ".json" else yaml.safe_load(text)
+    except (OSError, ValueError, yaml.YAMLError) as exc:
+        raise SpecError(f"{path.name} can't be read: {exc}") from exc
+    if not isinstance(data, dict) or not isinstance(data.get("paths"), dict):
+        raise SpecError(f"{path.name} is not an OpenAPI document (no 'paths')")
+    return data
+
+
+def _key(method: str, path: str) -> str:
+    return f"{method.upper()} {re.sub(r'{[^}]*}', '{}', path.rstrip('/') or '/')}"
+
+
+def _resolve(spec: dict[str, Any], node: Any, depth: int = 0) -> dict[str, Any]:
+    if not isinstance(node, dict) or depth > 10:
+        return {}
+    ref = node.get("$ref")
+    if isinstance(ref, str) and ref.startswith("#/"):
+        target: Any = spec
+        for part in ref[2:].split("/"):
+            target = target.get(part, {}) if isinstance(target, dict) else {}
+        return _resolve(spec, target, depth + 1)
+    return node
+
+
+def _spec_fields(spec: dict[str, Any], operation: dict[str, Any]) -> set[str] | None:
+    """Top-level fields of the operation's success reply, or None when not described."""
+    for code in ("200", "201", "2XX", "default"):
+        reply = _resolve(spec, (operation.get("responses") or {}).get(code))
+        content = reply.get("content") or {}
+        media = next((v for k, v in content.items() if "json" in k), None)
+        if media is None:
+            continue
+        schema = _resolve(spec, media.get("schema"))
+        if schema.get("type") == "array" or "items" in schema:
+            schema = _resolve(spec, schema.get("items"))
+        props = schema.get("properties")
+        if isinstance(props, dict):
+            return set(props)
+    return None
+
+
+def _seen_fields(ep: Endpoint) -> set[str]:
+    schema = ep.response_schema or {}
+    if schema.get("type") == "array" or "items" in schema:
+        schema = schema.get("items") or {}
+    props = schema.get("properties")
+    return set(props) if isinstance(props, dict) else set()
+
+
+def compare(spec: dict[str, Any], endpoints: list[Endpoint]) -> list[EndpointCheck]:
+    documented: dict[str, dict[str, Any]] = {}
+    for path, item in (spec.get("paths") or {}).items():
+        if not isinstance(item, dict):
+            continue
+        for method, operation in item.items():
+            if method in METHODS and isinstance(operation, dict):
+                documented[_key(method, str(path))] = operation
+    observed: dict[str, Endpoint] = {}
+    for ep in endpoints:
+        path, _ = split_template(ep.path_template)
+        observed.setdefault(_key(ep.method, path), ep)
+    checks = []
+    for key in sorted(observed.keys() | documented.keys()):
+        if key not in documented:
+            checks.append(EndpointCheck(key, "undocumented"))
+        elif key not in observed:
+            checks.append(EndpointCheck(key, "not seen"))
+        else:
+            check = EndpointCheck(key, "documented")
+            listed, seen = _spec_fields(spec, documented[key]), _seen_fields(observed[key])
+            if listed is not None and seen:
+                check.extra_fields = sorted(seen - listed)
+                check.missing_fields = sorted(listed - seen)
+            checks.append(check)
+    return checks
+
+
+def to_json(checks: list[EndpointCheck]) -> list[dict[str, Any]]:
+    return [asdict(c) for c in checks]

ghost_api/ui/keys.py

@@ -0,0 +1,134 @@
+"""Ctrl+C during a run inside the interactive session, read as a key, not a signal.
+
+On Windows a Ctrl+C *signal* also reaches cmd.exe, which runs the `ghost-api.cmd`
+launcher; when Ghost API later exits, cmd asks "Terminate batch job (Y/N)?". While a run
+is going (and only then: never while you type), `ctrl_c_as_key()` switches the console so
+Ctrl+C arrives as an ordinary key, read by a small thread. The first press asks the run
+to stop at the next safe point (`stop_requested()`, checked between steps); a second
+press interrupts at once; a third quits. The session itself keeps going.
+
+That console switch also turns off Windows' line editing (Backspace, Enter), so anything
+that reads a typed answer during a run must use `input_paused()`, which puts the normal
+mode back while the user types.
+
+On other systems, and when input isn't a console, these do nothing.
+"""
+
+from __future__ import annotations
+
+import _thread
+import contextlib
+import os
+import sys
+import threading
+import time
+from collections.abc import Callable, Iterator
+
+ENABLE_PROCESSED_INPUT = 0x0001
+STD_INPUT_HANDLE = -10
+
+
+class _Watcher:
+    def __init__(self, set_mode: Callable[[bool], None]) -> None:
+        self.stop = threading.Event()
+        self.pause = threading.Event()
+        self.presses = 0
+        self.set_mode = set_mode  # True: normal console mode (typing works)
+
+    def run(self) -> None:
+        if sys.platform != "win32":
+            return
+        import msvcrt
+
+        while not self.stop.is_set():
+            if self.pause.is_set() or not msvcrt.kbhit():
+                time.sleep(0.05)
+                continue
+            if self.pause.is_set():  # paused while waiting: leave the key to the reader
+                continue
+            if msvcrt.getwch() == "\x03":
+                self.presses += 1
+                pressed(self.presses)
+
+
+_active: _Watcher | None = None
+_stop = threading.Event()
+
+
+def _say(message: str) -> None:
+    pass
+
+
+# Set by the session: how to tell the user that a stop is on its way.
+on_stop_requested: Callable[[str], None] = _say
+
+
+def stop_requested() -> bool:
+    """Has the user asked the current run to stop? (Checked between steps.)"""
+    return _stop.is_set()
+
+
+def pressed(presses: int) -> None:
+    """1st Ctrl+C: stop at the next safe point (a browser that is starting or loading a
+    page must not be interrupted half-way: it can hang). 2nd: interrupt now. 3rd: quit."""
+    if presses == 1:
+        _stop.set()
+        on_stop_requested("Stopping after the current step... (Ctrl+C again to stop at once)")
+    elif presses == 2:
+        _thread.interrupt_main()
+    else:
+        os._exit(130)
+
+
+@contextlib.contextmanager
+def ctrl_c_as_key() -> Iterator[None]:
+    """Use around running something (not around reading input)."""
+    global _active
+    if sys.platform != "win32" or not sys.stdin.isatty() or _active is not None:
+        yield
+        return
+    import ctypes
+
+    kernel = ctypes.windll.kernel32
+    handle = kernel.GetStdHandle(STD_INPUT_HANDLE)
+    mode = ctypes.c_uint32()
+    if not kernel.GetConsoleMode(handle, ctypes.byref(mode)):
+        yield
+        return
+    normal = mode.value
+
+    def set_mode(typing: bool) -> None:
+        kernel.SetConsoleMode(handle, normal if typing else normal & ~ENABLE_PROCESSED_INPUT)
+
+    watcher = _Watcher(set_mode)
+    _stop.clear()
+    set_mode(False)
+    thread = threading.Thread(target=watcher.run, daemon=True)
+    _active = watcher
+    thread.start()
+    try:
+        yield
+    finally:
+        watcher.stop.set()
+        thread.join(timeout=1)
+        set_mode(True)
+        _active = None
+        _stop.clear()
+
+
+@contextlib.contextmanager
+def input_paused() -> Iterator[None]:
+    """Let the user type an answer during a run: the watcher stops reading keys and the
+    console's normal mode (Backspace, Enter, Ctrl+C) comes back until the answer is in."""
+    watcher = _active
+    if watcher is None:
+        yield
+        return
+    watcher.pause.set()
+    time.sleep(0.06)  # let the watcher finish any key it was reading
+    watcher.set_mode(True)
+    try:
+        yield
+    finally:
+        watcher.set_mode(False)
+        watcher.pause.clear()