ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/generator/__init__.py

@@ -0,0 +1,221 @@
+"""Generate a typed Python client from an endpoint map (Stage 6)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from ghost_api.analysis.models import Endpoint
+from ghost_api.capture.models import CapturedRequest
+from ghost_api.generator.naming import (
+    NAMING_SYSTEM,
+    NamingPlan,
+    apply_plan,
+    default_client_name,
+    naming_prompt,
+)
+from ghost_api.generator.python_gen import GeneratedClient, write
+from ghost_api.generator.spec import ClientSpec, build_spec, is_allowed
+from ghost_api.generator.ts_gen import write_typescript
+from ghost_api.llm.gateway import LLMError, LLMGateway
+from ghost_api.memory.store import Store
+
+if TYPE_CHECKING:
+    from ghost_api.drift import VersionInfo
+
+__all__ = [
+    "GenerationResult",
+    "NothingToGenerate",
+    "generate_client",
+    "generate_from",
+    "name_endpoints",
+    "plan_for",
+    "site_spec",
+]
+
+
+class NothingToGenerate(Exception):
+    pass
+
+
+@dataclass
+class GenerationResult:
+    client: GeneratedClient
+    spec: ClientSpec
+    plan: NamingPlan | None  # reuse it when regenerating, to save LLM calls
+    version: VersionInfo | None = None  # with a store: the client's version and changes
+
+    @property
+    def named_by_llm(self) -> bool:
+        return self.plan is not None
+
+
+def name_endpoints(
+    llm: LLMGateway | None, domain: str, endpoints: list[Endpoint]
+) -> NamingPlan | None:
+    """One LLM call for nicer names. None if there's no LLM or it fails."""
+    if llm is None or not endpoints:
+        return None
+    try:
+        reply = llm.ask(
+            [
+                {"role": "system", "content": NAMING_SYSTEM},
+                {"role": "user", "content": naming_prompt(domain, endpoints)},
+            ],
+            response_model=NamingPlan,
+        )
+    except LLMError:
+        return None  # deterministic names still work
+    return reply.parsed
+
+
+def plan_for(
+    store: Store, llm: LLMGateway | None, domain: str, endpoints: list[Endpoint]
+) -> NamingPlan | None:
+    """Reuse the stored naming plan if it covers every endpoint; otherwise ask the LLM once
+    and store the new plan (Stage 8: no LLM call when nothing changed).
+
+    Once a client version exists, names are part of its public API: existing names never
+    change (a rename would break code using the client); only new endpoints get names."""
+    stored = store.load_naming_plan(domain)
+    plan: NamingPlan | None = None
+    if stored:
+        try:
+            plan = NamingPlan.model_validate_json(stored)
+        except ValueError:
+            plan = None
+        if plan is not None and {e.key for e in endpoints} <= {n.key for n in plan.endpoints}:
+            return plan
+    if store.api_versions(domain):
+        return _name_new_endpoints(store, llm, domain, endpoints, plan)
+    plan = name_endpoints(llm, domain, endpoints)
+    if plan is not None:
+        store.save_naming_plan(domain, plan.model_dump_json())
+    return plan
+
+
+def _name_new_endpoints(
+    store: Store,
+    llm: LLMGateway | None,
+    domain: str,
+    endpoints: list[Endpoint],
+    plan: NamingPlan | None,
+) -> NamingPlan | None:
+    """A released client keeps every name it has; the LLM names only what is new."""
+    named = {n.key for n in plan.endpoints} if plan else set()
+    if plan is None:
+        # The released client used names derived from URLs: keep those for what exists.
+        previous = store.api_versions(domain)[0]["snapshot"]["methods"]
+        calls = {m["call"] for m in previous.values()}
+        named = {e.key for e in endpoints if f"{e.method} {e.path_template}" in calls}
+        named |= {e.key for e in endpoints if any(c.endswith(e.path_template) for c in calls)}
+    new = [e for e in endpoints if e.key not in named]
+    fresh = name_endpoints(llm, domain, new) if new else None
+    if fresh is None:
+        return plan
+    base = plan or NamingPlan(client_class=default_client_name(domain), endpoints=[])
+    merged = NamingPlan(
+        client_class=base.client_class,
+        endpoints=[*base.endpoints, *(n for n in fresh.endpoints if n.key not in named)],
+    )
+    store.save_naming_plan(domain, merged.model_dump_json())
+    return merged
+
+
+def generate_from(
+    domain: str,
+    start_url: str,
+    endpoints: list[Endpoint],
+    requests: list[CapturedRequest],
+    out_root: Path,
+    plan: NamingPlan | None = None,
+    lang: str = "python",
+    allow: set[str] | None = None,
+    store: Store | None = None,
+) -> GenerationResult:
+    """`store`: compare with the stored client versions, so the packages carry the right
+    version and changelog (nothing is recorded here; see drift.record)."""
+    class_name, names = apply_plan(plan, endpoints, domain)
+    spec = build_spec(domain, start_url, endpoints, requests, class_name, names, allow)
+    if not spec.methods:
+        reasons = "; ".join(spec.skipped) or "no endpoints"
+        raise NothingToGenerate(f"no endpoint can be turned into a method ({reasons})")
+    info = None
+    if store is not None:
+        from ghost_api import drift  # drift imports this package
+
+        info = drift.version_for(store, domain, spec)
+        spec.version, spec.changelog = info.version, drift.changelog(spec, info)
+    folder = out_root / domain
+    client = write_typescript(spec, folder) if lang == "typescript" else write(spec, folder)
+    _write_cassettes(spec, requests, folder, lang)
+    return GenerationResult(client, spec, plan, info)
+
+
+def _write_cassettes(
+    spec: ClientSpec, requests: list[CapturedRequest], folder: Path, lang: str
+) -> None:
+    """Recorded replies next to the tests, so they run offline (generator/cassettes.py)."""
+    from ghost_api.generator import cassettes
+
+    text = cassettes.dumps(cassettes.build(spec, requests))
+    targets = [folder / cassettes.FILE]
+    if lang == "python":
+        targets.append(folder / "package" / "python" / "tests" / cassettes.FILE)
+    for target in targets:
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(text, encoding="utf-8", newline="\n")
+
+
+def generate_client(
+    store: Store,
+    domain: str,
+    out_root: Path,
+    llm: LLMGateway | None = None,
+    lang: str = "python",
+    allow: set[str] | None = None,
+) -> GenerationResult:
+    """Build the client for `domain` from everything known about it (all runs)."""
+    run_id = store.latest_endpoint_run(domain)
+    run = store.latest_run(domain)
+    if run_id is None or run is None:
+        raise NothingToGenerate(f"no endpoints stored for {domain}; run `ghost-api run` first")
+    endpoints = store.current_endpoints(domain)
+    requests = store.requests_for_domain(domain)
+    plan = plan_for(store, llm, domain, endpoints)
+    # Remember what the user allowed, so a later plain `generate` keeps those methods.
+    if allow:
+        unknown = {a for a in allow if not any(is_allowed(e, {a}) for e in endpoints)}
+        if unknown:
+            names = ", ".join(sorted(unknown))
+            raise NothingToGenerate(f"--allow names no stored endpoint: {names}")
+        store.allow_endpoints(domain, [e.key for e in endpoints if is_allowed(e, allow)])
+    allowed = store.allowed_endpoints(domain)
+    result = generate_from(
+        domain, run.start_url, endpoints, requests, out_root, plan, lang, allowed, store
+    )
+    from ghost_api import drift
+
+    result.version = drift.record(store, domain, result.spec, run.id)
+    return result
+
+
+def site_spec(store: Store, domain: str) -> ClientSpec:
+    """The client `generate` would build from what is stored now, without writing files
+    or calling an LLM (the stored naming plan is reused)."""
+    run = store.latest_run(domain)
+    endpoints = store.current_endpoints(domain)
+    if run is None or not endpoints:
+        raise NothingToGenerate(f"no endpoints stored for {domain}; run `ghost-api run` first")
+    plan = None
+    stored = store.load_naming_plan(domain)
+    if stored:
+        try:
+            plan = NamingPlan.model_validate_json(stored)
+        except ValueError:
+            plan = None
+    class_name, names = apply_plan(plan, endpoints, domain)
+    requests = store.requests_for_domain(domain)
+    allowed = store.allowed_endpoints(domain)
+    return build_spec(domain, run.start_url, endpoints, requests, class_name, names, allowed)

ghost_api/generator/cassettes.py

@@ -0,0 +1,80 @@
+"""Recorded replies ("cassettes") so a generated client's tests run offline.
+
+The generated tests replay these by default: deterministic, fast, no traffic to the site,
+fine for CI. `GHOST_API_LIVE=1` runs the same tests against the real API (Ghost API's own
+verification always does).
+
+Replies come from what was stored while exploring, so cookies, tokens and keys are already
+redacted and personal data hidden; every entry is checked again before it is written.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+from urllib.parse import parse_qsl, urlsplit
+
+from ghost_api.analysis.analyzer import endpoint_keys
+from ghost_api.analysis.graphql import graphql_payload, operation_name
+from ghost_api.capture.filters import REDACTED, redact_text
+from ghost_api.capture.models import CapturedRequest
+from ghost_api.generator.spec import ClientSpec
+
+PER_ENDPOINT = 6  # the latest replies for each endpoint (pages, search terms...)
+FILE = "cassettes.json"
+
+
+def path_regex(path: str) -> str:
+    """'/api/items/{item_id}' -> '/api/items/[^/]+' (matched against the request path)."""
+    parts = re.split(r"\{[^}]*\}", path)
+    return "[^/]+".join(re.escape(p) for p in parts)
+
+
+def build(spec: ClientSpec, requests: list[CapturedRequest]) -> dict[str, Any]:
+    by_key: dict[str, list[CapturedRequest]] = {}
+    for key, req in zip(endpoint_keys(requests), requests, strict=False):
+        by_key.setdefault(key, []).append(req)
+    replies: list[dict[str, Any]] = []
+    for method in spec.methods:
+        usable = [r for r in by_key.get(method.endpoint_key, []) if _usable(r)]
+        for req in usable[-PER_ENDPOINT:]:
+            entry = _entry(method.http_method, method.path, req)
+            if entry is not None:
+                replies.append(entry)
+    return {
+        "about": "Replies recorded by Ghost API, replayed by the generated tests. "
+        "Set GHOST_API_LIVE=1 to test against the real API instead.",
+        "site": spec.domain,
+        "replies": replies,
+    }
+
+
+def _usable(req: CapturedRequest) -> bool:
+    return 200 <= req.status < 300 and bool(req.resp_body) and not req.truncated
+
+
+def _entry(http_method: str, path: str, req: CapturedRequest) -> dict[str, Any] | None:
+    url = urlsplit(req.url)
+    query = {
+        k: v for k, v in parse_qsl(url.query, keep_blank_values=True) if REDACTED not in v
+    }  # a redacted key is not something a request can match on
+    payload = graphql_payload(req.method, req.url, req.req_body)
+    entry: dict[str, Any] = {
+        "method": http_method,
+        "path": url.path,
+        "path_regex": path_regex(path),
+        "query": query,
+        "operation": operation_name(payload) if payload else None,
+        "status": req.status,
+        "content_type": req.resp_content_type or "application/json",
+        "body": req.resp_body,
+    }
+    text = json.dumps(entry, ensure_ascii=False)
+    if redact_text(text) != text:
+        return None  # never write anything secret-looking, even into test data
+    return entry
+
+
+def dumps(cassettes: dict[str, Any]) -> str:
+    return json.dumps(cassettes, indent=1, ensure_ascii=False) + "\n"

ghost_api/generator/models_gen.py

@@ -0,0 +1,144 @@
+"""Turn inferred JSON schemas into pydantic model definitions (FR-28).
+
+- Nested objects become their own classes, named after the field (quotes[] -> Quote).
+- Objects with the same structure share one class.
+- Fields missing from some samples, or seen as null, become `T | None = None`.
+- Field names that aren't valid Python get a snake_case name plus an alias.
+- Generated models allow extra fields, so a new field in the API doesn't break them.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from pydantic import BaseModel
+
+from ghost_api.analysis.url_grouping import singular
+from ghost_api.generator.naming import pascal, snake, unique
+
+PRIMITIVES = {
+    "string": "str",
+    "integer": "int",
+    "number": "float",
+    "boolean": "bool",
+    "null": "None",
+}
+# Names that would shadow pydantic BaseModel attributes (json, copy, schema, model_*...).
+_RESERVED = {n for n in dir(BaseModel) if not n.startswith("_")}
+
+
+@dataclass
+class FieldDef:
+    name: str  # Python attribute name
+    alias: str | None  # original JSON key, when different
+    type: str  # type expression, e.g. "list[Quote] | None"
+    optional: bool
+
+
+@dataclass
+class ModelDef:
+    name: str
+    fields: list[FieldDef] = field(default_factory=list)
+    doc: str = ""
+
+
+class ModelBuilder:
+    def __init__(self, reserved_names: set[str] | None = None) -> None:
+        self.models: list[ModelDef] = []
+        self._by_signature: dict[tuple[Any, ...], str] = {}
+        self._used: set[str] = set(reserved_names or ())
+        # Treat "seen in every sample" as "always present" only with enough evidence:
+        # many items from one reply (one area, one search) say little about the next one.
+        self.trust_required = True
+
+    def root_type(self, schema: dict[str, Any] | None, name: str, doc: str = "") -> str:
+        """Type expression for a whole response; creates models as needed."""
+        if not schema:
+            return "Any"
+        if _is_object(schema):
+            return self._model(schema, name, doc, exact_name=True)
+        return self.type_for(schema, name)
+
+    def type_for(self, schema: dict[str, Any] | None, hint: str) -> str:
+        if not schema:
+            return "Any"
+        if "anyOf" in schema:
+            return _union(self.type_for(s, hint) for s in schema["anyOf"])
+        kinds = schema.get("type")
+        if isinstance(kinds, list):
+            parts = []
+            for k in kinds:
+                sub = {**schema, "type": k}
+                parts.append(self.type_for(sub, hint))
+            return _union(parts)
+        if kinds == "object":
+            if schema.get("properties"):
+                return self._model(schema, hint)
+            return "dict[str, Any]"
+        if kinds == "array":
+            items = schema.get("items")
+            item_hint = pascal(singular(snake(hint))) if hint else "Item"
+            return f"list[{self.type_for(items, item_hint) if items else 'Any'}]"
+        return PRIMITIVES.get(str(kinds), "Any")
+
+    def _model(
+        self, schema: dict[str, Any], hint: str, doc: str = "", exact_name: bool = False
+    ) -> str:
+        props: dict[str, Any] = schema.get("properties") or {}
+        required = set(schema.get("required") or [])
+        fields: list[FieldDef] = []
+        used_fields: set[str] = set()
+        for key, sub in props.items():
+            type_expr = self.type_for(sub, pascal(key))
+            if type_expr == "None":  # only ever seen as null: real type unknown
+                type_expr = "Any"
+            nullable = "None" in _top_level_members(type_expr) or type_expr == "Any"
+            optional = key not in required or nullable or not self.trust_required
+            if optional and not nullable:
+                type_expr = _union([type_expr, "None"])
+            py = snake(key)
+            if py in _RESERVED or py.startswith("model_"):
+                py += "_"
+            py = unique(py, used_fields)
+            fields.append(FieldDef(py, None if py == key else key, type_expr, optional))
+
+        signature = tuple((f.alias or f.name, f.type, f.optional) for f in fields)
+        if signature in self._by_signature and not exact_name:
+            return self._by_signature[signature]
+        name = unique(pascal(hint) or "Model", self._used)
+        self.models.append(ModelDef(name, fields, doc))
+        self._by_signature.setdefault(signature, name)
+        return name
+
+
+def _is_object(schema: dict[str, Any]) -> bool:
+    return schema.get("type") == "object" and bool(schema.get("properties"))
+
+
+def _top_level_members(type_expr: str) -> list[str]:
+    """'int | list[int | None] | None' -> ['int', 'list[int | None]', 'None']: split
+    only outside brackets, or nested unions fall apart (seen on Google Maps data)."""
+    members, depth, start = [], 0, 0
+    for i, ch in enumerate(type_expr):
+        if ch == "[":
+            depth += 1
+        elif ch == "]":
+            depth -= 1
+        elif depth == 0 and type_expr.startswith(" | ", i):
+            members.append(type_expr[start:i])
+            start = i + 3
+    members.append(type_expr[start:])
+    return members
+
+
+def _union(parts: Any) -> str:
+    seen: list[str] = []
+    for part in parts:
+        for p in _top_level_members(str(part)):
+            if p not in seen:
+                seen.append(p)
+    if "Any" in seen:
+        return "Any"
+    seen.sort(key=lambda p: p == "None")  # None last
+    return " | ".join(seen) or "Any"

ghost_api/generator/naming.py

@@ -0,0 +1,174 @@
+"""Names for generated code: safe Python identifiers, with or without the LLM.
+
+Deterministic names always work (get_quotes, QuotesResponse). One optional LLM call
+(FR-22) can replace them with better ones (search_items, SearchResults); every name it
+returns is validated, and anything unusable falls back to the deterministic name.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import keyword
+import re
+
+from pydantic import BaseModel, Field
+
+from ghost_api.analysis.graphql import split_template
+from ghost_api.analysis.models import Endpoint
+from ghost_api.analysis.schema import field_paths
+from ghost_api.analysis.url_grouping import singular
+
+_VERSION_RE = re.compile(r"^(v\d+(\.\d+)*|\d{1,2})$", re.I)
+_NOISE_SEGMENTS = {"api", "rest", "json", "public", "ajax", "www"}
+
+
+def snake(name: str) -> str:
+    """'nbHits' -> 'nb_hits', 'x-algolia-agent' -> 'x_algolia_agent', '2fa' -> 'n_2fa'."""
+    name = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", name)
+    name = re.sub(r"[^0-9a-zA-Z]+", "_", name).strip("_").lower() or "value"
+    if name[0].isdigit():
+        name = f"n_{name}"
+    if keyword.iskeyword(name) or name in ("self", "cls"):
+        name += "_"
+    return name
+
+
+def pascal(name: str) -> str:
+    """'quotes response' / 'quotes_response' -> 'QuotesResponse'."""
+    parts = re.split(r"[^0-9a-zA-Z]+", re.sub(r"(?<=[a-z0-9])(?=[A-Z])", " ", name))
+    out = "".join(p[:1].upper() + p[1:] for p in parts if p) or "Model"
+    return out if not out[0].isdigit() else f"M{out}"
+
+
+def is_identifier(name: str) -> bool:
+    return name.isidentifier() and not keyword.iskeyword(name)
+
+
+MAX_NAME_WORDS = 3
+MAX_WORD_CHARS = 30
+
+
+def path_words(endpoint: Endpoint) -> list[str]:
+    """Meaningful literal segments: /api/v1/books/{id}/reviews -> ['books', 'reviews']."""
+    words = []
+    for seg in endpoint.path_template.strip("/").split("/"):
+        if not seg or seg.startswith("{") or seg.lower() in _NOISE_SEGMENTS:
+            continue
+        if _VERSION_RE.match(seg):
+            continue
+        words.append(snake(seg)[:MAX_WORD_CHARS].rstrip("_"))
+    # Keep names readable: the last few words say what the endpoint is.
+    return words[-MAX_NAME_WORDS:]
+
+
+def default_method_name(endpoint: Endpoint) -> str:
+    _, operation = split_template(endpoint.path_template)
+    if operation:  # GraphQL: the operation's own name (GetUser -> get_user)
+        return snake(operation)
+    words = path_words(endpoint) or [snake(endpoint.host.split(".")[0])]
+    verb = {"GET": "get", "HEAD": "head", "POST": "post"}.get(
+        endpoint.method, endpoint.method.lower()
+    )
+    if endpoint.path_template.rstrip("/").endswith("}") and words:
+        words[-1] = singular(words[-1])  # /books/{id} -> get_book
+    return snake(f"{verb}_{'_'.join(words)}")
+
+
+def default_model_name(endpoint: Endpoint) -> str:
+    _, operation = split_template(endpoint.path_template)
+    if operation:
+        return f"{pascal(operation)}Response"
+    words = path_words(endpoint) or [endpoint.host.split(".")[0]]
+    base = pascal("_".join(words))
+    if endpoint.path_template.rstrip("/").endswith("}"):
+        base = pascal(singular(words[-1]))
+    return f"{base}Response"
+
+
+def _is_ip(host: str) -> bool:
+    try:
+        ipaddress.ip_address(host)
+    except ValueError:
+        return False
+    return True
+
+
+def default_client_name(domain: str) -> str:
+    """'quotes.toscrape.com' -> 'QuotesToscrapeClient'; 'hn.algolia.com' -> 'HnAlgoliaClient';
+    a site with a section: 'www.google.com/maps' -> 'GoogleMapsClient'."""
+    host, _, section = domain.partition("/")
+    host = host.lower().split(":")[0].strip("[]")
+    if host == "localhost" or _is_ip(host):
+        base = "local"  # '127.0.0.1' would otherwise give 'M1270Client'
+    else:
+        labels = [p for p in host.split(".") if p not in ("www", "com", "org", "net", "io")]
+        base = "_".join(labels[:2]) or "api"
+    return pascal(f"{base}_{section}" if section else base) + "Client"
+
+
+def unique(name: str, used: set[str]) -> str:
+    out, n = name, 2
+    while out in used:
+        out, n = f"{name}{n}", n + 1
+    used.add(out)
+    return out
+
+
+# --- optional LLM naming pass ----------------------------------------------------------------
+
+
+class EndpointNames(BaseModel):
+    key: str = Field(description="The endpoint key exactly as given.")
+    method_name: str = Field(description="snake_case Python method name, e.g. search_stories.")
+    response_model: str = Field(description="PascalCase model name, e.g. SearchResults.")
+    summary: str = Field(description="One short sentence: what this endpoint returns.")
+
+
+class NamingPlan(BaseModel):
+    client_class: str = Field(description="PascalCase client class name ending in Client.")
+    endpoints: list[EndpointNames]
+
+
+NAMING_SYSTEM = """\
+You name the parts of a generated Python API client. Names must be short, clear and
+describe what the endpoint does for a user (search_stories, get_quote, list_tags), not
+how the URL looks. Use snake_case for methods and PascalCase for models."""
+
+
+def naming_prompt(domain: str, endpoints: list[Endpoint]) -> str:
+    lines = [f"Website: {domain}", "", "Endpoints:"]
+    for ep in endpoints:
+        params = [p.name for p in ep.path_params] + [p.name for p in ep.query_params]
+        fields = ", ".join(field_paths(ep.response_schema, limit=12)) or "-"
+        lines.append(f"- key: {ep.key}\n  params: {', '.join(params) or '-'}\n  returns: {fields}")
+    return "\n".join(lines)
+
+
+def apply_plan(
+    plan: NamingPlan | None, endpoints: list[Endpoint], domain: str
+) -> tuple[str, dict[str, EndpointNames]]:
+    """Validated names for the client and every endpoint, falling back where needed."""
+    client = default_client_name(domain)
+    if plan and is_identifier(plan.client_class) and plan.client_class[:1].isupper():
+        client = (
+            plan.client_class
+            if plan.client_class.endswith("Client")
+            else f"{plan.client_class}Client"
+        )
+    suggested = {e.key: e for e in plan.endpoints} if plan else {}
+    used_methods: set[str] = set()
+    used_models: set[str] = {client}
+    names: dict[str, EndpointNames] = {}
+    for ep in endpoints:
+        s = suggested.get(ep.key)
+        method = snake(s.method_name) if s and s.method_name else default_method_name(ep)
+        model = s.response_model if s and is_identifier(s.response_model) else ""
+        model = pascal(model) if model else default_model_name(ep)
+        summary = (s.summary.strip() if s else "") or f"{ep.method} {ep.path_template}"
+        names[ep.key] = EndpointNames(
+            key=ep.key,
+            method_name=unique(method, used_methods),
+            response_model=unique(model, used_models),
+            summary=summary.replace('"""', "'''")[:200],
+        )
+    return client, names