pyhttp-app 2.0.1__py3-none-any.whl

pyhttp/mock.py

@@ -0,0 +1,204 @@
+"""Mock servers served from pyhttp's own port.
+
+A mock server is a name, a slug and a list of routes. Everything under
+`/mock/<slug>/…` on the port pyhttp already listens on is answered from those
+routes, so there is no second process and no second port to remember:
+
+    http://127.0.0.1:8765/mock/payments/users/42
+
+A route path may hold parameters written either way (`/users/:id` or
+`/users/{id}`); `*` matches one segment and `**` matches the rest of the path
+(as `{{req.params.rest}}`).
+The response body is a `{{ ... }}` template, so besides variables and the
+dynamic values (`{{$guid}}`, `{{$randomEmail}}`…) it can read the incoming
+request:
+
+    {{req.params.id}}       a path parameter (from `:id` / `{id}` in the route path)
+    {{req.query.limit}}     a query-string value
+    {{req.headers.x-key}}   a request header (case-insensitive)
+    {{req.body}}            the raw request body
+    {{req.json.user.name}}  a path into the JSON request body
+    {{req.method}} {{req.path}}
+"""
+from __future__ import annotations
+
+import json
+import re
+import time
+from typing import Any
+from urllib.parse import parse_qsl
+
+from . import db
+from .templating import build_context, render
+
+METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS")
+MAX_LOGGED_BODY = 4000   # a logged call keeps at most this much of the request body
+_PARAM = re.compile(r":([A-Za-z_][A-Za-z0-9_]*)|\{([A-Za-z_][A-Za-z0-9_]*)\}")
+
+
+def normalize_path(path: str) -> str:
+    """`users/:id/` -> `/users/:id` so routes and incoming paths compare equal."""
+    path = (path or "").strip()
+    path = path.split("?", 1)[0].split("#", 1)[0]
+    if not path.startswith("/"):
+        path = "/" + path
+    return path.rstrip("/") or "/"
+
+
+def compile_path(path: str) -> tuple[re.Pattern[str], int]:
+    """Regex for a route path, plus how many wildcard segments it has (lower = more specific)."""
+    parts: list[str] = []
+    wild = 0
+    for seg in normalize_path(path).strip("/").split("/"):
+        if not seg:
+            continue
+        if seg == "**":
+            parts.append("(?P<rest>.*)")
+            wild += 100
+            break
+        if seg == "*":
+            parts.append("[^/]+")
+            wild += 1
+            continue
+        m = _PARAM.fullmatch(seg)
+        if m:
+            parts.append(f"(?P<{m.group(1) or m.group(2)}>[^/]+)")
+            wild += 1
+        else:
+            parts.append(re.escape(seg))
+    pattern = "/" + "/".join(parts) if parts else "/"
+    return re.compile(f"^{pattern}/?$"), wild
+
+
+def _query_conditions(match_query: str) -> list[tuple[str, str]]:
+    """`status=error&kind=card` -> the pairs an incoming query must contain."""
+    return [(k, v) for k, v in parse_qsl(match_query.lstrip("?"), keep_blank_values=True)]
+
+
+def match_route(routes: list[dict[str, Any]], method: str, path: str,
+                query: dict[str, str]) -> tuple[dict[str, Any] | None, dict[str, str]]:
+    """The best route for a request: exact paths beat parameters, and a route with
+    query conditions beats a plain one. Ties go to the earlier route."""
+    method, path = method.upper(), normalize_path(path)
+    best: tuple[tuple[int, int, int], dict[str, Any], dict[str, str]] | None = None
+    for pos, route in enumerate(routes):
+        if not route.get("enabled", True):
+            continue
+        route_method = str(route.get("method") or "GET").upper()
+        if route_method not in ("ANY", "*") and route_method != method:
+            continue
+        rx, wild = compile_path(route.get("path") or "/")
+        m = rx.match(path)
+        if not m:
+            continue
+        conditions = _query_conditions(route.get("match_query") or "")
+        if any(query.get(k) != v for k, v in conditions):
+            continue
+        rank = (wild, -len(conditions), pos)   # smaller wins
+        if best is None or rank < best[0]:
+            best = (rank, route, {k: v for k, v in m.groupdict().items() if v is not None})
+    if best is None and method == "HEAD":
+        return match_route(routes, "GET", path, query)   # HEAD is a GET without the body
+    return (best[1], best[2]) if best else (None, {})
+
+
+def request_context(method: str, path: str, query: dict[str, str], headers: dict[str, str],
+                    body: str, params: dict[str, str]) -> dict[str, Any]:
+    """Template context for a mock response: variables plus a `req` object."""
+    parsed: Any = {}
+    if body:
+        try:
+            parsed = json.loads(body)
+        except ValueError:
+            parsed = {}
+    variables, _env = db.effective_variables()
+    ctx = build_context(variables)
+    ctx["req"] = {"method": method.upper(), "path": normalize_path(path), "params": params,
+                  "query": dict(query), "headers": dict(headers), "body": body, "json": parsed}
+    return ctx
+
+
+def cors_headers(origin: str = "*", request_headers: str = "") -> dict[str, str]:
+    return {
+        "Access-Control-Allow-Origin": origin or "*",
+        "Access-Control-Allow-Methods": ", ".join(METHODS),
+        "Access-Control-Allow-Headers": request_headers or "*",
+        "Access-Control-Max-Age": "600",
+    }
+
+
+def build_response(server: dict[str, Any], route: dict[str, Any], ctx: dict[str, Any]) -> dict[str, Any]:
+    """Status, headers and rendered body for a matched route."""
+    body = render(route.get("body") or "", ctx)
+    headers = {"Content-Type": route.get("content_type") or "application/json"}
+    for row in route.get("headers") or []:
+        if row.get("enabled", True) and str(row.get("key") or "").strip():
+            headers[str(row["key"]).strip()] = render(str(row.get("value") or ""), ctx)
+    if server.get("cors", True):
+        headers.update(cors_headers())
+    return {"status_code": int(route.get("status_code") or 200), "headers": headers, "body": body,
+            "delay_s": (float(server.get("delay_ms") or 0) + float(route.get("delay_ms") or 0)) / 1000.0}
+
+
+def not_found_body(server: dict[str, Any], method: str, path: str) -> dict[str, Any]:
+    """The answer when nothing matches: a 404 that says what the server does know."""
+    routes = server.get("routes") or []
+    known = [f"{r.get('method', 'GET')} {normalize_path(r.get('path') or '/')}" for r in routes
+             if r.get("enabled", True)]
+    detail = (f"No mock route in \"{server['name']}\" matches {method.upper()} {normalize_path(path)}")
+    body = json.dumps({"error": "no_mock_route", "detail": detail, "mock": server["name"],
+                       "routes": known}, indent=2)
+    headers = {"Content-Type": "application/json"}
+    if server.get("cors", True):
+        headers.update(cors_headers())
+    return {"status_code": 404, "headers": headers, "body": body, "delay_s": 0.0}
+
+
+def log_call(server: dict[str, Any], route: dict[str, Any] | None, method: str, path: str,
+             query_string: str, headers: dict[str, str], body: str, status_code: int,
+             duration_ms: float) -> None:
+    if not server.get("log_calls", True):
+        return
+    db.add_mock_call(server["id"], {
+        "route_id": route["id"] if route else None,
+        "route_name": (route.get("name") or f"{route.get('method')} {route.get('path')}") if route else "",
+        "ts": time.time(), "method": method.upper(), "path": normalize_path(path),
+        "query": query_string, "status_code": status_code, "matched": bool(route),
+        "duration_ms": duration_ms, "req_headers": headers, "req_body": body[:MAX_LOGGED_BODY],
+    })
+
+
+# --------------------------------------------------------- seeding from requests
+def _path_of(url: str) -> str:
+    """The path part of a saved request's URL, variables and all: `{{base}}/users/:id` -> `/users/:id`."""
+    url = (url or "").strip()
+    url = re.sub(r"^\s*\{\{[^}]*\}\}", "", url)            # a leading {{base_url}}
+    url = re.sub(r"^[a-zA-Z][\w+.-]*://[^/]+", "", url)    # or a real scheme + host
+    return normalize_path(url.split("?", 1)[0] or "/")
+
+
+def routes_from_requests(request_ids: list[str]) -> list[dict[str, Any]]:
+    """Draft routes for saved requests, answering with the response each last gave
+    (from the request history) or an empty JSON object."""
+    history = db.list_history(limit=db.MAX_HISTORY)
+    last: dict[str, dict[str, Any]] = {}
+    for entry in history:                                  # newest first
+        rid = entry.get("request_id")
+        if rid and rid not in last and entry.get("status_code"):
+            last[rid] = entry
+    drafts = []
+    for rid in request_ids:
+        req = db.get_request(rid)
+        if not req:
+            continue
+        seen = last.get(rid)
+        full = db.get_history(seen["id"]) if seen else None
+        resp = ((full or {}).get("detail") or {}).get("response") or {}
+        body = resp.get("body_text") or ""
+        status = int(resp.get("status_code") or 200)
+        content_type = next((v for k, v in (resp.get("headers") or []) if str(k).lower() == "content-type"),
+                            "application/json")
+        drafts.append({"name": req["name"], "method": req.get("method") or "GET", "path": _path_of(req.get("url")),
+                       "status_code": status, "content_type": content_type.split(";")[0].strip(),
+                       "body": body or "{}"})
+    return drafts

pyhttp/monitors.py

@@ -0,0 +1,116 @@
+"""Scheduled monitors: picked requests sent again on an interval, in the background of the
+running app. Each check stores a compact result row (no run folders, no history entries, so
+a monitor every minute does not flood the disk); failures can ping a webhook.
+
+A check fails when any request errors, answers 4xx/5xx, or fails one of its tests —
+the same rule the collection runner uses.
+"""
+from __future__ import annotations
+
+import asyncio
+import time
+from typing import Any
+
+import httpx
+
+from . import USER_AGENT, db, runner
+from .templating import build_context
+
+TICK_SECONDS = 5.0  # how often the scheduler looks for due monitors
+
+
+async def run_check(monitor: dict[str, Any]) -> dict[str, Any]:
+    """Send the monitor's requests in order and store one check row. Returns the check."""
+    started = time.time()
+    results: list[dict[str, Any]] = []
+    failed = 0
+    prev: dict[str, Any] | None = None
+    for rid in monitor.get("request_ids") or []:
+        req = db.get_request(rid)
+        if not req:
+            results.append({"request_id": rid, "name": "(deleted request)", "ok": False,
+                            "error": "the request no longer exists"})
+            failed += 1
+            continue
+        for item in runner.expand(req):
+            variables, _env = db.effective_variables(item.get("folder_id"))
+            ctx = build_context(variables, prev)
+            result = await runner.execute(item, ctx)
+            prev = runner.prev_context(result)
+            ok = result["status"] != "error" and (result.get("status_code") or 0) < 400 \
+                and not result.get("tests_failed")
+            if not ok:
+                failed += 1
+            results.append({
+                "request_id": rid, "name": runner.display_name(item), "method": item.get("method", ""),
+                "ok": ok, "status_code": result.get("status_code"),
+                "duration_ms": result.get("duration_ms"), "error": result.get("error"),
+                "tests_failed": result.get("tests_failed", 0),
+            })
+    check = {"ts": started, "ok": failed == 0, "failed": failed, "total": len(results),
+             "duration_ms": round((time.time() - started) * 1000, 1), "results": results}
+    stored = db.add_monitor_check(monitor["id"], check)
+    db.update_monitor(monitor["id"], last_run_at=started, last_status="fail" if failed else "pass")
+    if failed and (monitor.get("webhook_url") or "").strip():
+        await _webhook(monitor, check)
+    return stored
+
+
+async def _webhook(monitor: dict[str, Any], check: dict[str, Any]) -> None:
+    """POST the failure to the monitor's webhook. Errors are swallowed: an unreachable
+    webhook must never break the schedule."""
+    payload = {
+        "monitor": monitor["name"], "monitor_id": monitor["id"], "status": "fail",
+        "failed": check["failed"], "total": check["total"], "ts": check["ts"],
+        "results": [r for r in check["results"] if not r["ok"]],
+    }
+    try:
+        async with httpx.AsyncClient(timeout=10, headers={"User-Agent": USER_AGENT}) as client:
+            await client.post(monitor["webhook_url"].strip(), json=payload)
+    except Exception:  # noqa: BLE001
+        pass
+
+
+class MonitorScheduler:
+    """Runs due monitors from the app's event loop. One check per monitor at a time."""
+
+    def __init__(self) -> None:
+        self._task: asyncio.Task[None] | None = None
+        self._running: set[str] = set()
+
+    def start(self) -> None:
+        if self._task is None or self._task.done():
+            self._task = asyncio.create_task(self._loop())
+
+    def stop(self) -> None:
+        if self._task is not None:
+            self._task.cancel()
+            self._task = None
+
+    async def _loop(self) -> None:
+        while True:
+            try:
+                self.tick()
+            except Exception:  # noqa: BLE001 - the schedule must survive anything
+                pass
+            await asyncio.sleep(TICK_SECONDS)
+
+    def tick(self, now: float | None = None) -> list[asyncio.Task[Any]]:
+        """Start a check task for every due monitor; called by the loop and by tests."""
+        tasks = []
+        for monitor in db.due_monitors(now if now is not None else time.time()):
+            if monitor["id"] in self._running:
+                continue
+            self._running.add(monitor["id"])
+            # claim at once, so the monitor is not due again while its check runs
+            db.update_monitor(monitor["id"], last_run_at=time.time())
+            tasks.append(asyncio.create_task(self._run(monitor)))
+        return tasks
+
+    async def _run(self, monitor: dict[str, Any]) -> None:
+        try:
+            await run_check(monitor)
+        except Exception:  # noqa: BLE001
+            db.update_monitor(monitor["id"], last_status="fail")
+        finally:
+            self._running.discard(monitor["id"])

pyhttp/oauth2.py

@@ -0,0 +1,153 @@
+"""OAuth 2.0 tokens for the "OAuth 2.0" auth type.
+
+Grants that need no browser: client credentials and password (resource owner).
+Tokens are cached in memory per (token URL, client, user, scope) and fetched
+again when they expire; a refresh token is used first when the server gave one.
+"""
+from __future__ import annotations
+
+import asyncio
+import base64
+import threading
+import time
+from typing import Any
+
+import httpx
+
+GRANTS = ("client_credentials", "password")
+_EARLY = 30  # seconds: treat a token as expired this long before it really is
+
+_cache: dict[tuple[str, ...], dict[str, Any]] = {}
+_lock = threading.Lock()
+_fetch_locks: dict[tuple[str, ...], asyncio.Lock] = {}
+
+
+def _key(auth: dict[str, Any]) -> tuple[str, ...]:
+    return (auth.get("grant") or "client_credentials", auth.get("token_url") or "", auth.get("client_id") or "",
+            auth.get("username") or "", auth.get("scope") or "", auth.get("audience") or "")
+
+
+def cached(auth: dict[str, Any]) -> dict[str, Any] | None:
+    """The cached token for these (rendered) settings, if it is still valid."""
+    with _lock:
+        tok = _cache.get(_key(auth))
+    if tok and (not tok.get("expires_at") or tok["expires_at"] - _EARLY > time.time()):
+        return tok
+    return None
+
+
+def status(auth: dict[str, Any]) -> dict[str, Any]:
+    """Token info for the UI (the token itself is shortened)."""
+    with _lock:
+        tok = _cache.get(_key(auth))
+    if not tok:
+        return {"has_token": False}
+    exp = tok.get("expires_at")
+    return {
+        "has_token": True, "valid": not exp or exp - _EARLY > time.time(), "expires_at": exp,
+        "token_type": tok.get("token_type") or "Bearer", "scope": tok.get("scope") or "",
+        "preview": _preview(tok["access_token"]), "has_refresh": bool(tok.get("refresh_token")),
+        "fetched_at": tok.get("fetched_at"),
+    }
+
+
+def _preview(token: str) -> str:
+    return token if len(token) <= 16 else f"{token[:8]}…{token[-6:]}"
+
+
+def forget(auth: dict[str, Any] | None = None) -> None:
+    with _lock:
+        if auth is None:
+            _cache.clear()
+        else:
+            _cache.pop(_key(auth), None)
+
+
+def _store(auth: dict[str, Any], data: dict[str, Any], old_refresh: str | None = None) -> dict[str, Any]:
+    if not isinstance(data, dict) or not data.get("access_token"):
+        raise ValueError("token response has no access_token")
+    try:
+        expires_in = float(data.get("expires_in")) if data.get("expires_in") not in (None, "") else None
+    except (TypeError, ValueError):
+        expires_in = None
+    now = time.time()
+    tok = {
+        "access_token": str(data["access_token"]),
+        "token_type": str(data.get("token_type") or "Bearer"),
+        "refresh_token": data.get("refresh_token") or old_refresh,
+        "scope": str(data.get("scope") or auth.get("scope") or ""),
+        "expires_at": now + expires_in if expires_in else None,
+        "fetched_at": now,
+    }
+    with _lock:
+        _cache[_key(auth)] = tok
+    return tok
+
+
+def _token_request(auth: dict[str, Any], refresh_token: str | None) -> tuple[dict[str, str], dict[str, str]]:
+    """Form fields and headers of the token request."""
+    if refresh_token:
+        form = {"grant_type": "refresh_token", "refresh_token": refresh_token}
+    elif auth.get("grant") == "password":
+        form = {"grant_type": "password", "username": auth.get("username") or "", "password": auth.get("password") or ""}
+    else:
+        form = {"grant_type": "client_credentials"}
+    if auth.get("scope"):
+        form["scope"] = auth["scope"]
+    if auth.get("audience"):
+        form["audience"] = auth["audience"]
+    headers = {"Accept": "application/json"}
+    cid, secret = auth.get("client_id") or "", auth.get("client_secret") or ""
+    if auth.get("client_auth") == "body":
+        if cid:
+            form["client_id"] = cid
+        if secret:
+            form["client_secret"] = secret
+    elif cid or secret:
+        headers["Authorization"] = "Basic " + base64.b64encode(f"{cid}:{secret}".encode()).decode()
+    return form, headers
+
+
+async def fetch(auth: dict[str, Any], client_kwargs: dict[str, Any] | None = None, *, force: bool = False) -> dict[str, Any]:
+    """A valid token: cached, refreshed, or newly requested. Raises ValueError with a readable message."""
+    if not force:
+        tok = cached(auth)
+        if tok:
+            return tok
+    if not (auth.get("token_url") or "").strip():
+        raise ValueError("OAuth 2.0: no token URL")
+    key = _key(auth)
+    lock = _fetch_locks.setdefault(key, asyncio.Lock())
+    async with lock:
+        if not force:
+            tok = cached(auth)  # another request may have fetched it while we waited
+            if tok:
+                return tok
+        with _lock:
+            old = _cache.get(key)
+        refresh = old.get("refresh_token") if old else None
+        kwargs = {k: v for k, v in (client_kwargs or {}).items() if k in ("verify", "proxy", "timeout")}
+        kwargs.setdefault("timeout", 30)
+        async with httpx.AsyncClient(**kwargs) as client:
+            if refresh:
+                form, headers = _token_request(auth, refresh)
+                resp = await client.post(auth["token_url"], data=form, headers=headers)
+                if resp.status_code < 400:
+                    try:
+                        return _store(auth, resp.json(), refresh)
+                    except ValueError:
+                        pass
+            form, headers = _token_request(auth, None)
+            resp = await client.post(auth["token_url"], data=form, headers=headers)
+        if resp.status_code >= 400:
+            detail = resp.text.strip()[:300]
+            raise ValueError(f"OAuth 2.0 token request failed: HTTP {resp.status_code}" + (f" — {detail}" if detail else ""))
+        try:
+            data = resp.json()
+        except ValueError:
+            # some servers answer with a form-encoded body
+            data = dict(httpx.QueryParams(resp.text))
+        try:
+            return _store(auth, data)
+        except ValueError as exc:
+            raise ValueError(f"OAuth 2.0: {exc}") from None