layover 0.1.0.dev0__py3-none-any.whl

layover/oracles/record.py

@@ -0,0 +1,98 @@
+"""Record a live response as a test fixture (ORACLES §3).
+
+Parsers are the part of this system that breaks, and the only defence that works is a
+recorded response plus a golden parse: CI then re-runs the parser against real bytes
+without touching the network. This is the tool that captures those bytes.
+
+Sanitising is deliberate rather than thorough-looking. The page's *data* is exactly
+what the parser needs, so it stays; what goes is anything that identifies the machine
+or the session that fetched it. The repo is private (DESIGN §13), which is what makes
+committing a site's markup acceptable at all — that is a decision, not an oversight.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from layover.oracles.base import OracleQuery
+from layover.oracles.transport import HttpTransport, is_eu_egress
+
+# Session-ish identifiers that can appear in an embedded state blob. Values are
+# replaced, keys kept, so the shape the parser sees is unchanged.
+_SCRUB_PATTERNS: tuple[tuple[str, re.Pattern[str], str], ...] = (
+    ("email", re.compile(r"[\w.+-]+@[\w-]+\.[\w.]+"), "someone@example.invalid"),
+    (
+        "sapisid",
+        re.compile(r'"(SAPISID|APISID|SID|HSID|SSID)"\s*:\s*"[^"]*"'),
+        r'"\1":"REDACTED"',
+    ),
+    ("bearer", re.compile(r"(ya29|1//)[\w.\-]{10,}"), "REDACTED-TOKEN"),
+    ("cookie", re.compile(r"(?i)(set-cookie:)[^\n]*"), r"\1 REDACTED"),
+)
+
+
+def sanitize(text: str) -> tuple[str, list[str]]:
+    """Strip machine/session identifiers. Returns the text and what was touched."""
+    touched: list[str] = []
+    for name, pattern, replacement in _SCRUB_PATTERNS:
+        text, count = pattern.subn(replacement, text)
+        if count:
+            touched.append(f"{name}x{count}")
+    return text, touched
+
+
+def record(
+    query: OracleQuery,
+    *,
+    url: str,
+    directory: Path,
+    name: str,
+    transport: HttpTransport | None = None,
+) -> dict[str, Any]:
+    """Fetch one URL and write `<name>.html` plus `<name>.meta.json` beside it.
+
+    The metadata is the part people forget: a fixture without its egress country,
+    point of sale and capture date is a mystery in six months, and a parser tuned to
+    a mystery is worse than no fixture at all.
+    """
+    transport = transport or HttpTransport(pos=query.pos)
+    egress = transport.detect_egress_country()
+    response = transport.get(url)
+    body, scrubbed = sanitize(response.text)
+
+    directory = Path(directory)
+    directory.mkdir(parents=True, exist_ok=True)
+    (directory / f"{name}.html").write_text(body)
+
+    meta = {
+        "captured_at": datetime.now(UTC).isoformat(timespec="seconds"),
+        "query": query.describe(),
+        "dedup_key": query.dedup_key,
+        "request_url": url,
+        "final_url": response.url,
+        "status_code": response.status_code,
+        "bytes": len(body),
+        "egress_country": egress,
+        "eu_egress": is_eu_egress(egress),
+        "pos": transport.pos_with_egress().model_dump(),
+        "scrubbed": scrubbed,
+        "note": "Recorded for offline parser tests. Private repo (DESIGN §13).",
+    }
+    (directory / f"{name}.meta.json").write_text(json.dumps(meta, indent=2, sort_keys=True))
+    return meta
+
+
+def looks_like_results(html: str) -> bool:
+    """A cheap "did we actually get a results page?" check for the recorder.
+
+    Not a parser — just enough signal to tell a captured results page from a captured
+    consent wall, so a useless fixture is never committed by accident.
+    """
+    if "consent.google.com" in html[:5000] or "Before you continue" in html[:5000]:
+        return False
+    markers = ("AF_initDataCallback", "aria-label", "flt-", "Nonstop", "nonstop")
+    return sum(marker in html for marker in markers) >= 2

layover/oracles/transport.py

@@ -0,0 +1,429 @@
+"""HTTP transport for API oracles, with the point of sale made explicit (§9).
+
+The reason this exists rather than calling `httpx.get` inline: **which exit a price
+came from is part of the price**. A fare seen from a US exit and one seen from a
+Hungarian exit are different observations, and mixing them silently would produce a
+ranking that cannot be reproduced or booked. So the transport carries a `POSContext`,
+stamps it on what it returns, and refuses to guess.
+
+The exit itself is whatever the owner has running — a VPN on the machine, or a
+per-request proxy from `config/pos.toml`. Both are the same idea; only the second is
+per-oracle, which is why `pos.toml` exists at all.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from typing import Any
+
+from layover.models import POSContext
+
+USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+)
+
+EGRESS_LOOKUPS: tuple[tuple[str, str, str], ...] = (
+    ("https://ipapi.co/json", "country_name", "ip"),
+    ("https://ifconfig.co/json", "country", "ip"),
+    ("https://ipinfo.io/json", "country", "ip"),
+)
+"""Independent places to ask where we are leaving from: (url, country key, ip key).
+
+Vendor-neutral on purpose. These are plain geolocation echoes that answer the same
+way whatever the tunnel is — or whether there is one — and none of them belongs to a
+VPN provider: asking one provider's server where you are, while using a different
+provider, is both confusing to read and a needless disclosure.
+
+**Two of them, deliberately.** A country label from a single service was treated as
+ground truth once in this project and turned out to be load-bearing for a comparison;
+geolocation databases mislocate exits, especially datacentre ranges. So at least two
+are asked, their answers are recorded verbatim, and disagreement is surfaced rather
+than resolved by picking a favourite."""
+
+
+@dataclass(frozen=True, slots=True)
+class Egress:
+    """Where requests actually leave from, and how confident we are about it."""
+
+    country: str | None
+    ip: str | None = None
+    family: str = "ipv4"
+    """Which address family this was measured over. A dual-stack machine can leave
+    via different countries per family, so the label is meaningless without it."""
+    sources: tuple[tuple[str, str | None, str | None], ...] = ()
+    """(url, country, ip) as each service answered — kept verbatim for the record."""
+
+    @property
+    def answered(self) -> int:
+        return sum(1 for _url, country, _ip in self.sources if country)
+
+    @property
+    def dissenting(self) -> tuple[str, ...]:
+        """Countries some service reported that the majority did not."""
+        countries = [canonical_country(c) for _u, c, _i in self.sources if c]
+        if not countries:
+            return ()
+        winner = max(set(countries), key=countries.count)
+        return tuple(sorted({c for c in countries if c != winner}))
+
+    @property
+    def agreed(self) -> bool:
+        """Do the services that answered agree on the country *and* the IP?
+
+        Compared canonically — "HU" and "Hungary" are the same answer in different
+        clothes, and treating them as a conflict would drown the real conflicts.
+        """
+        countries = {canonical_country(c) for _u, c, _i in self.sources if c}
+        ips = {i for _u, _c, i in self.sources if i}
+        return len(countries) <= 1 and len(ips) <= 1
+
+    @property
+    def verified(self) -> bool:
+        """Two or more independent services, in agreement."""
+        return self.answered >= 2 and self.agreed
+
+    def describe(self) -> str:
+        if self.country is None:
+            return "unknown"
+        if self.verified:
+            return f"{self.country} ({self.ip} {self.family}, {self.answered} agree)"
+        if self.answered >= 2:
+            detail = "; ".join(f"{u.split('/')[2]}={c}" for u, c, _i in self.sources if c)
+            return f"{self.country} by majority ({self.ip}) — sources disagree: {detail}"
+        return f"{self.country} (single source, unverified)"
+
+
+@dataclass(frozen=True, slots=True)
+class Response:
+    status_code: int
+    url: str
+    text: str
+    latency_ms: float
+    egress_country: str | None = None
+    """Where the request actually left from, when it could be determined."""
+
+
+class HttpTransport:
+    """A thin, POS-aware wrapper. One in-flight request at a time, by construction."""
+
+    def __init__(
+        self,
+        *,
+        pos: POSContext | None = None,
+        proxy: str | None = None,
+        timeout: float = 20.0,
+        headers: dict[str, str] | None = None,
+        egress_country: str | None = None,
+        cookies: dict[str, str] | None = None,
+        force_ipv4: bool = True,
+    ) -> None:
+        self.pos = pos or POSContext()
+        self.proxy = proxy
+        self.timeout = timeout
+        self.cookies = cookies or {}
+        """Consent cookies the owner chose to share (ORACLES §2). Never credentials."""
+        self.force_ipv4 = force_ipv4
+        """Pin requests to IPv4.
+
+        A dual-stack machine behind a VPN can leave via **two different countries at
+        once**: measured 2026-08-05, a UAE tunnel carried IPv4 while IPv6 went out
+        through France — so Google was fetched from an EU address and answered with a
+        consent wall, on a run labelled "United Arab Emirates". Whichever family the
+        request happens to use is the exit that counts, so both the lookups and the
+        oracle traffic are pinned to one family and the label means something."""
+        self.headers = {
+            "User-Agent": USER_AGENT,
+            "Accept-Language": self._accept_language(),
+            **(headers or {}),
+        }
+        self._egress_country = egress_country
+        self._egress: Egress | None = Egress(country=egress_country) if egress_country else None
+
+    def _accept_language(self) -> str:
+        locale = (self.pos.locale or "en-US").replace("_", "-")
+        primary = locale.split("-")[0]
+        return f"{locale},{primary};q=0.9"
+
+    def _client_kwargs(self) -> dict[str, Any]:
+        import httpx
+
+        kwargs: dict[str, Any] = {"proxy": self.proxy, "timeout": self.timeout}
+        if self.force_ipv4 and self.proxy is None:
+            # local_address="0.0.0.0" binds the socket to an IPv4 source, which makes
+            # the connection use IPv4 whatever the DNS answer offers.
+            kwargs["transport"] = httpx.HTTPTransport(local_address="0.0.0.0")
+        return kwargs
+
+    def get(self, url: str, headers: dict[str, str] | None = None, **kwargs: Any) -> Response:
+        """One GET, with this transport's headers plus any the caller adds."""
+        import httpx
+
+        started = time.monotonic()
+        with httpx.Client(
+            **self._client_kwargs(),
+            follow_redirects=True,
+            cookies=self.cookies or None,
+        ) as client:
+            response = client.get(url, headers={**self.headers, **(headers or {})}, **kwargs)
+        return Response(
+            status_code=response.status_code,
+            url=str(response.url),
+            text=response.text,
+            latency_ms=(time.monotonic() - started) * 1000,
+            egress_country=self._egress_country,
+        )
+
+    # ── egress ───────────────────────────────────────────────────────────────
+
+    def detect_egress(self, *, timeout: float = 8.0) -> Egress:
+        """Ask several services where we are, and keep every answer.
+
+        Worth a couple of requests per session: a measurement whose result depends on
+        the exit should be able to say which exit, and how it knows.
+        """
+        import httpx
+
+        sources: list[tuple[str, str | None, str | None]] = []
+        for url, country_key, ip_key in EGRESS_LOOKUPS:
+            try:
+                kwargs = self._client_kwargs()
+                kwargs["timeout"] = timeout
+                with httpx.Client(**kwargs) as client:
+                    response = client.get(url, headers={"Accept": "application/json"})
+                if response.status_code != 200:
+                    continue
+                body = response.json()
+                sources.append((url, body.get(country_key), body.get(ip_key)))
+            except Exception:
+                continue
+            if len(sources) >= 2:
+                break  # two agreeing sources is the bar; a third is only for ties
+
+        countries = [c for _u, c, _i in sources if c]
+        ips = [i for _u, _c, i in sources if i]
+        egress = Egress(
+            country=countries[0] if countries else None,
+            ip=ips[0] if ips else None,
+            sources=tuple(sources),
+        )
+        self._egress = egress
+        self._egress_country = egress.country
+        return egress
+
+    def detect_egress_country(self, *, timeout: float = 8.0) -> str | None:
+        """Ask where we are actually coming from, and remember it.
+
+        Worth one request per session: a search whose results depend on the exit
+        should be able to state which exit it used, and "I assumed the VPN was on" is
+        not a statement. `None` means unknown — never assumed, and never inferred
+        from which provider the config says is configured.
+        """
+        import httpx
+
+        self._egress_country = None
+        for url in EGRESS_LOOKUPS:
+            try:
+                with httpx.Client(proxy=self.proxy, timeout=timeout) as client:
+                    response = client.get(url)
+                if response.status_code == 200 and response.text.strip():
+                    self._egress_country = response.text.strip().splitlines()[0][:40]
+                    break
+            except Exception:
+                continue
+        return self._egress_country
+
+    @property
+    def egress_country(self) -> str | None:
+        return self._egress_country
+
+    @property
+    def egress(self) -> Egress | None:
+        return self._egress
+
+    def pos_with_egress(self) -> POSContext:
+        """The POS as it should be recorded on an offer: what was *observed*.
+
+        If the exit country is known and differs from what the intent asked for, the
+        observed value wins — the offer says where it really came from, and the
+        mismatch is the report's problem to raise, not something to quietly overwrite.
+        """
+        if self._egress_country is None:
+            return self.pos
+        return self.pos.model_copy(update={"ip_country": self._egress_country})
+
+
+COUNTRY_NAMES: dict[str, str] = {
+    "AE": "united arab emirates",
+    "AT": "austria",
+    "AU": "australia",
+    "BE": "belgium",
+    "BG": "bulgaria",
+    "BR": "brazil",
+    "CA": "canada",
+    "CH": "switzerland",
+    "CY": "cyprus",
+    "CZ": "czechia",
+    "DE": "germany",
+    "DK": "denmark",
+    "EE": "estonia",
+    "ES": "spain",
+    "FI": "finland",
+    "FR": "france",
+    "GB": "united kingdom",
+    "GR": "greece",
+    "HR": "croatia",
+    "HU": "hungary",
+    "IE": "ireland",
+    "IN": "india",
+    "IS": "iceland",
+    "IT": "italy",
+    "JP": "japan",
+    "LI": "liechtenstein",
+    "LT": "lithuania",
+    "LU": "luxembourg",
+    "LV": "latvia",
+    "MT": "malta",
+    "NL": "netherlands",
+    "NO": "norway",
+    "PL": "poland",
+    "PT": "portugal",
+    "RO": "romania",
+    "SE": "sweden",
+    "SG": "singapore",
+    "SI": "slovenia",
+    "SK": "slovakia",
+    "TR": "turkey",
+    "US": "united states",
+}
+"""Enough ISO alpha-2 to compare answers, because services differ in *format*.
+
+One returns "Hungary", another "HU". Treating that as a disagreement would cry wolf
+on every measurement and hide the disagreements that matter — like a US exit one
+service calls Czechia."""
+
+
+def canonical_country(value: str | None) -> str | None:
+    """A comparable form: lower-cased name, with alpha-2 codes expanded."""
+    if not value:
+        return None
+    text = value.strip()
+    if len(text) == 2 and text.isalpha():
+        return COUNTRY_NAMES.get(text.upper(), text.lower())
+    return text.lower()
+
+
+EU_EEA_CODES = frozenset(
+    [
+        "AT",
+        "BE",
+        "BG",
+        "HR",
+        "CY",
+        "CZ",
+        "DK",
+        "EE",
+        "FI",
+        "FR",
+        "DE",
+        "GR",
+        "HU",
+        "IE",
+        "IT",
+        "LV",
+        "LT",
+        "LU",
+        "MT",
+        "NL",
+        "PL",
+        "PT",
+        "RO",
+        "SK",
+        "SI",
+        "ES",
+        "SE",
+        "IS",
+        "LI",
+        "NO",
+    ]
+)
+
+EU_EEA_NAMES = frozenset(
+    [
+        "austria",
+        "belgium",
+        "bulgaria",
+        "croatia",
+        "cyprus",
+        "czechia",
+        "czech republic",
+        "denmark",
+        "estonia",
+        "finland",
+        "france",
+        "germany",
+        "greece",
+        "hungary",
+        "ireland",
+        "italy",
+        "latvia",
+        "lithuania",
+        "luxembourg",
+        "malta",
+        "netherlands",
+        "poland",
+        "portugal",
+        "romania",
+        "slovakia",
+        "slovenia",
+        "spain",
+        "sweden",
+        "iceland",
+        "liechtenstein",
+        "norway",
+    ]
+)
+"""Lookups answer with a country *name* ("Hungary"), not always a code — so both forms
+are recognised. Getting this wrong would report an EU exit as non-EU and make an
+entirely expected consent wall look like a site change."""
+
+
+class UnknownEgress(RuntimeError):
+    """The exit could not be determined, and the caller needs it to be."""
+
+
+def check_egress(
+    egress: str | None, *, required: bool = True, override: bool = False
+) -> str | None:
+    """Refusal message when an unknown exit must not be used, else `None`.
+
+    The fail-closed rule in `pos.toml` was written for a *configured proxy* that is
+    down. This is the other way to end up somewhere unknown: the geolocation lookups
+    themselves fail, and the search carries on and records prices with no idea which
+    market produced them. Those observations look identical to good ones in the
+    session store, and a later comparison reads them as peers.
+
+    So an unknown exit is a refusal by default, overridable per run — because for a
+    home-market oracle it genuinely does not matter, and the owner is entitled to say
+    so out loud rather than have it assumed.
+    """
+    if egress is not None or not required or override:
+        return None
+    return (
+        "cannot determine which exit this request would leave from — every "
+        "geolocation lookup failed. Prices recorded now could not be attributed to a "
+        "market, and would sit in the session store looking like any other "
+        "observation. Re-run when the network settles, or pass --allow-unknown-exit "
+        "if the market does not matter for this oracle."
+    )
+
+
+def is_eu_egress(country: str | None) -> bool | None:
+    """Whether a consent wall is expected. `None` when the egress is unknown."""
+    if country is None:
+        return None
+    return (
+        country.strip().upper() in EU_EEA_CODES
+        or (canonical_country(country) or "") in EU_EEA_NAMES
+    )

layover/prefs/__init__.py

@@ -0,0 +1 @@
+"""Preferences: valuations, travellers, parties."""