nextflight 0.3.0__py3-none-any.whl

nextflight/__init__.py

@@ -0,0 +1,68 @@
+"""
+nextflight
+==========
+
+Generic parser for the React Server Components "Flight" wire format that
+Next.js 13+ (App Router) embeds in ``<script>self.__next_f.push([...])</script>``
+tags. Works on any Next.js App Router site.
+
+    from nextflight import extract
+
+    page = extract(html_text)
+    listing = page.find_by_keys({"sections", "meta"})
+"""
+
+import warnings
+
+from .extractor import (
+    FlightExtractor,
+    FlightParseError,
+    extract,
+    find_json_ld,
+)
+
+__all__ = [
+    "FlightExtractor",
+    "FlightParseError",
+    "extract",
+    "find_json_ld",
+    "NextFlightExtractor",  # deprecated alias, see below
+    "extract_json_ld",      # deprecated alias, see below
+]
+
+__version__ = "0.3.0"
+
+
+# ---------------------------------------------------------------------- #
+# Backwards-compatible aliases for the pre-rename API (nextjs_flight_extractor
+# 0.1.x). These will be removed in a future major version -- switch to
+# FlightExtractor / find_json_ld when convenient.
+# ---------------------------------------------------------------------- #
+class NextFlightExtractor(FlightExtractor):
+    """Deprecated alias for :class:`FlightExtractor`. Use ``FlightExtractor`` instead."""
+
+    def __init__(self, *args, **kwargs):
+        warnings.warn(
+            "NextFlightExtractor is deprecated, use nextflight.FlightExtractor instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        super().__init__(*args, **kwargs)
+
+    def find_first(self, *args, **kwargs):
+        warnings.warn(
+            "find_first() is deprecated, use find_one() instead",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return self.find_one(*args, **kwargs)
+
+
+def extract_json_ld(html: str, schema_type=None) -> list:
+    """Deprecated alias for :func:`find_json_ld`. Use ``find_json_ld`` instead."""
+    warnings.warn(
+        "extract_json_ld() is deprecated, use nextflight.find_json_ld() instead",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return find_json_ld(html, type_=schema_type)

nextflight/cli.py

@@ -0,0 +1,86 @@
+"""
+Command-line entry point for quick exploration:
+
+    nextflight page.html --keys sections,meta
+    nextflight https://example.com/product/123 --keys price,title
+    nextflight page.html --all > everything.json
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from .extractor import FlightExtractor
+
+
+def _load_html(source: str) -> str:
+    if source.startswith("http://") or source.startswith("https://"):
+        extractor = FlightExtractor.from_url(source)
+        return extractor.html
+    with open(source, "r", encoding="utf-8") as f:
+        return f.read()
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="nextflight",
+        description="Extract Next.js Flight (__next_f.push) data from a page.",
+    )
+    parser.add_argument("source", help="Path to an HTML file, or a URL to fetch.")
+    parser.add_argument(
+        "--keys", help="Comma-separated keys: find the first object containing all of them."
+    )
+    parser.add_argument(
+        "--all-by-keys", dest="all_by_keys",
+        help="Comma-separated keys: find EVERY object containing all of them (not just the first).",
+    )
+    parser.add_argument(
+        "--type", dest="type_value",
+        help='Find every object whose "@type" (or --type-key) equals this value.',
+    )
+    parser.add_argument("--type-key", default="@type", help='Key to match --type against (default "@type").')
+    parser.add_argument("--text", dest="text_pattern", help="Regex: list every distinct string value on the page that matches it.")
+    parser.add_argument("--get", dest="get_path", help='Dotted path into the resolved page, e.g. "3f.props.price".')
+    parser.add_argument("--all", action="store_true", help="Dump every resolved chunk.")
+    parser.add_argument("--stats", action="store_true", help="Print a quick diagnostic summary instead of data.")
+    parser.add_argument("--save", dest="save_path", help="Write output to this file instead of stdout.")
+    parser.add_argument("--indent", type=int, default=2, help="JSON indent for output (default 2).")
+    args = parser.parse_args(argv)
+
+    html = _load_html(args.source)
+    extractor = FlightExtractor(html)
+
+    if args.keys:
+        keys = {k.strip() for k in args.keys.split(",") if k.strip()}
+        result = extractor.find_by_keys(keys)
+    elif args.all_by_keys:
+        keys = {k.strip() for k in args.all_by_keys.split(",") if k.strip()}
+        result = extractor.find_all_by_keys(keys)
+    elif args.type_value:
+        result = extractor.find_by_type(args.type_value, key=args.type_key)
+    elif args.text_pattern:
+        result = extractor.find_text(args.text_pattern)
+    elif args.get_path:
+        result = extractor.get(args.get_path)
+    elif args.stats:
+        result = extractor.stats()
+    elif args.all:
+        result = extractor.resolve_all()
+    else:
+        parser.error("Provide one of --keys, --all-by-keys, --type, --text, --get, --stats, or --all.")
+        return 2
+
+    output = json.dumps(result, indent=args.indent, ensure_ascii=False, default=str)
+    if args.save_path:
+        with open(args.save_path, "w", encoding="utf-8") as f:
+            f.write(output)
+    else:
+        sys.stdout.write(output)
+        sys.stdout.write("\n")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

nextflight/extractor.py

@@ -0,0 +1,556 @@
+"""
+nextflight.extractor
+
+Core parser for the React Server Components "Flight" wire format that
+Next.js 13+ (App Router) embeds in ``<script>self.__next_f.push([...])</script>``
+tags. Works on any Next.js App Router site -- nothing here is tied to a
+specific project.
+
+Why not just str.split('\n') on the payload?
+---------------------------------------------
+Because two of the row kinds break that assumption:
+
+  * Text rows:    `id:T<hexByteLen>,<raw text of exactly hexByteLen bytes>`
+    The raw text is a byte-length-prefixed blob, not newline-terminated.
+    It can legitimately CONTAIN literal newlines, and it can run directly
+    into the NEXT row's id with zero separator.
+  * Module rows:  `id:I[...]` / anonymous preload rows: `:HL[...]`
+
+And even once rows are split correctly, the values are full of `$`-sigil
+references Next.js uses to dedupe repeated subtrees (`$3`, `$L41`, `$@20`,
+`$Sreact.fragment`, and path-suffixed refs like
+`$3f:props:children:0:props:sections:...`). Hardcoding array indices like
+`data[3]["children"][0][3]["children"][3][3]` breaks the moment the
+surrounding component tree reshuffles on a redeploy. This module resolves
+those references and lets you *search* for the shape of data you want
+instead.
+
+Quick start
+-----------
+    from nextflight import extract
+
+    page = extract(html_text)
+
+    # Find whatever object looks like the data you need, wherever
+    # Next.js decided to put it in this particular build:
+    listing = page.find_by_keys({"sections", "meta"})
+
+    # Or with a custom predicate:
+    products = page.find_all(lambda n: isinstance(n, dict) and n.get("@type") == "Product")
+
+    # Or just get everything, fully dereferenced:
+    everything = page.resolve_all()
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.request
+from typing import Any, Callable, Iterable, Iterator, Optional
+
+
+class FlightParseError(Exception):
+    """Raised only when ``strict=True`` and a row cannot be parsed at all."""
+
+
+def _coerce_html(source: Any) -> str:
+    """Accept a raw HTML string/bytes, or a response-like object (Scrapy's
+    ``Response``, ``requests.Response``, httpx, etc.) and return plain text.
+
+    This means both of these just work:
+
+        FlightExtractor(response)          # Scrapy / requests response
+        FlightExtractor(response.text)     # or the plain string, as before
+    """
+    if isinstance(source, str):
+        return source
+    if isinstance(source, (bytes, bytearray)):
+        return bytes(source).decode("utf-8", errors="replace")
+    text_attr = getattr(source, "text", None)
+    if isinstance(text_attr, str):
+        return text_attr
+    body_attr = getattr(source, "body", None)
+    if isinstance(body_attr, (bytes, bytearray)):
+        return bytes(body_attr).decode("utf-8", errors="replace")
+    raise TypeError(
+        "Expected an HTML string, bytes, or a response-like object with a "
+        ".text or .body attribute (e.g. a Scrapy or requests Response); "
+        f"got {type(source).__name__}"
+    )
+
+
+class FlightExtractor:
+    """Parses and searches the Next.js Flight payloads embedded in a page.
+
+    Parameters
+    ----------
+    html:
+        The full HTML of a server-rendered Next.js App Router page. Also
+        accepts bytes, or a response-like object with a `.text`/`.body`
+        attribute (Scrapy's `Response`, `requests.Response`, etc.).
+    strict:
+        If True, raise :class:`FlightParseError` when a row's value isn't
+        valid JSON and doesn't look like a bare `$`-reference marker.
+        Default False: such rows are kept as raw strings so a handful of
+        odd rows never take down extraction of everything else on the page.
+    """
+
+    _REF_RE = re.compile(r"^\$(?P<sigil>[A-Z@]{0,2})(?P<id>[^:\s]+)(?::(?P<path>.+))?$")
+    _PUSH_CALL_RE = re.compile(r"self\.__next_f\.push\(")
+    _ROW_START_RE = re.compile(r"[0-9a-zA-Z_\-]*:")
+
+    def __init__(self, html: Any, *, strict: bool = False):
+        self.html = _coerce_html(html)
+        self.strict = strict
+        self.raw_chunks: dict[str, Any] = {}
+        self._resolved_cache: dict[str, Any] = {}
+        self._resolving: set[str] = set()
+        self._extract_all()
+
+    def __repr__(self) -> str:
+        return f"<FlightExtractor chunks={len(self.raw_chunks)}>"
+
+    def __len__(self) -> int:
+        return len(self.raw_chunks)
+
+    def __iter__(self):
+        return iter(self.raw_chunks)
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.raw_chunks
+
+    def __getitem__(self, key: str) -> Any:
+        """``page[key]`` is shorthand for ``page.resolve_chunk(key)``, but
+        raises KeyError (like a normal dict) instead of returning None for
+        a key that was never pushed onto the page at all."""
+        if key not in self.raw_chunks:
+            raise KeyError(key)
+        return self.resolve_chunk(key)
+
+    def keys(self) -> list:
+        """Every chunk id found on the page, in the order they were pushed.
+        This is 'step 1': see what's there before deciding what to resolve."""
+        return list(self.raw_chunks.keys())
+
+    # ------------------------------------------------------------------ #
+    # Construction helpers
+    # ------------------------------------------------------------------ #
+    @classmethod
+    def from_url(cls, url: str, *, timeout: float = 15.0, headers: Optional[dict] = None,
+                 strict: bool = False) -> "FlightExtractor":
+        """Fetch a URL with the stdlib (no extra dependencies) and parse it.
+
+        For anything beyond quick, one-off exploration -- retries, proxies,
+        rendering JS, respecting robots.txt -- fetch the page with your own
+        HTTP client / Scrapy / Zyte and pass ``response.text`` to the
+        normal constructor instead.
+        """
+        req = urllib.request.Request(
+            url,
+            headers=headers or {"User-Agent": "Mozilla/5.0 (nextflight)"},
+        )
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            charset = resp.headers.get_content_charset() or "utf-8"
+            html = resp.read().decode(charset, errors="replace")
+        return cls(html, strict=strict)
+
+    # ------------------------------------------------------------------ #
+    # Step 1 -- find every push([...]) call, bracket/quote aware, so it
+    # doesn't matter how many <script> tags they're spread across.
+    # ------------------------------------------------------------------ #
+    def _iter_push_payloads(self) -> Iterator[str]:
+        for m in self._PUSH_CALL_RE.finditer(self.html):
+            array_text, _end = self._read_balanced(self.html, m.end(), "[", "]")
+            if array_text is None:
+                continue
+            try:
+                value = json.loads(array_text)
+            except json.JSONDecodeError:
+                continue
+            # push([0]) is an init call with no payload string; the ones we
+            # want look like push([1, "...rows..."])
+            if isinstance(value, list) and len(value) > 1 and isinstance(value[1], str):
+                yield value[1]
+
+    @staticmethod
+    def _read_balanced(text: str, start: int, open_ch: str, close_ch: str):
+        """Find `open_ch` at/after `start`, then return (substring, end_idx)
+        where substring spans to its matching `close_ch`, respecting quoted
+        strings and backslash escapes inside them. (None, start) if no
+        balanced match exists (e.g. truncated HTML)."""
+        i = start
+        n = len(text)
+        while i < n and text[i] != open_ch:
+            i += 1
+        if i >= n:
+            return None, start
+        begin = i
+        depth = 0
+        in_str = False
+        esc = False
+        while i < n:
+            ch = text[i]
+            if in_str:
+                if esc:
+                    esc = False
+                elif ch == "\\":
+                    esc = True
+                elif ch == '"':
+                    in_str = False
+            else:
+                if ch == '"':
+                    in_str = True
+                elif ch == open_ch:
+                    depth += 1
+                elif ch == close_ch:
+                    depth -= 1
+                    if depth == 0:
+                        return text[begin:i + 1], i + 1
+            i += 1
+        return None, n
+
+    # ------------------------------------------------------------------ #
+    # Step 2 -- split a payload string into (chunk_id, row_type, raw_value)
+    # honouring the real row grammar (this is what a '\n'.split() breaks).
+    # ------------------------------------------------------------------ #
+    def _split_rows(self, payload: str) -> Iterator[tuple]:
+        i = 0
+        n = len(payload)
+        while i < n:
+            if payload[i] == "\n":
+                i += 1
+                continue
+            m = self._ROW_START_RE.match(payload, i)
+            if not m:
+                break
+            chunk_id = m.group(0)[:-1]
+            i = m.end()
+            if i >= n:
+                # Truncated payload: an id: was matched but nothing follows
+                # it (e.g. the row got cut off at a chunk boundary, which
+                # happens with proxies/CDNs that truncate responses).
+                # Nothing more to parse.
+                break
+            if payload[i] == "T":
+                j = payload.index(",", i + 1)
+                hex_len = int(payload[i + 1:j], 16)
+                body_start = j + 1
+                remaining_bytes = payload[body_start:].encode("utf-8")
+                text_bytes = remaining_bytes[:hex_len]
+                text_str = text_bytes.decode("utf-8")
+                i = body_start + len(text_str)
+                yield chunk_id, "text", text_str
+            elif payload[i:i + 2] == "HL":
+                val, end = self._read_balanced(payload, i + 2, "[", "]")
+                if val is None:
+                    break
+                i = end
+                yield chunk_id, "preload", val
+            elif payload[i] == "I":
+                val, end = self._read_balanced(payload, i + 1, "[", "]")
+                if val is None:
+                    break
+                i = end
+                yield chunk_id, "module", val
+            else:
+                if payload[i] in "[{":
+                    close_ch = "]" if payload[i] == "[" else "}"
+                    val, end = self._read_balanced(payload, i, payload[i], close_ch)
+                    if val is None:
+                        val, i = payload[i:], n
+                    else:
+                        i = end
+                elif payload[i] == '"':
+                    val = self._read_quoted_string(payload, i)
+                    i += len(val)
+                else:
+                    nxt = re.search(r"\n[0-9a-zA-Z_\-]*:", payload[i:])
+                    end = i + nxt.start() if nxt else n
+                    val, i = payload[i:end], end
+                yield chunk_id, "json", val
+
+    @staticmethod
+    def _read_quoted_string(text: str, start: int) -> str:
+        i, n, esc = start + 1, len(text), False
+        while i < n:
+            ch = text[i]
+            if esc:
+                esc = False
+            elif ch == "\\":
+                esc = True
+            elif ch == '"':
+                return text[start:i + 1]
+            i += 1
+        return text[start:]
+
+    def _extract_all(self) -> None:
+        for payload in self._iter_push_payloads():
+            for chunk_id, row_type, raw_value in self._split_rows(payload):
+                if row_type == "text":
+                    self.raw_chunks[chunk_id] = raw_value
+                elif row_type in ("module", "preload"):
+                    try:
+                        self.raw_chunks[chunk_id] = json.loads(raw_value)
+                    except json.JSONDecodeError:
+                        if self.strict:
+                            raise FlightParseError(
+                                f"chunk {chunk_id!r}: invalid {row_type} JSON: {raw_value[:80]!r}"
+                            )
+                        self.raw_chunks[chunk_id] = raw_value
+                else:
+                    if raw_value == "$undefined":
+                        self.raw_chunks[chunk_id] = None
+                        continue
+                    try:
+                        self.raw_chunks[chunk_id] = json.loads(raw_value)
+                    except json.JSONDecodeError:
+                        if self.strict and not raw_value.startswith("$"):
+                            raise FlightParseError(
+                                f"chunk {chunk_id!r}: invalid JSON: {raw_value[:80]!r}"
+                            )
+                        self.raw_chunks[chunk_id] = raw_value  # bare marker e.g. "X"
+
+    # ------------------------------------------------------------------ #
+    # Step 3 -- resolve '$'-sigil references into real values, recursively.
+    # ------------------------------------------------------------------ #
+    def resolve_chunk(self, chunk_id: str) -> Any:
+        """Resolve a single chunk (by its id) with all `$`-refs dereferenced."""
+        if chunk_id in self._resolved_cache:
+            return self._resolved_cache[chunk_id]
+        if chunk_id in self._resolving or chunk_id not in self.raw_chunks:
+            return None
+        self._resolving.add(chunk_id)
+        resolved = self._resolve_value(self.raw_chunks[chunk_id])
+        self._resolving.discard(chunk_id)
+        self._resolved_cache[chunk_id] = resolved
+        return resolved
+
+    def _resolve_value(self, value: Any) -> Any:
+        if isinstance(value, str):
+            return self._resolve_ref_string(value)
+        if isinstance(value, list):
+            return [self._resolve_value(v) for v in value]
+        if isinstance(value, dict):
+            return {k: self._resolve_value(v) for k, v in value.items()}
+        return value
+
+    def _resolve_ref_string(self, s: str) -> Any:
+        if s == "$undefined":
+            return None
+        if not s.startswith("$"):
+            return s
+        if s.startswith("$$"):  # escaped literal '$...'
+            return s[1:]
+        m = self._REF_RE.match(s)
+        if not m:
+            return s
+        sigil, ref_id, path = m.group("sigil"), m.group("id"), m.group("path")
+        if sigil == "S":
+            return {"__symbol__": ref_id}
+        if ref_id not in self.raw_chunks:
+            return s
+        value = self.resolve_chunk(ref_id)
+        if path:
+            value = self._walk_path(value, path.split(":"))
+        return value
+
+    @staticmethod
+    def _walk_path(value: Any, parts: list) -> Any:
+        for part in parts:
+            if isinstance(value, list):
+                try:
+                    value = value[int(part)]
+                except (ValueError, IndexError):
+                    return None
+            elif isinstance(value, dict):
+                value = value.get(part)
+            else:
+                return None
+        return value
+
+    def resolve_all(self) -> dict:
+        """Every chunk on the page, fully dereferenced."""
+        return {cid: self.resolve_chunk(cid) for cid in list(self.raw_chunks)}
+
+    # ------------------------------------------------------------------ #
+    # Step 4 -- schema-free search over the fully resolved data.
+    # ------------------------------------------------------------------ #
+    def find_all(self, predicate: Callable[[Any], bool], root: Any = None,
+                 max_results: Optional[int] = None) -> list:
+        """Walk the whole resolved tree and collect every node matching `predicate`."""
+        data = self.resolve_all() if root is None else root
+        results: list = []
+        seen: set = set()
+
+        def walk(node: Any):
+            if max_results is not None and len(results) >= max_results:
+                return
+            if id(node) in seen:
+                return
+            if isinstance(node, (dict, list)):
+                seen.add(id(node))
+            if predicate(node):
+                results.append(node)
+                if max_results is not None and len(results) >= max_results:
+                    return
+            if isinstance(node, dict):
+                for v in node.values():
+                    walk(v)
+            elif isinstance(node, list):
+                for v in node:
+                    walk(v)
+
+        if isinstance(data, dict):
+            for v in data.values():
+                walk(v)
+        else:
+            walk(data)
+        return results
+
+    def find_one(self, predicate: Callable[[Any], bool], root: Any = None) -> Any:
+        """Like :meth:`find_all` but returns just the first match, or None."""
+        r = self.find_all(predicate, root=root, max_results=1)
+        return r[0] if r else None
+
+    def find_by_keys(self, required_keys: Iterable[str], root: Any = None) -> Any:
+        """Find the first dict containing ALL of `required_keys` -- the
+        pattern you almost always want: 'give me whatever object looks
+        like the data I need', regardless of where this build's component
+        tree happened to put it."""
+        required_keys = set(required_keys)
+        return self.find_one(
+            lambda n: isinstance(n, dict) and required_keys <= n.keys(), root=root
+        )
+
+    def find_all_by_keys(self, required_keys: Iterable[str], root: Any = None) -> list:
+        """Like :meth:`find_by_keys` but returns every matching dict, not
+        just the first -- useful for pages with repeated cards/listings
+        that all share the same shape (product cards, search results, ...)."""
+        required_keys = set(required_keys)
+        return self.find_all(
+            lambda n: isinstance(n, dict) and required_keys <= n.keys(), root=root
+        )
+
+    def find_by_type(self, type_value: str, *, key: str = "@type", root: Any = None) -> list:
+        """Find every dict whose `key` field equals `type_value` (default key
+        "@type", matching schema.org-style typed objects Next.js often embeds
+        e.g. {"@type": "Product", ...})."""
+        return self.find_all(
+            lambda n: isinstance(n, dict) and n.get(key) == type_value, root=root
+        )
+
+    def find_text(self, pattern, root: Any = None) -> list:
+        """Regex-search every string value in the resolved tree and return
+        the distinct whole string values that contain a match (this is a
+        substring search, like `re.search`, not an exact-match filter), in
+        the order first seen. Handy for pulling emails, phone numbers,
+        prices, or SKUs out of a page without having to know which object
+        they live on.
+
+            page.find_text(r"^\\$[\\d,]+(\\.\\d{2})?$")   # dollar amounts
+            page.find_text(re.compile(r"[\\w.+-]+@[\\w-]+\\.\\w+"))  # emails
+        """
+        compiled = re.compile(pattern) if isinstance(pattern, str) else pattern
+        data = self.resolve_all() if root is None else root
+        matches: list = []
+        seen: set = set()
+
+        def walk(node: Any):
+            if isinstance(node, str):
+                if node not in seen and compiled.search(node):
+                    seen.add(node)
+                    matches.append(node)
+            elif isinstance(node, dict):
+                for v in node.values():
+                    walk(v)
+            elif isinstance(node, list):
+                for v in node:
+                    walk(v)
+
+        walk(data)
+        return matches
+
+    def get(self, path: str, default: Any = None, sep: str = ".") -> Any:
+        """Navigate the fully resolved page with a dotted path of dict keys
+        and/or list indices, e.g. ``page.get("3f.props.product.price")`` or
+        ``page.get("items.0.name")``. Returns `default` if any segment is
+        missing, instead of raising -- meant for quick, tolerant lookups
+        once you already know roughly where something lives on this site."""
+        current: Any = self.resolve_all()
+        for part in path.split(sep):
+            if isinstance(current, dict):
+                if part not in current:
+                    return default
+                current = current[part]
+            elif isinstance(current, list):
+                try:
+                    current = current[int(part)]
+                except (ValueError, IndexError):
+                    return default
+            else:
+                return default
+        return current
+
+    def stats(self) -> dict:
+        """A quick diagnostic snapshot -- handy the first time you point
+        this at a new site and want a feel for what's on the page before
+        writing search predicates."""
+        row_types: dict[str, int] = {}
+        for value in self.raw_chunks.values():
+            kind = type(value).__name__
+            row_types[kind] = row_types.get(kind, 0) + 1
+        return {
+            "chunk_count": len(self.raw_chunks),
+            "chunk_ids": self.keys(),
+            "value_type_counts": row_types,
+            "html_size_bytes": len(self.html.encode("utf-8")),
+        }
+
+    def to_json(self, path: Optional[str] = None, *, indent: int = 2) -> Optional[str]:
+        """Dump the fully resolved page as JSON. Writes to `path` if given
+        (returns None), otherwise returns the JSON string."""
+        text = json.dumps(self.resolve_all(), indent=indent, ensure_ascii=False, default=str)
+        if path is None:
+            return text
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(text)
+        return None
+
+
+def find_json_ld(html: Any, type_: Optional[str] = None) -> list:
+    """Parse any <script type="application/ld+json"> blocks on the page,
+    independent of Flight data and often more stable across redesigns --
+    worth trying first for structured product/article/breadcrumb data.
+
+    `html` accepts a raw string/bytes, or a response-like object (Scrapy's
+    `Response`, `requests.Response`, etc.). `type_` optionally filters
+    results by their "@type" (e.g. "Product")."""
+    html = _coerce_html(html)
+    blocks = re.findall(
+        r'<script[^>]+type=["\']application/ld\+json["\'][^>]*>(.*?)</script>',
+        html, re.DOTALL,
+    )
+    out = []
+    for b in blocks:
+        try:
+            parsed = json.loads(b)
+        except json.JSONDecodeError:
+            continue
+        for c in (parsed if isinstance(parsed, list) else [parsed]):
+            if not isinstance(c, dict):
+                continue
+            types = c.get("@type")
+            if type_ is None or types == type_ or (
+                isinstance(types, list) and type_ in types
+            ):
+                out.append(c)
+    return out
+
+
+def extract(html: Any, *, strict: bool = False) -> FlightExtractor:
+    """Shorthand for ``FlightExtractor(html)``. `html` accepts a raw
+    string/bytes, or a response-like object (Scrapy's `Response`,
+    `requests.Response`, etc.) -- you can pass `response` straight from a
+    Scrapy `parse()` method without writing `response.text` yourself."""
+    return FlightExtractor(html, strict=strict)

nextflight-0.3.0.dist-info/METADATA

@@ -0,0 +1,244 @@
+Metadata-Version: 2.4
+Name: nextflight
+Version: 0.3.0
+Summary: Parse Next.js App Router 'Flight' (__next_f.push) payloads embedded in server-rendered HTML -- works on any Next.js 13+ site.
+Author: Aly Reda
+License: MIT
+Project-URL: Homepage, https://github.com/Aly-Reda/nextflight
+Project-URL: Issues, https://github.com/Aly-Reda/nextflight/issues
+Keywords: nextjs,scraping,scrapy,flight,rsc,react-server-components,web-scraping,zyte
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Dynamic: license-file
+
+# nextflight
+
+A general-purpose parser for the data Next.js (App Router) embeds in
+`<script>self.__next_f.push([...])</script>` tags — the React Server
+Components "Flight" wire format. Works on **any** Next.js 13+ App Router
+site, not just one particular project.
+
+Instead of hardcoding array indices like `data[3]["children"][0][3]...`,
+which break the moment a site's component tree reshuffles on redeploy,
+`nextflight` resolves the `$`-sigil references Next.js uses internally
+and lets you *search* for the shape of data you want.
+
+## Install
+
+```bash
+pip install nextflight
+```
+
+## Quick start
+
+The core workflow is two steps: hand it any HTML, see what keys are on
+the page, then fetch the resolved JSON for whichever key you want.
+
+```python
+from nextflight import extract
+
+# Step 1: send any HTML, get the list of keys (one per __next_f.push chunk)
+page = extract(html_text)
+print(page.keys())          # e.g. ['0', '1', '3f', '20', ...]
+
+# Step 2: fetch the resolved JSON for a specific key
+data = page["3f"]           # same as page.resolve_chunk("3f")
+```
+
+Chunk ids are arbitrary per build though (a redeploy can renumber them),
+so in practice you'll usually skip straight to *searching* for the shape
+of data you want instead of a specific id:
+
+```python
+from nextflight import extract
+
+page = extract(html_text)
+
+# Find the first object anywhere in the page that has all of these keys,
+# wherever this build's component tree happened to put it:
+listing = page.find_by_keys({"sections", "meta"})
+
+# Find every node with a given @type (or any custom key):
+products = page.find_by_type("Product")
+
+# Or search with a fully custom predicate:
+items = page.find_all(lambda n: isinstance(n, dict) and "price" in n)
+
+# Or grab everything, fully dereferenced, and inspect by hand:
+everything = page.resolve_all()
+```
+
+### Command line
+
+For quick, no-script exploration of a page you've already saved (or a live URL):
+
+```bash
+nextflight page.html --keys sections,meta
+nextflight https://example.com/product/123 --type Product
+nextflight page.html --all > everything.json
+```
+
+### In a Scrapy / Zyte spider
+
+```python
+import scrapy
+from nextflight import extract
+
+class MySpider(scrapy.Spider):
+    name = "my_spider"
+
+    def parse(self, response):
+        page = extract(response.text)
+
+        items = page.find_all(
+            lambda n: isinstance(n, dict) and "price" in n and "title" in n
+        )
+        for item in items:
+            yield {
+                "title": item.get("title"),
+                "price": item.get("price"),
+                "url": response.url,
+            }
+```
+
+### Fetching a URL directly (no Scrapy needed)
+
+```python
+from nextflight import FlightExtractor
+
+page = FlightExtractor.from_url("https://example.com/product/123")
+product = page.find_by_keys({"price", "title"})
+```
+
+(`from_url` uses only the stdlib for quick one-off exploration. For
+production crawling — retries, proxies, JS rendering, robots.txt — fetch
+the page with your own HTTP client / Scrapy / Zyte and pass
+`response.text` to `FlightExtractor(...)` / `extract(...)` instead.)
+
+## API
+
+- **`extract(html) -> FlightExtractor`** — shorthand constructor. `html`
+  accepts a plain string, bytes, or a response-like object (Scrapy's
+  `Response`, `requests.Response`, etc.) — pass `response` straight from a
+  `parse()` method without writing `response.text` yourself.
+- **`FlightExtractor(html, *, strict: bool = False)`**
+  - `.keys() -> list[str]` — every chunk id found on the page, in order.
+  - `page["3f"]` / `.resolve_chunk("3f")` — the resolved JSON for one
+    specific chunk id (`page[...]` raises `KeyError` if it doesn't exist;
+    `resolve_chunk` returns `None`). `"3f" in page` and `for k in page`
+    also work, like a dict.
+  - `.resolve_all() -> dict` — every chunk, fully dereferenced.
+  - `.find_all(predicate, root=None, max_results=None) -> list` — walk the
+    resolved tree and collect every node matching `predicate`.
+  - `.find_one(predicate, root=None) -> Any | None`
+  - `.find_by_keys(required_keys, root=None) -> dict | None` — find the
+    first dict containing all of `required_keys`.
+  - `.find_all_by_keys(required_keys, root=None) -> list` — like
+    `find_by_keys` but returns every match, for pages with repeated
+    cards/listings that share the same shape.
+  - `.find_by_type(type_value, key="@type", root=None) -> list` — find
+    every dict whose `key` field equals `type_value`.
+  - `.find_text(pattern, root=None) -> list` — regex-search every string
+    value on the page and return the distinct whole values that contain a
+    match (emails, prices, phone numbers, SKUs, ...) without needing to
+    know which object they live on.
+  - `.get("path.to.value", default=None) -> Any` — tolerant dotted-path
+    lookup into the resolved page (dict keys and/or list indices), once
+    you already know roughly where something lives on this site.
+  - `.stats() -> dict` — quick diagnostic snapshot (chunk count, ids,
+    value type counts, page size) for exploring a new site.
+  - `.to_json(path=None, indent=2) -> str | None` — dump the fully
+    resolved page to a file, or return it as a JSON string.
+  - `.from_url(url, timeout=15.0, headers=None) -> FlightExtractor`
+    (classmethod) — fetch and parse a URL using only the stdlib.
+  - `strict=True` raises `FlightParseError` on a row that's neither valid
+    JSON nor a recognizable `$`-reference marker, instead of silently
+    keeping it as a raw string (useful while developing a new scraper;
+    leave off in production so a handful of odd rows never take down
+    extraction of everything else on the page).
+- **`find_json_ld(html, type_=None) -> list`** — parse any
+  `<script type="application/ld+json">` blocks on the page, optionally
+  filtered by `@type`. Also accepts response-like objects.
+- **CLI**: `nextflight <file-or-url> [--keys a,b | --all-by-keys a,b | --type Product | --text PATTERN | --get path.to.value | --stats | --all] [--save out.json]`
+
+No runtime dependencies — stdlib only (`json`, `re`, `urllib`, `argparse`)
+— so it's safe to drop into any existing Scrapy/Zyte project without
+touching the rest of your dependency tree.
+
+### Upgrading from `nextjs-flight-extractor` / `NextFlightExtractor`
+
+The old names still work but emit a `DeprecationWarning`:
+
+| Old (0.1.x)                          | New (0.2.x+)                     |
+|---------------------------------------|----------------------------------|
+| `from nextjs_flight_extractor import NextFlightExtractor` | `from nextflight import FlightExtractor` |
+| `extractor.find_first(...)`           | `page.find_one(...)`            |
+| `extract_json_ld(html, schema_type=…)`| `find_json_ld(html, type_=…)`   |
+
+## Why not just `str.split('\n')`?
+
+Two of the Flight row kinds break that assumption:
+
+- **Text rows** (`id:T<hexByteLen>,<raw text>`) are byte-length-prefixed
+  blobs, not newline-terminated, and can contain literal newlines or run
+  directly into the next row's id with zero separator.
+- **Module / preload rows** (`id:I[...]` / `:HL[...]`) need bracket-aware
+  parsing.
+
+`nextflight` implements the real row grammar, quote/escape aware, so it
+holds up on both well-formed and truncated payloads (e.g. from a proxy
+that cuts a response off mid-chunk).
+
+## Building / publishing
+
+### Manual (twine)
+
+```bash
+pip install build twine
+python -m build              # produces dist/*.whl and dist/*.tar.gz
+twine check dist/*           # validate metadata before uploading
+twine upload dist/*          # publish to PyPI (or use --repository testpypi for a dry run)
+```
+
+### Automatic (GitHub Actions + PyPI Trusted Publishing)
+
+This repo ships with `.github/workflows/ci.yml`, which:
+- runs the test suite on every push/PR across Python 3.9–3.12,
+- builds and validates the sdist/wheel,
+- publishes to PyPI automatically whenever a tag like `v0.2.1` is pushed.
+
+Publishing uses PyPI's **Trusted Publisher** flow — no API token stored in
+GitHub secrets. One-time setup:
+
+1. On [pypi.org](https://pypi.org), go to your project → *Publishing* →
+   *Add a new publisher* (or, for a brand-new project name, do this from
+   your PyPI account's "Trusted Publishers" management page before the
+   project exists yet).
+2. Fill in: Owner = your GitHub username/org, Repository = this repo's
+   name, Workflow name = `ci.yml`, Environment name = `pypi`.
+3. In your GitHub repo, go to *Settings → Environments*, create an
+   environment named `pypi` (optionally require a manual approval before
+   deploys, for extra safety).
+4. Release a new version:
+   ```bash
+   # bump version in pyproject.toml and src/nextflight/__init__.py first
+   git commit -am "Release v0.2.2"
+   git tag v0.2.2
+   git push origin main --tags
+   ```
+   The workflow builds, tests, and publishes automatically.
+
+
+## License
+
+MIT

nextflight-0.3.0.dist-info/RECORD

@@ -0,0 +1,9 @@
+nextflight/__init__.py,sha256=YMtSkaAYlkPMSooKGapYJSr9_eaqQ5-jr57SyU38uU8,2052
+nextflight/cli.py,sha256=itGnfTs8saWByxcw3gOLQi3A2vVe2h_8uGleJ-CkRKA,3330
+nextflight/extractor.py,sha256=Xng1skJTVrt97Q8mCs15W2385Z4p5XRisITo91kv2b4,22965
+nextflight-0.3.0.dist-info/licenses/LICENSE,sha256=j9lAneLMy0MUiZBPpVcj7Jr284CZIpMjq-Z9yO82410,1065
+nextflight-0.3.0.dist-info/METADATA,sha256=fC8Zjw5QQVc-uJqnVRJrsYZm_Ep5pBjIv0g1A4mCRZQ,9862
+nextflight-0.3.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+nextflight-0.3.0.dist-info/entry_points.txt,sha256=px-_KjhkGmUfBR6qlnjvcbsZbBeLQ7PvfgyKDFRXxk4,51
+nextflight-0.3.0.dist-info/top_level.txt,sha256=IlFQ8rgdkbLdLtcoCDcFLLxUMnz_1RBLDhYyJzWHraE,11
+nextflight-0.3.0.dist-info/RECORD,,

nextflight-0.3.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

nextflight-0.3.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+nextflight = nextflight.cli:main

nextflight-0.3.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Aly Reda
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

nextflight-0.3.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+nextflight