nextflight 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Aly Reda
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,244 @@
1
+ Metadata-Version: 2.4
2
+ Name: nextflight
3
+ Version: 0.3.0
4
+ Summary: Parse Next.js App Router 'Flight' (__next_f.push) payloads embedded in server-rendered HTML -- works on any Next.js 13+ site.
5
+ Author: Aly Reda
6
+ License: MIT
7
+ Project-URL: Homepage, https://github.com/Aly-Reda/nextflight
8
+ Project-URL: Issues, https://github.com/Aly-Reda/nextflight/issues
9
+ Keywords: nextjs,scraping,scrapy,flight,rsc,react-server-components,web-scraping,zyte
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.9
12
+ Classifier: Programming Language :: Python :: 3.10
13
+ Classifier: Programming Language :: Python :: 3.11
14
+ Classifier: Programming Language :: Python :: 3.12
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
18
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
19
+ Requires-Python: >=3.9
20
+ Description-Content-Type: text/markdown
21
+ License-File: LICENSE
22
+ Dynamic: license-file
23
+
24
+ # nextflight
25
+
26
+ A general-purpose parser for the data Next.js (App Router) embeds in
27
+ `<script>self.__next_f.push([...])</script>` tags — the React Server
28
+ Components "Flight" wire format. Works on **any** Next.js 13+ App Router
29
+ site, not just one particular project.
30
+
31
+ Instead of hardcoding array indices like `data[3]["children"][0][3]...`,
32
+ which break the moment a site's component tree reshuffles on redeploy,
33
+ `nextflight` resolves the `$`-sigil references Next.js uses internally
34
+ and lets you *search* for the shape of data you want.
35
+
36
+ ## Install
37
+
38
+ ```bash
39
+ pip install nextflight
40
+ ```
41
+
42
+ ## Quick start
43
+
44
+ The core workflow is two steps: hand it any HTML, see what keys are on
45
+ the page, then fetch the resolved JSON for whichever key you want.
46
+
47
+ ```python
48
+ from nextflight import extract
49
+
50
+ # Step 1: send any HTML, get the list of keys (one per __next_f.push chunk)
51
+ page = extract(html_text)
52
+ print(page.keys()) # e.g. ['0', '1', '3f', '20', ...]
53
+
54
+ # Step 2: fetch the resolved JSON for a specific key
55
+ data = page["3f"] # same as page.resolve_chunk("3f")
56
+ ```
57
+
58
+ Chunk ids are arbitrary per build though (a redeploy can renumber them),
59
+ so in practice you'll usually skip straight to *searching* for the shape
60
+ of data you want instead of a specific id:
61
+
62
+ ```python
63
+ from nextflight import extract
64
+
65
+ page = extract(html_text)
66
+
67
+ # Find the first object anywhere in the page that has all of these keys,
68
+ # wherever this build's component tree happened to put it:
69
+ listing = page.find_by_keys({"sections", "meta"})
70
+
71
+ # Find every node with a given @type (or any custom key):
72
+ products = page.find_by_type("Product")
73
+
74
+ # Or search with a fully custom predicate:
75
+ items = page.find_all(lambda n: isinstance(n, dict) and "price" in n)
76
+
77
+ # Or grab everything, fully dereferenced, and inspect by hand:
78
+ everything = page.resolve_all()
79
+ ```
80
+
81
+ ### Command line
82
+
83
+ For quick, no-script exploration of a page you've already saved (or a live URL):
84
+
85
+ ```bash
86
+ nextflight page.html --keys sections,meta
87
+ nextflight https://example.com/product/123 --type Product
88
+ nextflight page.html --all > everything.json
89
+ ```
90
+
91
+ ### In a Scrapy / Zyte spider
92
+
93
+ ```python
94
+ import scrapy
95
+ from nextflight import extract
96
+
97
+ class MySpider(scrapy.Spider):
98
+ name = "my_spider"
99
+
100
+ def parse(self, response):
101
+ page = extract(response.text)
102
+
103
+ items = page.find_all(
104
+ lambda n: isinstance(n, dict) and "price" in n and "title" in n
105
+ )
106
+ for item in items:
107
+ yield {
108
+ "title": item.get("title"),
109
+ "price": item.get("price"),
110
+ "url": response.url,
111
+ }
112
+ ```
113
+
114
+ ### Fetching a URL directly (no Scrapy needed)
115
+
116
+ ```python
117
+ from nextflight import FlightExtractor
118
+
119
+ page = FlightExtractor.from_url("https://example.com/product/123")
120
+ product = page.find_by_keys({"price", "title"})
121
+ ```
122
+
123
+ (`from_url` uses only the stdlib for quick one-off exploration. For
124
+ production crawling — retries, proxies, JS rendering, robots.txt — fetch
125
+ the page with your own HTTP client / Scrapy / Zyte and pass
126
+ `response.text` to `FlightExtractor(...)` / `extract(...)` instead.)
127
+
128
+ ## API
129
+
130
+ - **`extract(html) -> FlightExtractor`** — shorthand constructor. `html`
131
+ accepts a plain string, bytes, or a response-like object (Scrapy's
132
+ `Response`, `requests.Response`, etc.) — pass `response` straight from a
133
+ `parse()` method without writing `response.text` yourself.
134
+ - **`FlightExtractor(html, *, strict: bool = False)`**
135
+ - `.keys() -> list[str]` — every chunk id found on the page, in order.
136
+ - `page["3f"]` / `.resolve_chunk("3f")` — the resolved JSON for one
137
+ specific chunk id (`page[...]` raises `KeyError` if it doesn't exist;
138
+ `resolve_chunk` returns `None`). `"3f" in page` and `for k in page`
139
+ also work, like a dict.
140
+ - `.resolve_all() -> dict` — every chunk, fully dereferenced.
141
+ - `.find_all(predicate, root=None, max_results=None) -> list` — walk the
142
+ resolved tree and collect every node matching `predicate`.
143
+ - `.find_one(predicate, root=None) -> Any | None`
144
+ - `.find_by_keys(required_keys, root=None) -> dict | None` — find the
145
+ first dict containing all of `required_keys`.
146
+ - `.find_all_by_keys(required_keys, root=None) -> list` — like
147
+ `find_by_keys` but returns every match, for pages with repeated
148
+ cards/listings that share the same shape.
149
+ - `.find_by_type(type_value, key="@type", root=None) -> list` — find
150
+ every dict whose `key` field equals `type_value`.
151
+ - `.find_text(pattern, root=None) -> list` — regex-search every string
152
+ value on the page and return the distinct whole values that contain a
153
+ match (emails, prices, phone numbers, SKUs, ...) without needing to
154
+ know which object they live on.
155
+ - `.get("path.to.value", default=None) -> Any` — tolerant dotted-path
156
+ lookup into the resolved page (dict keys and/or list indices), once
157
+ you already know roughly where something lives on this site.
158
+ - `.stats() -> dict` — quick diagnostic snapshot (chunk count, ids,
159
+ value type counts, page size) for exploring a new site.
160
+ - `.to_json(path=None, indent=2) -> str | None` — dump the fully
161
+ resolved page to a file, or return it as a JSON string.
162
+ - `.from_url(url, timeout=15.0, headers=None) -> FlightExtractor`
163
+ (classmethod) — fetch and parse a URL using only the stdlib.
164
+ - `strict=True` raises `FlightParseError` on a row that's neither valid
165
+ JSON nor a recognizable `$`-reference marker, instead of silently
166
+ keeping it as a raw string (useful while developing a new scraper;
167
+ leave off in production so a handful of odd rows never take down
168
+ extraction of everything else on the page).
169
+ - **`find_json_ld(html, type_=None) -> list`** — parse any
170
+ `<script type="application/ld+json">` blocks on the page, optionally
171
+ filtered by `@type`. Also accepts response-like objects.
172
+ - **CLI**: `nextflight <file-or-url> [--keys a,b | --all-by-keys a,b | --type Product | --text PATTERN | --get path.to.value | --stats | --all] [--save out.json]`
173
+
174
+ No runtime dependencies — stdlib only (`json`, `re`, `urllib`, `argparse`)
175
+ — so it's safe to drop into any existing Scrapy/Zyte project without
176
+ touching the rest of your dependency tree.
177
+
178
+ ### Upgrading from `nextjs-flight-extractor` / `NextFlightExtractor`
179
+
180
+ The old names still work but emit a `DeprecationWarning`:
181
+
182
+ | Old (0.1.x) | New (0.2.x+) |
183
+ |---------------------------------------|----------------------------------|
184
+ | `from nextjs_flight_extractor import NextFlightExtractor` | `from nextflight import FlightExtractor` |
185
+ | `extractor.find_first(...)` | `page.find_one(...)` |
186
+ | `extract_json_ld(html, schema_type=…)`| `find_json_ld(html, type_=…)` |
187
+
188
+ ## Why not just `str.split('\n')`?
189
+
190
+ Two of the Flight row kinds break that assumption:
191
+
192
+ - **Text rows** (`id:T<hexByteLen>,<raw text>`) are byte-length-prefixed
193
+ blobs, not newline-terminated, and can contain literal newlines or run
194
+ directly into the next row's id with zero separator.
195
+ - **Module / preload rows** (`id:I[...]` / `:HL[...]`) need bracket-aware
196
+ parsing.
197
+
198
+ `nextflight` implements the real row grammar, quote/escape aware, so it
199
+ holds up on both well-formed and truncated payloads (e.g. from a proxy
200
+ that cuts a response off mid-chunk).
201
+
202
+ ## Building / publishing
203
+
204
+ ### Manual (twine)
205
+
206
+ ```bash
207
+ pip install build twine
208
+ python -m build # produces dist/*.whl and dist/*.tar.gz
209
+ twine check dist/* # validate metadata before uploading
210
+ twine upload dist/* # publish to PyPI (or use --repository testpypi for a dry run)
211
+ ```
212
+
213
+ ### Automatic (GitHub Actions + PyPI Trusted Publishing)
214
+
215
+ This repo ships with `.github/workflows/ci.yml`, which:
216
+ - runs the test suite on every push/PR across Python 3.9–3.12,
217
+ - builds and validates the sdist/wheel,
218
+ - publishes to PyPI automatically whenever a tag like `v0.2.1` is pushed.
219
+
220
+ Publishing uses PyPI's **Trusted Publisher** flow — no API token stored in
221
+ GitHub secrets. One-time setup:
222
+
223
+ 1. On [pypi.org](https://pypi.org), go to your project → *Publishing* →
224
+ *Add a new publisher* (or, for a brand-new project name, do this from
225
+ your PyPI account's "Trusted Publishers" management page before the
226
+ project exists yet).
227
+ 2. Fill in: Owner = your GitHub username/org, Repository = this repo's
228
+ name, Workflow name = `ci.yml`, Environment name = `pypi`.
229
+ 3. In your GitHub repo, go to *Settings → Environments*, create an
230
+ environment named `pypi` (optionally require a manual approval before
231
+ deploys, for extra safety).
232
+ 4. Release a new version:
233
+ ```bash
234
+ # bump version in pyproject.toml and src/nextflight/__init__.py first
235
+ git commit -am "Release v0.2.2"
236
+ git tag v0.2.2
237
+ git push origin main --tags
238
+ ```
239
+ The workflow builds, tests, and publishes automatically.
240
+
241
+
242
+ ## License
243
+
244
+ MIT
@@ -0,0 +1,221 @@
1
+ # nextflight
2
+
3
+ A general-purpose parser for the data Next.js (App Router) embeds in
4
+ `<script>self.__next_f.push([...])</script>` tags — the React Server
5
+ Components "Flight" wire format. Works on **any** Next.js 13+ App Router
6
+ site, not just one particular project.
7
+
8
+ Instead of hardcoding array indices like `data[3]["children"][0][3]...`,
9
+ which break the moment a site's component tree reshuffles on redeploy,
10
+ `nextflight` resolves the `$`-sigil references Next.js uses internally
11
+ and lets you *search* for the shape of data you want.
12
+
13
+ ## Install
14
+
15
+ ```bash
16
+ pip install nextflight
17
+ ```
18
+
19
+ ## Quick start
20
+
21
+ The core workflow is two steps: hand it any HTML, see what keys are on
22
+ the page, then fetch the resolved JSON for whichever key you want.
23
+
24
+ ```python
25
+ from nextflight import extract
26
+
27
+ # Step 1: send any HTML, get the list of keys (one per __next_f.push chunk)
28
+ page = extract(html_text)
29
+ print(page.keys()) # e.g. ['0', '1', '3f', '20', ...]
30
+
31
+ # Step 2: fetch the resolved JSON for a specific key
32
+ data = page["3f"] # same as page.resolve_chunk("3f")
33
+ ```
34
+
35
+ Chunk ids are arbitrary per build though (a redeploy can renumber them),
36
+ so in practice you'll usually skip straight to *searching* for the shape
37
+ of data you want instead of a specific id:
38
+
39
+ ```python
40
+ from nextflight import extract
41
+
42
+ page = extract(html_text)
43
+
44
+ # Find the first object anywhere in the page that has all of these keys,
45
+ # wherever this build's component tree happened to put it:
46
+ listing = page.find_by_keys({"sections", "meta"})
47
+
48
+ # Find every node with a given @type (or any custom key):
49
+ products = page.find_by_type("Product")
50
+
51
+ # Or search with a fully custom predicate:
52
+ items = page.find_all(lambda n: isinstance(n, dict) and "price" in n)
53
+
54
+ # Or grab everything, fully dereferenced, and inspect by hand:
55
+ everything = page.resolve_all()
56
+ ```
57
+
58
+ ### Command line
59
+
60
+ For quick, no-script exploration of a page you've already saved (or a live URL):
61
+
62
+ ```bash
63
+ nextflight page.html --keys sections,meta
64
+ nextflight https://example.com/product/123 --type Product
65
+ nextflight page.html --all > everything.json
66
+ ```
67
+
68
+ ### In a Scrapy / Zyte spider
69
+
70
+ ```python
71
+ import scrapy
72
+ from nextflight import extract
73
+
74
+ class MySpider(scrapy.Spider):
75
+ name = "my_spider"
76
+
77
+ def parse(self, response):
78
+ page = extract(response.text)
79
+
80
+ items = page.find_all(
81
+ lambda n: isinstance(n, dict) and "price" in n and "title" in n
82
+ )
83
+ for item in items:
84
+ yield {
85
+ "title": item.get("title"),
86
+ "price": item.get("price"),
87
+ "url": response.url,
88
+ }
89
+ ```
90
+
91
+ ### Fetching a URL directly (no Scrapy needed)
92
+
93
+ ```python
94
+ from nextflight import FlightExtractor
95
+
96
+ page = FlightExtractor.from_url("https://example.com/product/123")
97
+ product = page.find_by_keys({"price", "title"})
98
+ ```
99
+
100
+ (`from_url` uses only the stdlib for quick one-off exploration. For
101
+ production crawling — retries, proxies, JS rendering, robots.txt — fetch
102
+ the page with your own HTTP client / Scrapy / Zyte and pass
103
+ `response.text` to `FlightExtractor(...)` / `extract(...)` instead.)
104
+
105
+ ## API
106
+
107
+ - **`extract(html) -> FlightExtractor`** — shorthand constructor. `html`
108
+ accepts a plain string, bytes, or a response-like object (Scrapy's
109
+ `Response`, `requests.Response`, etc.) — pass `response` straight from a
110
+ `parse()` method without writing `response.text` yourself.
111
+ - **`FlightExtractor(html, *, strict: bool = False)`**
112
+ - `.keys() -> list[str]` — every chunk id found on the page, in order.
113
+ - `page["3f"]` / `.resolve_chunk("3f")` — the resolved JSON for one
114
+ specific chunk id (`page[...]` raises `KeyError` if it doesn't exist;
115
+ `resolve_chunk` returns `None`). `"3f" in page` and `for k in page`
116
+ also work, like a dict.
117
+ - `.resolve_all() -> dict` — every chunk, fully dereferenced.
118
+ - `.find_all(predicate, root=None, max_results=None) -> list` — walk the
119
+ resolved tree and collect every node matching `predicate`.
120
+ - `.find_one(predicate, root=None) -> Any | None`
121
+ - `.find_by_keys(required_keys, root=None) -> dict | None` — find the
122
+ first dict containing all of `required_keys`.
123
+ - `.find_all_by_keys(required_keys, root=None) -> list` — like
124
+ `find_by_keys` but returns every match, for pages with repeated
125
+ cards/listings that share the same shape.
126
+ - `.find_by_type(type_value, key="@type", root=None) -> list` — find
127
+ every dict whose `key` field equals `type_value`.
128
+ - `.find_text(pattern, root=None) -> list` — regex-search every string
129
+ value on the page and return the distinct whole values that contain a
130
+ match (emails, prices, phone numbers, SKUs, ...) without needing to
131
+ know which object they live on.
132
+ - `.get("path.to.value", default=None) -> Any` — tolerant dotted-path
133
+ lookup into the resolved page (dict keys and/or list indices), once
134
+ you already know roughly where something lives on this site.
135
+ - `.stats() -> dict` — quick diagnostic snapshot (chunk count, ids,
136
+ value type counts, page size) for exploring a new site.
137
+ - `.to_json(path=None, indent=2) -> str | None` — dump the fully
138
+ resolved page to a file, or return it as a JSON string.
139
+ - `.from_url(url, timeout=15.0, headers=None) -> FlightExtractor`
140
+ (classmethod) — fetch and parse a URL using only the stdlib.
141
+ - `strict=True` raises `FlightParseError` on a row that's neither valid
142
+ JSON nor a recognizable `$`-reference marker, instead of silently
143
+ keeping it as a raw string (useful while developing a new scraper;
144
+ leave off in production so a handful of odd rows never take down
145
+ extraction of everything else on the page).
146
+ - **`find_json_ld(html, type_=None) -> list`** — parse any
147
+ `<script type="application/ld+json">` blocks on the page, optionally
148
+ filtered by `@type`. Also accepts response-like objects.
149
+ - **CLI**: `nextflight <file-or-url> [--keys a,b | --all-by-keys a,b | --type Product | --text PATTERN | --get path.to.value | --stats | --all] [--save out.json]`
150
+
151
+ No runtime dependencies — stdlib only (`json`, `re`, `urllib`, `argparse`)
152
+ — so it's safe to drop into any existing Scrapy/Zyte project without
153
+ touching the rest of your dependency tree.
154
+
155
+ ### Upgrading from `nextjs-flight-extractor` / `NextFlightExtractor`
156
+
157
+ The old names still work but emit a `DeprecationWarning`:
158
+
159
+ | Old (0.1.x) | New (0.2.x+) |
160
+ |---------------------------------------|----------------------------------|
161
+ | `from nextjs_flight_extractor import NextFlightExtractor` | `from nextflight import FlightExtractor` |
162
+ | `extractor.find_first(...)` | `page.find_one(...)` |
163
+ | `extract_json_ld(html, schema_type=…)`| `find_json_ld(html, type_=…)` |
164
+
165
+ ## Why not just `str.split('\n')`?
166
+
167
+ Two of the Flight row kinds break that assumption:
168
+
169
+ - **Text rows** (`id:T<hexByteLen>,<raw text>`) are byte-length-prefixed
170
+ blobs, not newline-terminated, and can contain literal newlines or run
171
+ directly into the next row's id with zero separator.
172
+ - **Module / preload rows** (`id:I[...]` / `:HL[...]`) need bracket-aware
173
+ parsing.
174
+
175
+ `nextflight` implements the real row grammar, quote/escape aware, so it
176
+ holds up on both well-formed and truncated payloads (e.g. from a proxy
177
+ that cuts a response off mid-chunk).
178
+
179
+ ## Building / publishing
180
+
181
+ ### Manual (twine)
182
+
183
+ ```bash
184
+ pip install build twine
185
+ python -m build # produces dist/*.whl and dist/*.tar.gz
186
+ twine check dist/* # validate metadata before uploading
187
+ twine upload dist/* # publish to PyPI (or use --repository testpypi for a dry run)
188
+ ```
189
+
190
+ ### Automatic (GitHub Actions + PyPI Trusted Publishing)
191
+
192
+ This repo ships with `.github/workflows/ci.yml`, which:
193
+ - runs the test suite on every push/PR across Python 3.9–3.12,
194
+ - builds and validates the sdist/wheel,
195
+ - publishes to PyPI automatically whenever a tag like `v0.2.1` is pushed.
196
+
197
+ Publishing uses PyPI's **Trusted Publisher** flow — no API token stored in
198
+ GitHub secrets. One-time setup:
199
+
200
+ 1. On [pypi.org](https://pypi.org), go to your project → *Publishing* →
201
+ *Add a new publisher* (or, for a brand-new project name, do this from
202
+ your PyPI account's "Trusted Publishers" management page before the
203
+ project exists yet).
204
+ 2. Fill in: Owner = your GitHub username/org, Repository = this repo's
205
+ name, Workflow name = `ci.yml`, Environment name = `pypi`.
206
+ 3. In your GitHub repo, go to *Settings → Environments*, create an
207
+ environment named `pypi` (optionally require a manual approval before
208
+ deploys, for extra safety).
209
+ 4. Release a new version:
210
+ ```bash
211
+ # bump version in pyproject.toml and src/nextflight/__init__.py first
212
+ git commit -am "Release v0.2.2"
213
+ git tag v0.2.2
214
+ git push origin main --tags
215
+ ```
216
+ The workflow builds, tests, and publishes automatically.
217
+
218
+
219
+ ## License
220
+
221
+ MIT
@@ -0,0 +1,39 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "nextflight"
7
+ version = "0.3.0"
8
+ description = "Parse Next.js App Router 'Flight' (__next_f.push) payloads embedded in server-rendered HTML -- works on any Next.js 13+ site."
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = { text = "MIT" }
12
+ authors = [
13
+ { name = "Aly Reda" }
14
+ ]
15
+ keywords = ["nextjs", "scraping", "scrapy", "flight", "rsc", "react-server-components", "web-scraping", "zyte"]
16
+ classifiers = [
17
+ "Programming Language :: Python :: 3",
18
+ "Programming Language :: Python :: 3.9",
19
+ "Programming Language :: Python :: 3.10",
20
+ "Programming Language :: Python :: 3.11",
21
+ "Programming Language :: Python :: 3.12",
22
+ "License :: OSI Approved :: MIT License",
23
+ "Operating System :: OS Independent",
24
+ "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
25
+ "Topic :: Software Development :: Libraries :: Python Modules",
26
+ ]
27
+ # Stdlib only -- no runtime dependencies, so it's safe to drop into any
28
+ # Scrapy / Zyte project without touching the rest of your dependency tree.
29
+ dependencies = []
30
+
31
+ [project.urls]
32
+ Homepage = "https://github.com/Aly-Reda/nextflight"
33
+ Issues = "https://github.com/Aly-Reda/nextflight/issues"
34
+
35
+ [project.scripts]
36
+ nextflight = "nextflight.cli:main"
37
+
38
+ [tool.setuptools.packages.find]
39
+ where = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,68 @@
1
+ """
2
+ nextflight
3
+ ==========
4
+
5
+ Generic parser for the React Server Components "Flight" wire format that
6
+ Next.js 13+ (App Router) embeds in ``<script>self.__next_f.push([...])</script>``
7
+ tags. Works on any Next.js App Router site.
8
+
9
+ from nextflight import extract
10
+
11
+ page = extract(html_text)
12
+ listing = page.find_by_keys({"sections", "meta"})
13
+ """
14
+
15
+ import warnings
16
+
17
+ from .extractor import (
18
+ FlightExtractor,
19
+ FlightParseError,
20
+ extract,
21
+ find_json_ld,
22
+ )
23
+
24
+ __all__ = [
25
+ "FlightExtractor",
26
+ "FlightParseError",
27
+ "extract",
28
+ "find_json_ld",
29
+ "NextFlightExtractor", # deprecated alias, see below
30
+ "extract_json_ld", # deprecated alias, see below
31
+ ]
32
+
33
+ __version__ = "0.3.0"
34
+
35
+
36
+ # ---------------------------------------------------------------------- #
37
+ # Backwards-compatible aliases for the pre-rename API (nextjs_flight_extractor
38
+ # 0.1.x). These will be removed in a future major version -- switch to
39
+ # FlightExtractor / find_json_ld when convenient.
40
+ # ---------------------------------------------------------------------- #
41
+ class NextFlightExtractor(FlightExtractor):
42
+ """Deprecated alias for :class:`FlightExtractor`. Use ``FlightExtractor`` instead."""
43
+
44
+ def __init__(self, *args, **kwargs):
45
+ warnings.warn(
46
+ "NextFlightExtractor is deprecated, use nextflight.FlightExtractor instead",
47
+ DeprecationWarning,
48
+ stacklevel=2,
49
+ )
50
+ super().__init__(*args, **kwargs)
51
+
52
+ def find_first(self, *args, **kwargs):
53
+ warnings.warn(
54
+ "find_first() is deprecated, use find_one() instead",
55
+ DeprecationWarning,
56
+ stacklevel=2,
57
+ )
58
+ return self.find_one(*args, **kwargs)
59
+
60
+
61
+ def extract_json_ld(html: str, schema_type=None) -> list:
62
+ """Deprecated alias for :func:`find_json_ld`. Use ``find_json_ld`` instead."""
63
+ warnings.warn(
64
+ "extract_json_ld() is deprecated, use nextflight.find_json_ld() instead",
65
+ DeprecationWarning,
66
+ stacklevel=2,
67
+ )
68
+ return find_json_ld(html, type_=schema_type)
@@ -0,0 +1,86 @@
1
+ """
2
+ Command-line entry point for quick exploration:
3
+
4
+ nextflight page.html --keys sections,meta
5
+ nextflight https://example.com/product/123 --keys price,title
6
+ nextflight page.html --all > everything.json
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import argparse
12
+ import json
13
+ import sys
14
+
15
+ from .extractor import FlightExtractor
16
+
17
+
18
+ def _load_html(source: str) -> str:
19
+ if source.startswith("http://") or source.startswith("https://"):
20
+ extractor = FlightExtractor.from_url(source)
21
+ return extractor.html
22
+ with open(source, "r", encoding="utf-8") as f:
23
+ return f.read()
24
+
25
+
26
+ def main(argv=None) -> int:
27
+ parser = argparse.ArgumentParser(
28
+ prog="nextflight",
29
+ description="Extract Next.js Flight (__next_f.push) data from a page.",
30
+ )
31
+ parser.add_argument("source", help="Path to an HTML file, or a URL to fetch.")
32
+ parser.add_argument(
33
+ "--keys", help="Comma-separated keys: find the first object containing all of them."
34
+ )
35
+ parser.add_argument(
36
+ "--all-by-keys", dest="all_by_keys",
37
+ help="Comma-separated keys: find EVERY object containing all of them (not just the first).",
38
+ )
39
+ parser.add_argument(
40
+ "--type", dest="type_value",
41
+ help='Find every object whose "@type" (or --type-key) equals this value.',
42
+ )
43
+ parser.add_argument("--type-key", default="@type", help='Key to match --type against (default "@type").')
44
+ parser.add_argument("--text", dest="text_pattern", help="Regex: list every distinct string value on the page that matches it.")
45
+ parser.add_argument("--get", dest="get_path", help='Dotted path into the resolved page, e.g. "3f.props.price".')
46
+ parser.add_argument("--all", action="store_true", help="Dump every resolved chunk.")
47
+ parser.add_argument("--stats", action="store_true", help="Print a quick diagnostic summary instead of data.")
48
+ parser.add_argument("--save", dest="save_path", help="Write output to this file instead of stdout.")
49
+ parser.add_argument("--indent", type=int, default=2, help="JSON indent for output (default 2).")
50
+ args = parser.parse_args(argv)
51
+
52
+ html = _load_html(args.source)
53
+ extractor = FlightExtractor(html)
54
+
55
+ if args.keys:
56
+ keys = {k.strip() for k in args.keys.split(",") if k.strip()}
57
+ result = extractor.find_by_keys(keys)
58
+ elif args.all_by_keys:
59
+ keys = {k.strip() for k in args.all_by_keys.split(",") if k.strip()}
60
+ result = extractor.find_all_by_keys(keys)
61
+ elif args.type_value:
62
+ result = extractor.find_by_type(args.type_value, key=args.type_key)
63
+ elif args.text_pattern:
64
+ result = extractor.find_text(args.text_pattern)
65
+ elif args.get_path:
66
+ result = extractor.get(args.get_path)
67
+ elif args.stats:
68
+ result = extractor.stats()
69
+ elif args.all:
70
+ result = extractor.resolve_all()
71
+ else:
72
+ parser.error("Provide one of --keys, --all-by-keys, --type, --text, --get, --stats, or --all.")
73
+ return 2
74
+
75
+ output = json.dumps(result, indent=args.indent, ensure_ascii=False, default=str)
76
+ if args.save_path:
77
+ with open(args.save_path, "w", encoding="utf-8") as f:
78
+ f.write(output)
79
+ else:
80
+ sys.stdout.write(output)
81
+ sys.stdout.write("\n")
82
+ return 0
83
+
84
+
85
+ if __name__ == "__main__":
86
+ raise SystemExit(main())