nextflight 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nextflight-0.3.0/LICENSE +21 -0
- nextflight-0.3.0/PKG-INFO +244 -0
- nextflight-0.3.0/README.md +221 -0
- nextflight-0.3.0/pyproject.toml +39 -0
- nextflight-0.3.0/setup.cfg +4 -0
- nextflight-0.3.0/src/nextflight/__init__.py +68 -0
- nextflight-0.3.0/src/nextflight/cli.py +86 -0
- nextflight-0.3.0/src/nextflight/extractor.py +556 -0
- nextflight-0.3.0/src/nextflight.egg-info/PKG-INFO +244 -0
- nextflight-0.3.0/src/nextflight.egg-info/SOURCES.txt +12 -0
- nextflight-0.3.0/src/nextflight.egg-info/dependency_links.txt +1 -0
- nextflight-0.3.0/src/nextflight.egg-info/entry_points.txt +2 -0
- nextflight-0.3.0/src/nextflight.egg-info/top_level.txt +1 -0
- nextflight-0.3.0/tests/test_extractor.py +312 -0
nextflight-0.3.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Aly Reda
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nextflight
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Parse Next.js App Router 'Flight' (__next_f.push) payloads embedded in server-rendered HTML -- works on any Next.js 13+ site.
|
|
5
|
+
Author: Aly Reda
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/Aly-Reda/nextflight
|
|
8
|
+
Project-URL: Issues, https://github.com/Aly-Reda/nextflight/issues
|
|
9
|
+
Keywords: nextjs,scraping,scrapy,flight,rsc,react-server-components,web-scraping,zyte
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
18
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
19
|
+
Requires-Python: >=3.9
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
License-File: LICENSE
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# nextflight
|
|
25
|
+
|
|
26
|
+
A general-purpose parser for the data Next.js (App Router) embeds in
|
|
27
|
+
`<script>self.__next_f.push([...])</script>` tags — the React Server
|
|
28
|
+
Components "Flight" wire format. Works on **any** Next.js 13+ App Router
|
|
29
|
+
site, not just one particular project.
|
|
30
|
+
|
|
31
|
+
Instead of hardcoding array indices like `data[3]["children"][0][3]...`,
|
|
32
|
+
which break the moment a site's component tree reshuffles on redeploy,
|
|
33
|
+
`nextflight` resolves the `$`-sigil references Next.js uses internally
|
|
34
|
+
and lets you *search* for the shape of data you want.
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
pip install nextflight
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Quick start
|
|
43
|
+
|
|
44
|
+
The core workflow is two steps: hand it any HTML, see what keys are on
|
|
45
|
+
the page, then fetch the resolved JSON for whichever key you want.
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
from nextflight import extract
|
|
49
|
+
|
|
50
|
+
# Step 1: send any HTML, get the list of keys (one per __next_f.push chunk)
|
|
51
|
+
page = extract(html_text)
|
|
52
|
+
print(page.keys()) # e.g. ['0', '1', '3f', '20', ...]
|
|
53
|
+
|
|
54
|
+
# Step 2: fetch the resolved JSON for a specific key
|
|
55
|
+
data = page["3f"] # same as page.resolve_chunk("3f")
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Chunk ids are arbitrary per build though (a redeploy can renumber them),
|
|
59
|
+
so in practice you'll usually skip straight to *searching* for the shape
|
|
60
|
+
of data you want instead of a specific id:
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
from nextflight import extract
|
|
64
|
+
|
|
65
|
+
page = extract(html_text)
|
|
66
|
+
|
|
67
|
+
# Find the first object anywhere in the page that has all of these keys,
|
|
68
|
+
# wherever this build's component tree happened to put it:
|
|
69
|
+
listing = page.find_by_keys({"sections", "meta"})
|
|
70
|
+
|
|
71
|
+
# Find every node with a given @type (or any custom key):
|
|
72
|
+
products = page.find_by_type("Product")
|
|
73
|
+
|
|
74
|
+
# Or search with a fully custom predicate:
|
|
75
|
+
items = page.find_all(lambda n: isinstance(n, dict) and "price" in n)
|
|
76
|
+
|
|
77
|
+
# Or grab everything, fully dereferenced, and inspect by hand:
|
|
78
|
+
everything = page.resolve_all()
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
### Command line
|
|
82
|
+
|
|
83
|
+
For quick, no-script exploration of a page you've already saved (or a live URL):
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
nextflight page.html --keys sections,meta
|
|
87
|
+
nextflight https://example.com/product/123 --type Product
|
|
88
|
+
nextflight page.html --all > everything.json
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### In a Scrapy / Zyte spider
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
import scrapy
|
|
95
|
+
from nextflight import extract
|
|
96
|
+
|
|
97
|
+
class MySpider(scrapy.Spider):
|
|
98
|
+
name = "my_spider"
|
|
99
|
+
|
|
100
|
+
def parse(self, response):
|
|
101
|
+
page = extract(response.text)
|
|
102
|
+
|
|
103
|
+
items = page.find_all(
|
|
104
|
+
lambda n: isinstance(n, dict) and "price" in n and "title" in n
|
|
105
|
+
)
|
|
106
|
+
for item in items:
|
|
107
|
+
yield {
|
|
108
|
+
"title": item.get("title"),
|
|
109
|
+
"price": item.get("price"),
|
|
110
|
+
"url": response.url,
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### Fetching a URL directly (no Scrapy needed)
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
from nextflight import FlightExtractor
|
|
118
|
+
|
|
119
|
+
page = FlightExtractor.from_url("https://example.com/product/123")
|
|
120
|
+
product = page.find_by_keys({"price", "title"})
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
(`from_url` uses only the stdlib for quick one-off exploration. For
|
|
124
|
+
production crawling — retries, proxies, JS rendering, robots.txt — fetch
|
|
125
|
+
the page with your own HTTP client / Scrapy / Zyte and pass
|
|
126
|
+
`response.text` to `FlightExtractor(...)` / `extract(...)` instead.)
|
|
127
|
+
|
|
128
|
+
## API
|
|
129
|
+
|
|
130
|
+
- **`extract(html) -> FlightExtractor`** — shorthand constructor. `html`
|
|
131
|
+
accepts a plain string, bytes, or a response-like object (Scrapy's
|
|
132
|
+
`Response`, `requests.Response`, etc.) — pass `response` straight from a
|
|
133
|
+
`parse()` method without writing `response.text` yourself.
|
|
134
|
+
- **`FlightExtractor(html, *, strict: bool = False)`**
|
|
135
|
+
- `.keys() -> list[str]` — every chunk id found on the page, in order.
|
|
136
|
+
- `page["3f"]` / `.resolve_chunk("3f")` — the resolved JSON for one
|
|
137
|
+
specific chunk id (`page[...]` raises `KeyError` if it doesn't exist;
|
|
138
|
+
`resolve_chunk` returns `None`). `"3f" in page` and `for k in page`
|
|
139
|
+
also work, like a dict.
|
|
140
|
+
- `.resolve_all() -> dict` — every chunk, fully dereferenced.
|
|
141
|
+
- `.find_all(predicate, root=None, max_results=None) -> list` — walk the
|
|
142
|
+
resolved tree and collect every node matching `predicate`.
|
|
143
|
+
- `.find_one(predicate, root=None) -> Any | None`
|
|
144
|
+
- `.find_by_keys(required_keys, root=None) -> dict | None` — find the
|
|
145
|
+
first dict containing all of `required_keys`.
|
|
146
|
+
- `.find_all_by_keys(required_keys, root=None) -> list` — like
|
|
147
|
+
`find_by_keys` but returns every match, for pages with repeated
|
|
148
|
+
cards/listings that share the same shape.
|
|
149
|
+
- `.find_by_type(type_value, key="@type", root=None) -> list` — find
|
|
150
|
+
every dict whose `key` field equals `type_value`.
|
|
151
|
+
- `.find_text(pattern, root=None) -> list` — regex-search every string
|
|
152
|
+
value on the page and return the distinct whole values that contain a
|
|
153
|
+
match (emails, prices, phone numbers, SKUs, ...) without needing to
|
|
154
|
+
know which object they live on.
|
|
155
|
+
- `.get("path.to.value", default=None) -> Any` — tolerant dotted-path
|
|
156
|
+
lookup into the resolved page (dict keys and/or list indices), once
|
|
157
|
+
you already know roughly where something lives on this site.
|
|
158
|
+
- `.stats() -> dict` — quick diagnostic snapshot (chunk count, ids,
|
|
159
|
+
value type counts, page size) for exploring a new site.
|
|
160
|
+
- `.to_json(path=None, indent=2) -> str | None` — dump the fully
|
|
161
|
+
resolved page to a file, or return it as a JSON string.
|
|
162
|
+
- `.from_url(url, timeout=15.0, headers=None) -> FlightExtractor`
|
|
163
|
+
(classmethod) — fetch and parse a URL using only the stdlib.
|
|
164
|
+
- `strict=True` raises `FlightParseError` on a row that's neither valid
|
|
165
|
+
JSON nor a recognizable `$`-reference marker, instead of silently
|
|
166
|
+
keeping it as a raw string (useful while developing a new scraper;
|
|
167
|
+
leave off in production so a handful of odd rows never take down
|
|
168
|
+
extraction of everything else on the page).
|
|
169
|
+
- **`find_json_ld(html, type_=None) -> list`** — parse any
|
|
170
|
+
`<script type="application/ld+json">` blocks on the page, optionally
|
|
171
|
+
filtered by `@type`. Also accepts response-like objects.
|
|
172
|
+
- **CLI**: `nextflight <file-or-url> [--keys a,b | --all-by-keys a,b | --type Product | --text PATTERN | --get path.to.value | --stats | --all] [--save out.json]`
|
|
173
|
+
|
|
174
|
+
No runtime dependencies — stdlib only (`json`, `re`, `urllib`, `argparse`)
|
|
175
|
+
— so it's safe to drop into any existing Scrapy/Zyte project without
|
|
176
|
+
touching the rest of your dependency tree.
|
|
177
|
+
|
|
178
|
+
### Upgrading from `nextjs-flight-extractor` / `NextFlightExtractor`
|
|
179
|
+
|
|
180
|
+
The old names still work but emit a `DeprecationWarning`:
|
|
181
|
+
|
|
182
|
+
| Old (0.1.x) | New (0.2.x+) |
|
|
183
|
+
|---------------------------------------|----------------------------------|
|
|
184
|
+
| `from nextjs_flight_extractor import NextFlightExtractor` | `from nextflight import FlightExtractor` |
|
|
185
|
+
| `extractor.find_first(...)` | `page.find_one(...)` |
|
|
186
|
+
| `extract_json_ld(html, schema_type=…)`| `find_json_ld(html, type_=…)` |
|
|
187
|
+
|
|
188
|
+
## Why not just `str.split('\n')`?
|
|
189
|
+
|
|
190
|
+
Two of the Flight row kinds break that assumption:
|
|
191
|
+
|
|
192
|
+
- **Text rows** (`id:T<hexByteLen>,<raw text>`) are byte-length-prefixed
|
|
193
|
+
blobs, not newline-terminated, and can contain literal newlines or run
|
|
194
|
+
directly into the next row's id with zero separator.
|
|
195
|
+
- **Module / preload rows** (`id:I[...]` / `:HL[...]`) need bracket-aware
|
|
196
|
+
parsing.
|
|
197
|
+
|
|
198
|
+
`nextflight` implements the real row grammar, quote/escape aware, so it
|
|
199
|
+
holds up on both well-formed and truncated payloads (e.g. from a proxy
|
|
200
|
+
that cuts a response off mid-chunk).
|
|
201
|
+
|
|
202
|
+
## Building / publishing
|
|
203
|
+
|
|
204
|
+
### Manual (twine)
|
|
205
|
+
|
|
206
|
+
```bash
|
|
207
|
+
pip install build twine
|
|
208
|
+
python -m build # produces dist/*.whl and dist/*.tar.gz
|
|
209
|
+
twine check dist/* # validate metadata before uploading
|
|
210
|
+
twine upload dist/* # publish to PyPI (or use --repository testpypi for a dry run)
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
### Automatic (GitHub Actions + PyPI Trusted Publishing)
|
|
214
|
+
|
|
215
|
+
This repo ships with `.github/workflows/ci.yml`, which:
|
|
216
|
+
- runs the test suite on every push/PR across Python 3.9–3.12,
|
|
217
|
+
- builds and validates the sdist/wheel,
|
|
218
|
+
- publishes to PyPI automatically whenever a tag like `v0.2.1` is pushed.
|
|
219
|
+
|
|
220
|
+
Publishing uses PyPI's **Trusted Publisher** flow — no API token stored in
|
|
221
|
+
GitHub secrets. One-time setup:
|
|
222
|
+
|
|
223
|
+
1. On [pypi.org](https://pypi.org), go to your project → *Publishing* →
|
|
224
|
+
*Add a new publisher* (or, for a brand-new project name, do this from
|
|
225
|
+
your PyPI account's "Trusted Publishers" management page before the
|
|
226
|
+
project exists yet).
|
|
227
|
+
2. Fill in: Owner = your GitHub username/org, Repository = this repo's
|
|
228
|
+
name, Workflow name = `ci.yml`, Environment name = `pypi`.
|
|
229
|
+
3. In your GitHub repo, go to *Settings → Environments*, create an
|
|
230
|
+
environment named `pypi` (optionally require a manual approval before
|
|
231
|
+
deploys, for extra safety).
|
|
232
|
+
4. Release a new version:
|
|
233
|
+
```bash
|
|
234
|
+
# bump version in pyproject.toml and src/nextflight/__init__.py first
|
|
235
|
+
git commit -am "Release v0.2.2"
|
|
236
|
+
git tag v0.2.2
|
|
237
|
+
git push origin main --tags
|
|
238
|
+
```
|
|
239
|
+
The workflow builds, tests, and publishes automatically.
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
## License
|
|
243
|
+
|
|
244
|
+
MIT
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
# nextflight
|
|
2
|
+
|
|
3
|
+
A general-purpose parser for the data Next.js (App Router) embeds in
|
|
4
|
+
`<script>self.__next_f.push([...])</script>` tags — the React Server
|
|
5
|
+
Components "Flight" wire format. Works on **any** Next.js 13+ App Router
|
|
6
|
+
site, not just one particular project.
|
|
7
|
+
|
|
8
|
+
Instead of hardcoding array indices like `data[3]["children"][0][3]...`,
|
|
9
|
+
which break the moment a site's component tree reshuffles on redeploy,
|
|
10
|
+
`nextflight` resolves the `$`-sigil references Next.js uses internally
|
|
11
|
+
and lets you *search* for the shape of data you want.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
pip install nextflight
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Quick start
|
|
20
|
+
|
|
21
|
+
The core workflow is two steps: hand it any HTML, see what keys are on
|
|
22
|
+
the page, then fetch the resolved JSON for whichever key you want.
|
|
23
|
+
|
|
24
|
+
```python
|
|
25
|
+
from nextflight import extract
|
|
26
|
+
|
|
27
|
+
# Step 1: send any HTML, get the list of keys (one per __next_f.push chunk)
|
|
28
|
+
page = extract(html_text)
|
|
29
|
+
print(page.keys()) # e.g. ['0', '1', '3f', '20', ...]
|
|
30
|
+
|
|
31
|
+
# Step 2: fetch the resolved JSON for a specific key
|
|
32
|
+
data = page["3f"] # same as page.resolve_chunk("3f")
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Chunk ids are arbitrary per build though (a redeploy can renumber them),
|
|
36
|
+
so in practice you'll usually skip straight to *searching* for the shape
|
|
37
|
+
of data you want instead of a specific id:
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from nextflight import extract
|
|
41
|
+
|
|
42
|
+
page = extract(html_text)
|
|
43
|
+
|
|
44
|
+
# Find the first object anywhere in the page that has all of these keys,
|
|
45
|
+
# wherever this build's component tree happened to put it:
|
|
46
|
+
listing = page.find_by_keys({"sections", "meta"})
|
|
47
|
+
|
|
48
|
+
# Find every node with a given @type (or any custom key):
|
|
49
|
+
products = page.find_by_type("Product")
|
|
50
|
+
|
|
51
|
+
# Or search with a fully custom predicate:
|
|
52
|
+
items = page.find_all(lambda n: isinstance(n, dict) and "price" in n)
|
|
53
|
+
|
|
54
|
+
# Or grab everything, fully dereferenced, and inspect by hand:
|
|
55
|
+
everything = page.resolve_all()
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
### Command line
|
|
59
|
+
|
|
60
|
+
For quick, no-script exploration of a page you've already saved (or a live URL):
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
nextflight page.html --keys sections,meta
|
|
64
|
+
nextflight https://example.com/product/123 --type Product
|
|
65
|
+
nextflight page.html --all > everything.json
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### In a Scrapy / Zyte spider
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
import scrapy
|
|
72
|
+
from nextflight import extract
|
|
73
|
+
|
|
74
|
+
class MySpider(scrapy.Spider):
|
|
75
|
+
name = "my_spider"
|
|
76
|
+
|
|
77
|
+
def parse(self, response):
|
|
78
|
+
page = extract(response.text)
|
|
79
|
+
|
|
80
|
+
items = page.find_all(
|
|
81
|
+
lambda n: isinstance(n, dict) and "price" in n and "title" in n
|
|
82
|
+
)
|
|
83
|
+
for item in items:
|
|
84
|
+
yield {
|
|
85
|
+
"title": item.get("title"),
|
|
86
|
+
"price": item.get("price"),
|
|
87
|
+
"url": response.url,
|
|
88
|
+
}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
### Fetching a URL directly (no Scrapy needed)
|
|
92
|
+
|
|
93
|
+
```python
|
|
94
|
+
from nextflight import FlightExtractor
|
|
95
|
+
|
|
96
|
+
page = FlightExtractor.from_url("https://example.com/product/123")
|
|
97
|
+
product = page.find_by_keys({"price", "title"})
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
(`from_url` uses only the stdlib for quick one-off exploration. For
|
|
101
|
+
production crawling — retries, proxies, JS rendering, robots.txt — fetch
|
|
102
|
+
the page with your own HTTP client / Scrapy / Zyte and pass
|
|
103
|
+
`response.text` to `FlightExtractor(...)` / `extract(...)` instead.)
|
|
104
|
+
|
|
105
|
+
## API
|
|
106
|
+
|
|
107
|
+
- **`extract(html) -> FlightExtractor`** — shorthand constructor. `html`
|
|
108
|
+
accepts a plain string, bytes, or a response-like object (Scrapy's
|
|
109
|
+
`Response`, `requests.Response`, etc.) — pass `response` straight from a
|
|
110
|
+
`parse()` method without writing `response.text` yourself.
|
|
111
|
+
- **`FlightExtractor(html, *, strict: bool = False)`**
|
|
112
|
+
- `.keys() -> list[str]` — every chunk id found on the page, in order.
|
|
113
|
+
- `page["3f"]` / `.resolve_chunk("3f")` — the resolved JSON for one
|
|
114
|
+
specific chunk id (`page[...]` raises `KeyError` if it doesn't exist;
|
|
115
|
+
`resolve_chunk` returns `None`). `"3f" in page` and `for k in page`
|
|
116
|
+
also work, like a dict.
|
|
117
|
+
- `.resolve_all() -> dict` — every chunk, fully dereferenced.
|
|
118
|
+
- `.find_all(predicate, root=None, max_results=None) -> list` — walk the
|
|
119
|
+
resolved tree and collect every node matching `predicate`.
|
|
120
|
+
- `.find_one(predicate, root=None) -> Any | None`
|
|
121
|
+
- `.find_by_keys(required_keys, root=None) -> dict | None` — find the
|
|
122
|
+
first dict containing all of `required_keys`.
|
|
123
|
+
- `.find_all_by_keys(required_keys, root=None) -> list` — like
|
|
124
|
+
`find_by_keys` but returns every match, for pages with repeated
|
|
125
|
+
cards/listings that share the same shape.
|
|
126
|
+
- `.find_by_type(type_value, key="@type", root=None) -> list` — find
|
|
127
|
+
every dict whose `key` field equals `type_value`.
|
|
128
|
+
- `.find_text(pattern, root=None) -> list` — regex-search every string
|
|
129
|
+
value on the page and return the distinct whole values that contain a
|
|
130
|
+
match (emails, prices, phone numbers, SKUs, ...) without needing to
|
|
131
|
+
know which object they live on.
|
|
132
|
+
- `.get("path.to.value", default=None) -> Any` — tolerant dotted-path
|
|
133
|
+
lookup into the resolved page (dict keys and/or list indices), once
|
|
134
|
+
you already know roughly where something lives on this site.
|
|
135
|
+
- `.stats() -> dict` — quick diagnostic snapshot (chunk count, ids,
|
|
136
|
+
value type counts, page size) for exploring a new site.
|
|
137
|
+
- `.to_json(path=None, indent=2) -> str | None` — dump the fully
|
|
138
|
+
resolved page to a file, or return it as a JSON string.
|
|
139
|
+
- `.from_url(url, timeout=15.0, headers=None) -> FlightExtractor`
|
|
140
|
+
(classmethod) — fetch and parse a URL using only the stdlib.
|
|
141
|
+
- `strict=True` raises `FlightParseError` on a row that's neither valid
|
|
142
|
+
JSON nor a recognizable `$`-reference marker, instead of silently
|
|
143
|
+
keeping it as a raw string (useful while developing a new scraper;
|
|
144
|
+
leave off in production so a handful of odd rows never take down
|
|
145
|
+
extraction of everything else on the page).
|
|
146
|
+
- **`find_json_ld(html, type_=None) -> list`** — parse any
|
|
147
|
+
`<script type="application/ld+json">` blocks on the page, optionally
|
|
148
|
+
filtered by `@type`. Also accepts response-like objects.
|
|
149
|
+
- **CLI**: `nextflight <file-or-url> [--keys a,b | --all-by-keys a,b | --type Product | --text PATTERN | --get path.to.value | --stats | --all] [--save out.json]`
|
|
150
|
+
|
|
151
|
+
No runtime dependencies — stdlib only (`json`, `re`, `urllib`, `argparse`)
|
|
152
|
+
— so it's safe to drop into any existing Scrapy/Zyte project without
|
|
153
|
+
touching the rest of your dependency tree.
|
|
154
|
+
|
|
155
|
+
### Upgrading from `nextjs-flight-extractor` / `NextFlightExtractor`
|
|
156
|
+
|
|
157
|
+
The old names still work but emit a `DeprecationWarning`:
|
|
158
|
+
|
|
159
|
+
| Old (0.1.x) | New (0.2.x+) |
|
|
160
|
+
|---------------------------------------|----------------------------------|
|
|
161
|
+
| `from nextjs_flight_extractor import NextFlightExtractor` | `from nextflight import FlightExtractor` |
|
|
162
|
+
| `extractor.find_first(...)` | `page.find_one(...)` |
|
|
163
|
+
| `extract_json_ld(html, schema_type=…)`| `find_json_ld(html, type_=…)` |
|
|
164
|
+
|
|
165
|
+
## Why not just `str.split('\n')`?
|
|
166
|
+
|
|
167
|
+
Two of the Flight row kinds break that assumption:
|
|
168
|
+
|
|
169
|
+
- **Text rows** (`id:T<hexByteLen>,<raw text>`) are byte-length-prefixed
|
|
170
|
+
blobs, not newline-terminated, and can contain literal newlines or run
|
|
171
|
+
directly into the next row's id with zero separator.
|
|
172
|
+
- **Module / preload rows** (`id:I[...]` / `:HL[...]`) need bracket-aware
|
|
173
|
+
parsing.
|
|
174
|
+
|
|
175
|
+
`nextflight` implements the real row grammar, quote/escape aware, so it
|
|
176
|
+
holds up on both well-formed and truncated payloads (e.g. from a proxy
|
|
177
|
+
that cuts a response off mid-chunk).
|
|
178
|
+
|
|
179
|
+
## Building / publishing
|
|
180
|
+
|
|
181
|
+
### Manual (twine)
|
|
182
|
+
|
|
183
|
+
```bash
|
|
184
|
+
pip install build twine
|
|
185
|
+
python -m build # produces dist/*.whl and dist/*.tar.gz
|
|
186
|
+
twine check dist/* # validate metadata before uploading
|
|
187
|
+
twine upload dist/* # publish to PyPI (or use --repository testpypi for a dry run)
|
|
188
|
+
```
|
|
189
|
+
|
|
190
|
+
### Automatic (GitHub Actions + PyPI Trusted Publishing)
|
|
191
|
+
|
|
192
|
+
This repo ships with `.github/workflows/ci.yml`, which:
|
|
193
|
+
- runs the test suite on every push/PR across Python 3.9–3.12,
|
|
194
|
+
- builds and validates the sdist/wheel,
|
|
195
|
+
- publishes to PyPI automatically whenever a tag like `v0.2.1` is pushed.
|
|
196
|
+
|
|
197
|
+
Publishing uses PyPI's **Trusted Publisher** flow — no API token stored in
|
|
198
|
+
GitHub secrets. One-time setup:
|
|
199
|
+
|
|
200
|
+
1. On [pypi.org](https://pypi.org), go to your project → *Publishing* →
|
|
201
|
+
*Add a new publisher* (or, for a brand-new project name, do this from
|
|
202
|
+
your PyPI account's "Trusted Publishers" management page before the
|
|
203
|
+
project exists yet).
|
|
204
|
+
2. Fill in: Owner = your GitHub username/org, Repository = this repo's
|
|
205
|
+
name, Workflow name = `ci.yml`, Environment name = `pypi`.
|
|
206
|
+
3. In your GitHub repo, go to *Settings → Environments*, create an
|
|
207
|
+
environment named `pypi` (optionally require a manual approval before
|
|
208
|
+
deploys, for extra safety).
|
|
209
|
+
4. Release a new version:
|
|
210
|
+
```bash
|
|
211
|
+
# bump version in pyproject.toml and src/nextflight/__init__.py first
|
|
212
|
+
git commit -am "Release v0.2.2"
|
|
213
|
+
git tag v0.2.2
|
|
214
|
+
git push origin main --tags
|
|
215
|
+
```
|
|
216
|
+
The workflow builds, tests, and publishes automatically.
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
## License
|
|
220
|
+
|
|
221
|
+
MIT
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "nextflight"
|
|
7
|
+
version = "0.3.0"
|
|
8
|
+
description = "Parse Next.js App Router 'Flight' (__next_f.push) payloads embedded in server-rendered HTML -- works on any Next.js 13+ site."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Aly Reda" }
|
|
14
|
+
]
|
|
15
|
+
keywords = ["nextjs", "scraping", "scrapy", "flight", "rsc", "react-server-components", "web-scraping", "zyte"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.9",
|
|
19
|
+
"Programming Language :: Python :: 3.10",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"License :: OSI Approved :: MIT License",
|
|
23
|
+
"Operating System :: OS Independent",
|
|
24
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
25
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
26
|
+
]
|
|
27
|
+
# Stdlib only -- no runtime dependencies, so it's safe to drop into any
|
|
28
|
+
# Scrapy / Zyte project without touching the rest of your dependency tree.
|
|
29
|
+
dependencies = []
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Homepage = "https://github.com/Aly-Reda/nextflight"
|
|
33
|
+
Issues = "https://github.com/Aly-Reda/nextflight/issues"
|
|
34
|
+
|
|
35
|
+
[project.scripts]
|
|
36
|
+
nextflight = "nextflight.cli:main"
|
|
37
|
+
|
|
38
|
+
[tool.setuptools.packages.find]
|
|
39
|
+
where = ["src"]
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
"""
|
|
2
|
+
nextflight
|
|
3
|
+
==========
|
|
4
|
+
|
|
5
|
+
Generic parser for the React Server Components "Flight" wire format that
|
|
6
|
+
Next.js 13+ (App Router) embeds in ``<script>self.__next_f.push([...])</script>``
|
|
7
|
+
tags. Works on any Next.js App Router site.
|
|
8
|
+
|
|
9
|
+
from nextflight import extract
|
|
10
|
+
|
|
11
|
+
page = extract(html_text)
|
|
12
|
+
listing = page.find_by_keys({"sections", "meta"})
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
import warnings
|
|
16
|
+
|
|
17
|
+
from .extractor import (
|
|
18
|
+
FlightExtractor,
|
|
19
|
+
FlightParseError,
|
|
20
|
+
extract,
|
|
21
|
+
find_json_ld,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
__all__ = [
|
|
25
|
+
"FlightExtractor",
|
|
26
|
+
"FlightParseError",
|
|
27
|
+
"extract",
|
|
28
|
+
"find_json_ld",
|
|
29
|
+
"NextFlightExtractor", # deprecated alias, see below
|
|
30
|
+
"extract_json_ld", # deprecated alias, see below
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
__version__ = "0.3.0"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# ---------------------------------------------------------------------- #
|
|
37
|
+
# Backwards-compatible aliases for the pre-rename API (nextjs_flight_extractor
|
|
38
|
+
# 0.1.x). These will be removed in a future major version -- switch to
|
|
39
|
+
# FlightExtractor / find_json_ld when convenient.
|
|
40
|
+
# ---------------------------------------------------------------------- #
|
|
41
|
+
class NextFlightExtractor(FlightExtractor):
|
|
42
|
+
"""Deprecated alias for :class:`FlightExtractor`. Use ``FlightExtractor`` instead."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, *args, **kwargs):
|
|
45
|
+
warnings.warn(
|
|
46
|
+
"NextFlightExtractor is deprecated, use nextflight.FlightExtractor instead",
|
|
47
|
+
DeprecationWarning,
|
|
48
|
+
stacklevel=2,
|
|
49
|
+
)
|
|
50
|
+
super().__init__(*args, **kwargs)
|
|
51
|
+
|
|
52
|
+
def find_first(self, *args, **kwargs):
|
|
53
|
+
warnings.warn(
|
|
54
|
+
"find_first() is deprecated, use find_one() instead",
|
|
55
|
+
DeprecationWarning,
|
|
56
|
+
stacklevel=2,
|
|
57
|
+
)
|
|
58
|
+
return self.find_one(*args, **kwargs)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def extract_json_ld(html: str, schema_type=None) -> list:
|
|
62
|
+
"""Deprecated alias for :func:`find_json_ld`. Use ``find_json_ld`` instead."""
|
|
63
|
+
warnings.warn(
|
|
64
|
+
"extract_json_ld() is deprecated, use nextflight.find_json_ld() instead",
|
|
65
|
+
DeprecationWarning,
|
|
66
|
+
stacklevel=2,
|
|
67
|
+
)
|
|
68
|
+
return find_json_ld(html, type_=schema_type)
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line entry point for quick exploration:
|
|
3
|
+
|
|
4
|
+
nextflight page.html --keys sections,meta
|
|
5
|
+
nextflight https://example.com/product/123 --keys price,title
|
|
6
|
+
nextflight page.html --all > everything.json
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
from .extractor import FlightExtractor
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _load_html(source: str) -> str:
|
|
19
|
+
if source.startswith("http://") or source.startswith("https://"):
|
|
20
|
+
extractor = FlightExtractor.from_url(source)
|
|
21
|
+
return extractor.html
|
|
22
|
+
with open(source, "r", encoding="utf-8") as f:
|
|
23
|
+
return f.read()
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def main(argv=None) -> int:
|
|
27
|
+
parser = argparse.ArgumentParser(
|
|
28
|
+
prog="nextflight",
|
|
29
|
+
description="Extract Next.js Flight (__next_f.push) data from a page.",
|
|
30
|
+
)
|
|
31
|
+
parser.add_argument("source", help="Path to an HTML file, or a URL to fetch.")
|
|
32
|
+
parser.add_argument(
|
|
33
|
+
"--keys", help="Comma-separated keys: find the first object containing all of them."
|
|
34
|
+
)
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"--all-by-keys", dest="all_by_keys",
|
|
37
|
+
help="Comma-separated keys: find EVERY object containing all of them (not just the first).",
|
|
38
|
+
)
|
|
39
|
+
parser.add_argument(
|
|
40
|
+
"--type", dest="type_value",
|
|
41
|
+
help='Find every object whose "@type" (or --type-key) equals this value.',
|
|
42
|
+
)
|
|
43
|
+
parser.add_argument("--type-key", default="@type", help='Key to match --type against (default "@type").')
|
|
44
|
+
parser.add_argument("--text", dest="text_pattern", help="Regex: list every distinct string value on the page that matches it.")
|
|
45
|
+
parser.add_argument("--get", dest="get_path", help='Dotted path into the resolved page, e.g. "3f.props.price".')
|
|
46
|
+
parser.add_argument("--all", action="store_true", help="Dump every resolved chunk.")
|
|
47
|
+
parser.add_argument("--stats", action="store_true", help="Print a quick diagnostic summary instead of data.")
|
|
48
|
+
parser.add_argument("--save", dest="save_path", help="Write output to this file instead of stdout.")
|
|
49
|
+
parser.add_argument("--indent", type=int, default=2, help="JSON indent for output (default 2).")
|
|
50
|
+
args = parser.parse_args(argv)
|
|
51
|
+
|
|
52
|
+
html = _load_html(args.source)
|
|
53
|
+
extractor = FlightExtractor(html)
|
|
54
|
+
|
|
55
|
+
if args.keys:
|
|
56
|
+
keys = {k.strip() for k in args.keys.split(",") if k.strip()}
|
|
57
|
+
result = extractor.find_by_keys(keys)
|
|
58
|
+
elif args.all_by_keys:
|
|
59
|
+
keys = {k.strip() for k in args.all_by_keys.split(",") if k.strip()}
|
|
60
|
+
result = extractor.find_all_by_keys(keys)
|
|
61
|
+
elif args.type_value:
|
|
62
|
+
result = extractor.find_by_type(args.type_value, key=args.type_key)
|
|
63
|
+
elif args.text_pattern:
|
|
64
|
+
result = extractor.find_text(args.text_pattern)
|
|
65
|
+
elif args.get_path:
|
|
66
|
+
result = extractor.get(args.get_path)
|
|
67
|
+
elif args.stats:
|
|
68
|
+
result = extractor.stats()
|
|
69
|
+
elif args.all:
|
|
70
|
+
result = extractor.resolve_all()
|
|
71
|
+
else:
|
|
72
|
+
parser.error("Provide one of --keys, --all-by-keys, --type, --text, --get, --stats, or --all.")
|
|
73
|
+
return 2
|
|
74
|
+
|
|
75
|
+
output = json.dumps(result, indent=args.indent, ensure_ascii=False, default=str)
|
|
76
|
+
if args.save_path:
|
|
77
|
+
with open(args.save_path, "w", encoding="utf-8") as f:
|
|
78
|
+
f.write(output)
|
|
79
|
+
else:
|
|
80
|
+
sys.stdout.write(output)
|
|
81
|
+
sys.stdout.write("\n")
|
|
82
|
+
return 0
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
if __name__ == "__main__":
|
|
86
|
+
raise SystemExit(main())
|