scrapeplanner 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scrapeplanner/__init__.py +5 -0
- scrapeplanner/__main__.py +5 -0
- scrapeplanner/analyze.py +389 -0
- scrapeplanner/cli.py +108 -0
- scrapeplanner/config.py +25 -0
- scrapeplanner/detectors/__init__.py +0 -0
- scrapeplanner/detectors/antibot.py +259 -0
- scrapeplanner/detectors/embedded.py +142 -0
- scrapeplanner/detectors/feeds.py +259 -0
- scrapeplanner/detectors/platform.py +349 -0
- scrapeplanner/detectors/rendering.py +65 -0
- scrapeplanner/detectors/robots.py +40 -0
- scrapeplanner/detectors/sitemaps.py +119 -0
- scrapeplanner/detectors/structured.py +321 -0
- scrapeplanner/fetch.py +378 -0
- scrapeplanner/html.py +131 -0
- scrapeplanner/models.py +370 -0
- scrapeplanner/output/__init__.py +0 -0
- scrapeplanner/output/json_out.py +9 -0
- scrapeplanner/output/profile.py +80 -0
- scrapeplanner/output/terminal.py +293 -0
- scrapeplanner/planner.py +465 -0
- scrapeplanner/py.typed +0 -0
- scrapeplanner/record.py +135 -0
- scrapeplanner-0.1.0.dist-info/METADATA +452 -0
- scrapeplanner-0.1.0.dist-info/RECORD +29 -0
- scrapeplanner-0.1.0.dist-info/WHEEL +4 -0
- scrapeplanner-0.1.0.dist-info/entry_points.txt +2 -0
- scrapeplanner-0.1.0.dist-info/licenses/LICENSE +21 -0
scrapeplanner/analyze.py
ADDED
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
"""Run one analysis: decide which URLs to request, call the detectors, build the plan.
|
|
2
|
+
|
|
3
|
+
Request order (each step only if the budget, robots.txt and the block state allow):
|
|
4
|
+
|
|
5
|
+
1. ``/robots.txt``
|
|
6
|
+
2. the target page (redirects followed, each hop counted)
|
|
7
|
+
3. one sitemap: from robots.txt, else ``/sitemap.xml``
|
|
8
|
+
4. the most promising feed probe for the detected platform
|
|
9
|
+
5. one child sitemap, if step 3 found a sitemap index
|
|
10
|
+
6. the remaining feed probes, best first
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import time
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from datetime import UTC, datetime
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from urllib.parse import urlsplit, urlunsplit
|
|
21
|
+
|
|
22
|
+
import httpx
|
|
23
|
+
|
|
24
|
+
from scrapeplanner import __version__, config
|
|
25
|
+
from scrapeplanner.detectors import (
|
|
26
|
+
antibot,
|
|
27
|
+
embedded,
|
|
28
|
+
feeds,
|
|
29
|
+
platform,
|
|
30
|
+
rendering,
|
|
31
|
+
robots,
|
|
32
|
+
structured,
|
|
33
|
+
)
|
|
34
|
+
from scrapeplanner.detectors import sitemaps as sitemap_detector
|
|
35
|
+
from scrapeplanner.fetch import (
|
|
36
|
+
BlockedError,
|
|
37
|
+
BudgetExceeded,
|
|
38
|
+
FetchedPage,
|
|
39
|
+
FetchError,
|
|
40
|
+
PoliteFetcher,
|
|
41
|
+
RobotsDisallowed,
|
|
42
|
+
host_key,
|
|
43
|
+
)
|
|
44
|
+
from scrapeplanner.html import HtmlDoc
|
|
45
|
+
from scrapeplanner.models import (
|
|
46
|
+
EmbeddedStateReport,
|
|
47
|
+
FeedCandidate,
|
|
48
|
+
FeedsReport,
|
|
49
|
+
PageInfo,
|
|
50
|
+
PlatformReport,
|
|
51
|
+
RenderingReport,
|
|
52
|
+
Report,
|
|
53
|
+
SitemapReport,
|
|
54
|
+
StopInfo,
|
|
55
|
+
StructuredDataReport,
|
|
56
|
+
ToolInfo,
|
|
57
|
+
)
|
|
58
|
+
from scrapeplanner.planner import Findings, build_plan
|
|
59
|
+
from scrapeplanner.record import Recorder
|
|
60
|
+
|
|
61
|
+
MAX_LINKED_FEED_PROBES = 2
|
|
62
|
+
_CHILD_HINTS = {
|
|
63
|
+
"commerce": ("sitemap_products", "product-sitemap", "products", "product", "catalog"),
|
|
64
|
+
"jobs": ("job", "career", "position"),
|
|
65
|
+
"articles": ("post", "article", "news", "story"),
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _catalog_answered(probed: list[FeedCandidate]) -> bool:
|
|
70
|
+
return any(
|
|
71
|
+
c.kind == "shopify_products_json" and c.status == "ok" and (c.items_seen or 0) > 0
|
|
72
|
+
for c in probed
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class InvalidURL(ValueError):
|
|
77
|
+
pass
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass
|
|
81
|
+
class Options:
|
|
82
|
+
items: int = config.DEFAULT_ITEMS
|
|
83
|
+
max_requests: int = config.MAX_REQUESTS
|
|
84
|
+
delay: float = config.DELAY_SECONDS
|
|
85
|
+
timeout: float = config.TIMEOUT_SECONDS
|
|
86
|
+
record_dir: Path | None = None
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def normalize_url(raw: str) -> str:
|
|
90
|
+
"""Add https:// when no scheme is given; only http(s) URLs with a host are allowed."""
|
|
91
|
+
text = raw.strip()
|
|
92
|
+
if "://" not in text:
|
|
93
|
+
text = "https://" + text
|
|
94
|
+
parts = urlsplit(text)
|
|
95
|
+
if parts.scheme not in ("http", "https") or not parts.hostname:
|
|
96
|
+
raise InvalidURL(f"not an http(s) URL: {raw!r}")
|
|
97
|
+
if parts.username or parts.password:
|
|
98
|
+
raise InvalidURL("URLs with credentials are not supported (no logins)")
|
|
99
|
+
return urlunsplit((parts.scheme, parts.netloc.lower(), parts.path or "/", parts.query, ""))
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _is_html(page: FetchedPage) -> bool:
|
|
103
|
+
if "html" in page.content_type:
|
|
104
|
+
return True
|
|
105
|
+
return page.content_type in ("", "text/plain") and page.body.lstrip()[:1] == b"<"
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _child_hints(platform_report: PlatformReport, key_types: list[str]) -> tuple[str, ...]:
|
|
109
|
+
if platform_report.name in ("shopify", "woocommerce", "magento", "bigcommerce"):
|
|
110
|
+
return _CHILD_HINTS["commerce"]
|
|
111
|
+
if "JobPosting" in key_types:
|
|
112
|
+
return _CHILD_HINTS["jobs"]
|
|
113
|
+
if any("Article" in name or name == "BlogPosting" for name in key_types):
|
|
114
|
+
return _CHILD_HINTS["articles"]
|
|
115
|
+
return ()
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
class _Run:
|
|
119
|
+
def __init__(self, url: str, options: Options, fetcher: PoliteFetcher) -> None:
|
|
120
|
+
self.url = url
|
|
121
|
+
self.options = options
|
|
122
|
+
self.fetcher = fetcher
|
|
123
|
+
self.page_info = PageInfo(url=url)
|
|
124
|
+
self.page: FetchedPage | None = None
|
|
125
|
+
self.doc: HtmlDoc | None = None
|
|
126
|
+
self.error: StopInfo | None = None
|
|
127
|
+
|
|
128
|
+
def get(self, url: str, purpose: str) -> FetchedPage | None:
|
|
129
|
+
"""Fetch, translating refusals into ``None`` (they are already logged)."""
|
|
130
|
+
try:
|
|
131
|
+
return self.fetcher.fetch(url, purpose)
|
|
132
|
+
except FetchError:
|
|
133
|
+
return None
|
|
134
|
+
except httpx.HTTPError as exc:
|
|
135
|
+
if purpose == "page":
|
|
136
|
+
self.error = StopInfo(reason="error", url=url, detail=f"{type(exc).__name__}")
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
# -- steps ------------------------------------------------------------
|
|
140
|
+
|
|
141
|
+
def fetch_page(self) -> None:
|
|
142
|
+
try:
|
|
143
|
+
self.fetcher.load_robots(self.url)
|
|
144
|
+
except (FetchError, httpx.HTTPError):
|
|
145
|
+
return
|
|
146
|
+
page = self.get(self.url, "page")
|
|
147
|
+
if page is None:
|
|
148
|
+
return
|
|
149
|
+
self.page = page
|
|
150
|
+
info = self.page_info
|
|
151
|
+
info.fetched = True
|
|
152
|
+
info.final_url = page.final_url
|
|
153
|
+
info.status = page.status
|
|
154
|
+
info.content_type = page.content_type or None
|
|
155
|
+
info.bytes = len(page.body)
|
|
156
|
+
if page.ok and _is_html(page):
|
|
157
|
+
self.doc = HtmlDoc(page.text, page.final_url)
|
|
158
|
+
info.title = self.doc.title
|
|
159
|
+
info.lang = self.doc.lang
|
|
160
|
+
|
|
161
|
+
@property
|
|
162
|
+
def base(self) -> str:
|
|
163
|
+
return self.page_info.final_url or self.url
|
|
164
|
+
|
|
165
|
+
def fetch_sitemap(self) -> tuple[SitemapReport, list[str]]:
|
|
166
|
+
origin = host_key(self.base)
|
|
167
|
+
robots_state = self.fetcher.robots.get(origin)
|
|
168
|
+
listed: list[str] = []
|
|
169
|
+
if robots_state is not None and robots_state.parser is not None:
|
|
170
|
+
listed = [str(url) for url in robots_state.parser.sitemaps]
|
|
171
|
+
same_host = [url for url in listed if host_key(url) == origin]
|
|
172
|
+
if same_host or listed:
|
|
173
|
+
url, source = (same_host or listed)[0], "robots"
|
|
174
|
+
else:
|
|
175
|
+
url, source = f"{origin}/sitemap.xml", "default"
|
|
176
|
+
report = SitemapReport(url=url, source=source) # type: ignore[arg-type]
|
|
177
|
+
page = self.get(url, "sitemap")
|
|
178
|
+
if page is None:
|
|
179
|
+
report.note = self._why_skipped(url) or "not fetched"
|
|
180
|
+
return report, []
|
|
181
|
+
return self._parse_sitemap(report, page)
|
|
182
|
+
|
|
183
|
+
def _parse_sitemap(
|
|
184
|
+
self, report: SitemapReport, page: FetchedPage
|
|
185
|
+
) -> tuple[SitemapReport, list[str]]:
|
|
186
|
+
report.fetched = True
|
|
187
|
+
report.status = page.status
|
|
188
|
+
if not page.ok:
|
|
189
|
+
report.note = f"HTTP {page.status}"
|
|
190
|
+
return report, []
|
|
191
|
+
parsed, children = sitemap_detector.parse(page.body, truncated=page.truncated)
|
|
192
|
+
merged = parsed.model_copy(
|
|
193
|
+
update={
|
|
194
|
+
"url": report.url,
|
|
195
|
+
"source": report.source,
|
|
196
|
+
"fetched": True,
|
|
197
|
+
"status": page.status,
|
|
198
|
+
}
|
|
199
|
+
)
|
|
200
|
+
return merged, children
|
|
201
|
+
|
|
202
|
+
def fetch_child_sitemap(
|
|
203
|
+
self, report: SitemapReport, children: list[str], hints: tuple[str, ...]
|
|
204
|
+
) -> None:
|
|
205
|
+
child_url = sitemap_detector.pick_child(children, hints)
|
|
206
|
+
if child_url is None:
|
|
207
|
+
return
|
|
208
|
+
child = SitemapReport(url=child_url, source=report.source)
|
|
209
|
+
page = self.get(child_url, "sitemap-child")
|
|
210
|
+
if page is None:
|
|
211
|
+
child.note = self._why_skipped(child_url) or "not fetched"
|
|
212
|
+
else:
|
|
213
|
+
child, _ = self._parse_sitemap(child, page)
|
|
214
|
+
report.child = child
|
|
215
|
+
|
|
216
|
+
def probe(self, candidate: FeedCandidate) -> FeedCandidate:
|
|
217
|
+
try:
|
|
218
|
+
page = self.fetcher.fetch(candidate.url, f"feed:{candidate.kind}")
|
|
219
|
+
except RobotsDisallowed:
|
|
220
|
+
return candidate.model_copy(update={"status": "skipped_robots"})
|
|
221
|
+
except BudgetExceeded:
|
|
222
|
+
return candidate.model_copy(update={"status": "skipped_budget"})
|
|
223
|
+
except BlockedError:
|
|
224
|
+
return candidate.model_copy(update={"status": "blocked"})
|
|
225
|
+
except httpx.HTTPError as exc:
|
|
226
|
+
return candidate.model_copy(
|
|
227
|
+
update={"status": "invalid", "note": f"{type(exc).__name__}"}
|
|
228
|
+
)
|
|
229
|
+
result = feeds.evaluate(
|
|
230
|
+
candidate, page.status, page.content_type, page.text, dict(page.headers)
|
|
231
|
+
)
|
|
232
|
+
if self.fetcher.stopped is not None and self.fetcher.stopped.url == page.final_url:
|
|
233
|
+
result.status = "blocked"
|
|
234
|
+
elif page.truncated and result.status == "invalid":
|
|
235
|
+
result.note = "response larger than the 5 MB cap; not parsed"
|
|
236
|
+
return result
|
|
237
|
+
|
|
238
|
+
def _why_skipped(self, url: str) -> str | None:
|
|
239
|
+
for skipped in reversed(self.fetcher.skipped):
|
|
240
|
+
if skipped.url == url:
|
|
241
|
+
return {
|
|
242
|
+
"robots": "disallowed by robots.txt",
|
|
243
|
+
"budget": "request budget used up",
|
|
244
|
+
"blocked": "run stopped at a block",
|
|
245
|
+
"crawl_delay": "crawl-delay above the cap",
|
|
246
|
+
}[skipped.reason]
|
|
247
|
+
return None
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def analyze(
|
|
251
|
+
raw_url: str,
|
|
252
|
+
options: Options | None = None,
|
|
253
|
+
*,
|
|
254
|
+
transport: httpx.BaseTransport | None = None,
|
|
255
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
256
|
+
now: datetime | None = None,
|
|
257
|
+
) -> Report:
|
|
258
|
+
options = options or Options()
|
|
259
|
+
url = normalize_url(raw_url)
|
|
260
|
+
started = now or datetime.now(UTC)
|
|
261
|
+
t0 = time.perf_counter()
|
|
262
|
+
recorder = Recorder(options.record_dir) if options.record_dir else None
|
|
263
|
+
fetcher = PoliteFetcher(
|
|
264
|
+
max_requests=options.max_requests,
|
|
265
|
+
delay=options.delay,
|
|
266
|
+
timeout=options.timeout,
|
|
267
|
+
transport=transport,
|
|
268
|
+
recorder=recorder,
|
|
269
|
+
sleep=sleep,
|
|
270
|
+
)
|
|
271
|
+
with fetcher:
|
|
272
|
+
run = _Run(url, options, fetcher)
|
|
273
|
+
run.fetch_page()
|
|
274
|
+
|
|
275
|
+
platform_report = PlatformReport()
|
|
276
|
+
structured_report = StructuredDataReport()
|
|
277
|
+
embedded_report = EmbeddedStateReport()
|
|
278
|
+
rendering_report = RenderingReport()
|
|
279
|
+
if run.page is not None and run.doc is not None:
|
|
280
|
+
platform_report = platform.detect(run.page.headers, run.page.cookie_names, run.doc)
|
|
281
|
+
structured_report = structured.detect(run.doc)
|
|
282
|
+
embedded_report = embedded.detect(run.doc)
|
|
283
|
+
rendering_report = rendering.detect(run.doc, structured_report, embedded_report)
|
|
284
|
+
stop = fetcher.stopped
|
|
285
|
+
if stop is not None and stop.url == run.page.final_url:
|
|
286
|
+
# The "page" was a block or challenge page: nothing of the site to look at.
|
|
287
|
+
platform_report.checked = False
|
|
288
|
+
elif run.page is not None:
|
|
289
|
+
# Headers only (blocked or non-HTML page): a name is possible, a full check is not.
|
|
290
|
+
platform_report = platform.detect(
|
|
291
|
+
run.page.headers, run.page.cookie_names, HtmlDoc("", run.base)
|
|
292
|
+
)
|
|
293
|
+
platform_report.checked = False
|
|
294
|
+
|
|
295
|
+
sitemap_report = SitemapReport()
|
|
296
|
+
children: list[str] = []
|
|
297
|
+
candidates: list[FeedCandidate] = []
|
|
298
|
+
if run.page is not None and run.page.ok:
|
|
299
|
+
sitemap_report, children = run.fetch_sitemap()
|
|
300
|
+
candidates = feeds.candidates(platform_report, run.doc, run.base)
|
|
301
|
+
probed: list[FeedCandidate] = []
|
|
302
|
+
linked_probes = 0
|
|
303
|
+
for index, candidate in enumerate(candidates):
|
|
304
|
+
if index == 1 and children:
|
|
305
|
+
hints = _child_hints(platform_report, structured_report.key_types)
|
|
306
|
+
run.fetch_child_sitemap(sitemap_report, children, hints)
|
|
307
|
+
children = []
|
|
308
|
+
generic = candidate.kind in ("rss", "atom", "json_feed", "json_link")
|
|
309
|
+
if generic and linked_probes >= MAX_LINKED_FEED_PROBES:
|
|
310
|
+
probed.append(candidate)
|
|
311
|
+
continue
|
|
312
|
+
if candidate.kind == "shopify_product_js" and _catalog_answered(probed):
|
|
313
|
+
# products.json already returned the products; the per-product JSON
|
|
314
|
+
# would add nothing to the plan, so keep the request (I-009).
|
|
315
|
+
probed.append(
|
|
316
|
+
candidate.model_copy(
|
|
317
|
+
update={"note": "not requested: /products.json already answered"}
|
|
318
|
+
)
|
|
319
|
+
)
|
|
320
|
+
continue
|
|
321
|
+
linked_probes += generic
|
|
322
|
+
probed.append(run.probe(candidate))
|
|
323
|
+
if children:
|
|
324
|
+
hints = _child_hints(platform_report, structured_report.key_types)
|
|
325
|
+
run.fetch_child_sitemap(sitemap_report, children, hints)
|
|
326
|
+
candidates = probed
|
|
327
|
+
|
|
328
|
+
origin = host_key(run.base)
|
|
329
|
+
used_urls = [run.base]
|
|
330
|
+
if sitemap_report.url:
|
|
331
|
+
used_urls.append(sitemap_report.url)
|
|
332
|
+
used_urls.extend(candidate.url for candidate in candidates)
|
|
333
|
+
if sitemap_report.child and sitemap_report.child.sample_urls:
|
|
334
|
+
used_urls.append(sitemap_report.child.sample_urls[0])
|
|
335
|
+
elif sitemap_report.sample_urls and sitemap_report.kind == "urlset":
|
|
336
|
+
used_urls.append(sitemap_report.sample_urls[0])
|
|
337
|
+
robots_report = robots.build_report(
|
|
338
|
+
f"{origin}/robots.txt",
|
|
339
|
+
fetcher.robots.get(origin),
|
|
340
|
+
[u for u in used_urls if host_key(u) == origin],
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
antibot_report = antibot.merge(
|
|
344
|
+
(
|
|
345
|
+
page.final_url,
|
|
346
|
+
antibot.detect(page.status, page.headers, page.cookie_names, page.text),
|
|
347
|
+
)
|
|
348
|
+
for page in fetcher.pages
|
|
349
|
+
)
|
|
350
|
+
stopped = fetcher.stopped or run.error
|
|
351
|
+
plan = build_plan(
|
|
352
|
+
Findings(
|
|
353
|
+
page=run.page_info,
|
|
354
|
+
robots=robots_report,
|
|
355
|
+
sitemap=sitemap_report,
|
|
356
|
+
platform=platform_report,
|
|
357
|
+
antibot=antibot_report,
|
|
358
|
+
structured=structured_report,
|
|
359
|
+
embedded=embedded_report,
|
|
360
|
+
rendering=rendering_report,
|
|
361
|
+
feeds=FeedsReport(candidates=candidates),
|
|
362
|
+
stopped=stopped,
|
|
363
|
+
items=options.items,
|
|
364
|
+
base_delay=options.delay,
|
|
365
|
+
today=started.date(),
|
|
366
|
+
)
|
|
367
|
+
)
|
|
368
|
+
return Report(
|
|
369
|
+
tool=ToolInfo(version=__version__, user_agent=config.USER_AGENT),
|
|
370
|
+
input_url=url,
|
|
371
|
+
site=urlsplit(run.base).netloc,
|
|
372
|
+
started_at=started.isoformat(timespec="seconds"),
|
|
373
|
+
duration_seconds=round(time.perf_counter() - t0, 2),
|
|
374
|
+
max_requests=options.max_requests,
|
|
375
|
+
requests_used=fetcher.requests_used,
|
|
376
|
+
requests=fetcher.log,
|
|
377
|
+
skipped=fetcher.skipped,
|
|
378
|
+
stopped=stopped,
|
|
379
|
+
page=run.page_info,
|
|
380
|
+
robots=robots_report,
|
|
381
|
+
sitemap=sitemap_report,
|
|
382
|
+
platform=platform_report,
|
|
383
|
+
antibot=antibot_report,
|
|
384
|
+
structured=structured_report,
|
|
385
|
+
embedded=embedded_report,
|
|
386
|
+
rendering=rendering_report,
|
|
387
|
+
feeds=FeedsReport(candidates=candidates),
|
|
388
|
+
plan=plan,
|
|
389
|
+
)
|
scrapeplanner/cli.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Command line: ``scrapeplanner URL [--json] [--profile PATH] ...``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Annotated
|
|
8
|
+
|
|
9
|
+
import typer
|
|
10
|
+
from rich.console import Console
|
|
11
|
+
|
|
12
|
+
from scrapeplanner import __version__, config
|
|
13
|
+
from scrapeplanner.analyze import InvalidURL, Options, analyze
|
|
14
|
+
from scrapeplanner.output.json_out import to_json
|
|
15
|
+
from scrapeplanner.output.profile import to_yaml
|
|
16
|
+
from scrapeplanner.output.terminal import render
|
|
17
|
+
|
|
18
|
+
app = typer.Typer(
|
|
19
|
+
add_completion=False,
|
|
20
|
+
no_args_is_help=True,
|
|
21
|
+
help=(
|
|
22
|
+
"Report the cheapest reliable, polite way to collect a site's data. "
|
|
23
|
+
f"Makes at most {config.MAX_REQUESTS} requests by default, reads robots.txt "
|
|
24
|
+
"first and stops at the first block."
|
|
25
|
+
),
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _version(value: bool) -> None:
|
|
30
|
+
if value:
|
|
31
|
+
typer.echo(f"scrapeplanner {__version__}")
|
|
32
|
+
raise typer.Exit()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@app.command()
|
|
36
|
+
def run(
|
|
37
|
+
url: Annotated[str, typer.Argument(help="Page to analyse: a homepage or a public listing.")],
|
|
38
|
+
json_output: Annotated[
|
|
39
|
+
bool, typer.Option("--json", help="Print the full report as JSON instead of text.")
|
|
40
|
+
] = False,
|
|
41
|
+
profile: Annotated[
|
|
42
|
+
Path | None,
|
|
43
|
+
typer.Option(
|
|
44
|
+
"--profile",
|
|
45
|
+
help="Write a starter profile.yaml to this path.",
|
|
46
|
+
dir_okay=False,
|
|
47
|
+
writable=True,
|
|
48
|
+
),
|
|
49
|
+
] = None,
|
|
50
|
+
items: Annotated[
|
|
51
|
+
int, typer.Option("--items", min=1, help="Number of records to estimate for.")
|
|
52
|
+
] = config.DEFAULT_ITEMS,
|
|
53
|
+
max_requests: Annotated[
|
|
54
|
+
int,
|
|
55
|
+
typer.Option(
|
|
56
|
+
"--max-requests",
|
|
57
|
+
min=1,
|
|
58
|
+
max=config.MAX_REQUESTS_CAP,
|
|
59
|
+
help="Request budget for this run (every redirect hop counts).",
|
|
60
|
+
),
|
|
61
|
+
] = config.MAX_REQUESTS,
|
|
62
|
+
delay: Annotated[
|
|
63
|
+
float,
|
|
64
|
+
typer.Option(
|
|
65
|
+
"--delay", min=1.0, help="Seconds between requests (robots crawl-delay wins if larger)."
|
|
66
|
+
),
|
|
67
|
+
] = config.DELAY_SECONDS,
|
|
68
|
+
timeout: Annotated[
|
|
69
|
+
float, typer.Option("--timeout", min=1.0, help="Per-request timeout in seconds.")
|
|
70
|
+
] = config.TIMEOUT_SECONDS,
|
|
71
|
+
record: Annotated[
|
|
72
|
+
Path | None,
|
|
73
|
+
typer.Option(
|
|
74
|
+
"--record",
|
|
75
|
+
file_okay=False,
|
|
76
|
+
help="Save every HTTP exchange to this directory (for tests).",
|
|
77
|
+
),
|
|
78
|
+
] = None,
|
|
79
|
+
version: Annotated[
|
|
80
|
+
bool | None,
|
|
81
|
+
typer.Option("--version", callback=_version, is_eager=True, help="Show the version."),
|
|
82
|
+
] = None,
|
|
83
|
+
) -> None:
|
|
84
|
+
"""Analyse URL and print a collection plan."""
|
|
85
|
+
options = Options(
|
|
86
|
+
items=items, max_requests=max_requests, delay=delay, timeout=timeout, record_dir=record
|
|
87
|
+
)
|
|
88
|
+
err = Console(stderr=True)
|
|
89
|
+
try:
|
|
90
|
+
if not json_output:
|
|
91
|
+
with err.status(f"Analysing {url} (polite: {delay:g}s between requests)"):
|
|
92
|
+
report = analyze(url, options)
|
|
93
|
+
else:
|
|
94
|
+
report = analyze(url, options)
|
|
95
|
+
except InvalidURL as exc:
|
|
96
|
+
err.print(f"[red]error:[/] {exc}")
|
|
97
|
+
raise typer.Exit(2) from exc
|
|
98
|
+
if json_output:
|
|
99
|
+
sys.stdout.write(to_json(report))
|
|
100
|
+
else:
|
|
101
|
+
render(report, Console())
|
|
102
|
+
if profile is not None:
|
|
103
|
+
profile.write_text(to_yaml(report), encoding="utf-8")
|
|
104
|
+
err.print(f"profile written to {profile}")
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def main() -> None:
|
|
108
|
+
app()
|
scrapeplanner/config.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Defaults that define how polite the tool is. See docs/decisions.md (D-006)."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Final
|
|
6
|
+
|
|
7
|
+
from scrapeplanner import __version__
|
|
8
|
+
|
|
9
|
+
PROJECT_URL: Final = "https://github.com/siddharthkumarx/scrapeplanner"
|
|
10
|
+
UA_TOKEN: Final = "scrapeplanner"
|
|
11
|
+
USER_AGENT: Final = f"{UA_TOKEN}/{__version__} (+{PROJECT_URL})"
|
|
12
|
+
|
|
13
|
+
MAX_REQUESTS: Final = 10
|
|
14
|
+
"""Default request budget per site per run (every redirect hop counts)."""
|
|
15
|
+
MAX_REQUESTS_CAP: Final = 50
|
|
16
|
+
"""Hard ceiling for ``--max-requests``."""
|
|
17
|
+
DELAY_SECONDS: Final = 1.0
|
|
18
|
+
"""Minimum gap between two requests to the same site."""
|
|
19
|
+
MAX_CRAWL_DELAY: Final = 30.0
|
|
20
|
+
"""A robots.txt crawl-delay above this stops the run instead of waiting."""
|
|
21
|
+
TIMEOUT_SECONDS: Final = 15.0
|
|
22
|
+
MAX_BODY_BYTES: Final = 5_000_000
|
|
23
|
+
MAX_REDIRECTS: Final = 5
|
|
24
|
+
DEFAULT_ITEMS: Final = 1000
|
|
25
|
+
"""Default N for the request estimate."""
|
|
File without changes
|