scrapeplanner 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ """scrapeplanner: find the cheapest reliable, polite way to collect a site's data."""
2
+
3
+ __version__ = "0.1.0"
4
+
5
+ __all__ = ["__version__"]
@@ -0,0 +1,5 @@
1
+ """``python -m scrapeplanner`` runs the CLI."""
2
+
3
+ from scrapeplanner.cli import main
4
+
5
+ main()
@@ -0,0 +1,389 @@
1
+ """Run one analysis: decide which URLs to request, call the detectors, build the plan.
2
+
3
+ Request order (each step only if the budget, robots.txt and the block state allow):
4
+
5
+ 1. ``/robots.txt``
6
+ 2. the target page (redirects followed, each hop counted)
7
+ 3. one sitemap: from robots.txt, else ``/sitemap.xml``
8
+ 4. the most promising feed probe for the detected platform
9
+ 5. one child sitemap, if step 3 found a sitemap index
10
+ 6. the remaining feed probes, best first
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import time
16
+ from collections.abc import Callable
17
+ from dataclasses import dataclass
18
+ from datetime import UTC, datetime
19
+ from pathlib import Path
20
+ from urllib.parse import urlsplit, urlunsplit
21
+
22
+ import httpx
23
+
24
+ from scrapeplanner import __version__, config
25
+ from scrapeplanner.detectors import (
26
+ antibot,
27
+ embedded,
28
+ feeds,
29
+ platform,
30
+ rendering,
31
+ robots,
32
+ structured,
33
+ )
34
+ from scrapeplanner.detectors import sitemaps as sitemap_detector
35
+ from scrapeplanner.fetch import (
36
+ BlockedError,
37
+ BudgetExceeded,
38
+ FetchedPage,
39
+ FetchError,
40
+ PoliteFetcher,
41
+ RobotsDisallowed,
42
+ host_key,
43
+ )
44
+ from scrapeplanner.html import HtmlDoc
45
+ from scrapeplanner.models import (
46
+ EmbeddedStateReport,
47
+ FeedCandidate,
48
+ FeedsReport,
49
+ PageInfo,
50
+ PlatformReport,
51
+ RenderingReport,
52
+ Report,
53
+ SitemapReport,
54
+ StopInfo,
55
+ StructuredDataReport,
56
+ ToolInfo,
57
+ )
58
+ from scrapeplanner.planner import Findings, build_plan
59
+ from scrapeplanner.record import Recorder
60
+
61
+ MAX_LINKED_FEED_PROBES = 2
62
+ _CHILD_HINTS = {
63
+ "commerce": ("sitemap_products", "product-sitemap", "products", "product", "catalog"),
64
+ "jobs": ("job", "career", "position"),
65
+ "articles": ("post", "article", "news", "story"),
66
+ }
67
+
68
+
69
+ def _catalog_answered(probed: list[FeedCandidate]) -> bool:
70
+ return any(
71
+ c.kind == "shopify_products_json" and c.status == "ok" and (c.items_seen or 0) > 0
72
+ for c in probed
73
+ )
74
+
75
+
76
+ class InvalidURL(ValueError):
77
+ pass
78
+
79
+
80
+ @dataclass
81
+ class Options:
82
+ items: int = config.DEFAULT_ITEMS
83
+ max_requests: int = config.MAX_REQUESTS
84
+ delay: float = config.DELAY_SECONDS
85
+ timeout: float = config.TIMEOUT_SECONDS
86
+ record_dir: Path | None = None
87
+
88
+
89
+ def normalize_url(raw: str) -> str:
90
+ """Add https:// when no scheme is given; only http(s) URLs with a host are allowed."""
91
+ text = raw.strip()
92
+ if "://" not in text:
93
+ text = "https://" + text
94
+ parts = urlsplit(text)
95
+ if parts.scheme not in ("http", "https") or not parts.hostname:
96
+ raise InvalidURL(f"not an http(s) URL: {raw!r}")
97
+ if parts.username or parts.password:
98
+ raise InvalidURL("URLs with credentials are not supported (no logins)")
99
+ return urlunsplit((parts.scheme, parts.netloc.lower(), parts.path or "/", parts.query, ""))
100
+
101
+
102
+ def _is_html(page: FetchedPage) -> bool:
103
+ if "html" in page.content_type:
104
+ return True
105
+ return page.content_type in ("", "text/plain") and page.body.lstrip()[:1] == b"<"
106
+
107
+
108
+ def _child_hints(platform_report: PlatformReport, key_types: list[str]) -> tuple[str, ...]:
109
+ if platform_report.name in ("shopify", "woocommerce", "magento", "bigcommerce"):
110
+ return _CHILD_HINTS["commerce"]
111
+ if "JobPosting" in key_types:
112
+ return _CHILD_HINTS["jobs"]
113
+ if any("Article" in name or name == "BlogPosting" for name in key_types):
114
+ return _CHILD_HINTS["articles"]
115
+ return ()
116
+
117
+
118
+ class _Run:
119
+ def __init__(self, url: str, options: Options, fetcher: PoliteFetcher) -> None:
120
+ self.url = url
121
+ self.options = options
122
+ self.fetcher = fetcher
123
+ self.page_info = PageInfo(url=url)
124
+ self.page: FetchedPage | None = None
125
+ self.doc: HtmlDoc | None = None
126
+ self.error: StopInfo | None = None
127
+
128
+ def get(self, url: str, purpose: str) -> FetchedPage | None:
129
+ """Fetch, translating refusals into ``None`` (they are already logged)."""
130
+ try:
131
+ return self.fetcher.fetch(url, purpose)
132
+ except FetchError:
133
+ return None
134
+ except httpx.HTTPError as exc:
135
+ if purpose == "page":
136
+ self.error = StopInfo(reason="error", url=url, detail=f"{type(exc).__name__}")
137
+ return None
138
+
139
+ # -- steps ------------------------------------------------------------
140
+
141
+ def fetch_page(self) -> None:
142
+ try:
143
+ self.fetcher.load_robots(self.url)
144
+ except (FetchError, httpx.HTTPError):
145
+ return
146
+ page = self.get(self.url, "page")
147
+ if page is None:
148
+ return
149
+ self.page = page
150
+ info = self.page_info
151
+ info.fetched = True
152
+ info.final_url = page.final_url
153
+ info.status = page.status
154
+ info.content_type = page.content_type or None
155
+ info.bytes = len(page.body)
156
+ if page.ok and _is_html(page):
157
+ self.doc = HtmlDoc(page.text, page.final_url)
158
+ info.title = self.doc.title
159
+ info.lang = self.doc.lang
160
+
161
+ @property
162
+ def base(self) -> str:
163
+ return self.page_info.final_url or self.url
164
+
165
+ def fetch_sitemap(self) -> tuple[SitemapReport, list[str]]:
166
+ origin = host_key(self.base)
167
+ robots_state = self.fetcher.robots.get(origin)
168
+ listed: list[str] = []
169
+ if robots_state is not None and robots_state.parser is not None:
170
+ listed = [str(url) for url in robots_state.parser.sitemaps]
171
+ same_host = [url for url in listed if host_key(url) == origin]
172
+ if same_host or listed:
173
+ url, source = (same_host or listed)[0], "robots"
174
+ else:
175
+ url, source = f"{origin}/sitemap.xml", "default"
176
+ report = SitemapReport(url=url, source=source) # type: ignore[arg-type]
177
+ page = self.get(url, "sitemap")
178
+ if page is None:
179
+ report.note = self._why_skipped(url) or "not fetched"
180
+ return report, []
181
+ return self._parse_sitemap(report, page)
182
+
183
+ def _parse_sitemap(
184
+ self, report: SitemapReport, page: FetchedPage
185
+ ) -> tuple[SitemapReport, list[str]]:
186
+ report.fetched = True
187
+ report.status = page.status
188
+ if not page.ok:
189
+ report.note = f"HTTP {page.status}"
190
+ return report, []
191
+ parsed, children = sitemap_detector.parse(page.body, truncated=page.truncated)
192
+ merged = parsed.model_copy(
193
+ update={
194
+ "url": report.url,
195
+ "source": report.source,
196
+ "fetched": True,
197
+ "status": page.status,
198
+ }
199
+ )
200
+ return merged, children
201
+
202
+ def fetch_child_sitemap(
203
+ self, report: SitemapReport, children: list[str], hints: tuple[str, ...]
204
+ ) -> None:
205
+ child_url = sitemap_detector.pick_child(children, hints)
206
+ if child_url is None:
207
+ return
208
+ child = SitemapReport(url=child_url, source=report.source)
209
+ page = self.get(child_url, "sitemap-child")
210
+ if page is None:
211
+ child.note = self._why_skipped(child_url) or "not fetched"
212
+ else:
213
+ child, _ = self._parse_sitemap(child, page)
214
+ report.child = child
215
+
216
+ def probe(self, candidate: FeedCandidate) -> FeedCandidate:
217
+ try:
218
+ page = self.fetcher.fetch(candidate.url, f"feed:{candidate.kind}")
219
+ except RobotsDisallowed:
220
+ return candidate.model_copy(update={"status": "skipped_robots"})
221
+ except BudgetExceeded:
222
+ return candidate.model_copy(update={"status": "skipped_budget"})
223
+ except BlockedError:
224
+ return candidate.model_copy(update={"status": "blocked"})
225
+ except httpx.HTTPError as exc:
226
+ return candidate.model_copy(
227
+ update={"status": "invalid", "note": f"{type(exc).__name__}"}
228
+ )
229
+ result = feeds.evaluate(
230
+ candidate, page.status, page.content_type, page.text, dict(page.headers)
231
+ )
232
+ if self.fetcher.stopped is not None and self.fetcher.stopped.url == page.final_url:
233
+ result.status = "blocked"
234
+ elif page.truncated and result.status == "invalid":
235
+ result.note = "response larger than the 5 MB cap; not parsed"
236
+ return result
237
+
238
+ def _why_skipped(self, url: str) -> str | None:
239
+ for skipped in reversed(self.fetcher.skipped):
240
+ if skipped.url == url:
241
+ return {
242
+ "robots": "disallowed by robots.txt",
243
+ "budget": "request budget used up",
244
+ "blocked": "run stopped at a block",
245
+ "crawl_delay": "crawl-delay above the cap",
246
+ }[skipped.reason]
247
+ return None
248
+
249
+
250
+ def analyze(
251
+ raw_url: str,
252
+ options: Options | None = None,
253
+ *,
254
+ transport: httpx.BaseTransport | None = None,
255
+ sleep: Callable[[float], None] = time.sleep,
256
+ now: datetime | None = None,
257
+ ) -> Report:
258
+ options = options or Options()
259
+ url = normalize_url(raw_url)
260
+ started = now or datetime.now(UTC)
261
+ t0 = time.perf_counter()
262
+ recorder = Recorder(options.record_dir) if options.record_dir else None
263
+ fetcher = PoliteFetcher(
264
+ max_requests=options.max_requests,
265
+ delay=options.delay,
266
+ timeout=options.timeout,
267
+ transport=transport,
268
+ recorder=recorder,
269
+ sleep=sleep,
270
+ )
271
+ with fetcher:
272
+ run = _Run(url, options, fetcher)
273
+ run.fetch_page()
274
+
275
+ platform_report = PlatformReport()
276
+ structured_report = StructuredDataReport()
277
+ embedded_report = EmbeddedStateReport()
278
+ rendering_report = RenderingReport()
279
+ if run.page is not None and run.doc is not None:
280
+ platform_report = platform.detect(run.page.headers, run.page.cookie_names, run.doc)
281
+ structured_report = structured.detect(run.doc)
282
+ embedded_report = embedded.detect(run.doc)
283
+ rendering_report = rendering.detect(run.doc, structured_report, embedded_report)
284
+ stop = fetcher.stopped
285
+ if stop is not None and stop.url == run.page.final_url:
286
+ # The "page" was a block or challenge page: nothing of the site to look at.
287
+ platform_report.checked = False
288
+ elif run.page is not None:
289
+ # Headers only (blocked or non-HTML page): a name is possible, a full check is not.
290
+ platform_report = platform.detect(
291
+ run.page.headers, run.page.cookie_names, HtmlDoc("", run.base)
292
+ )
293
+ platform_report.checked = False
294
+
295
+ sitemap_report = SitemapReport()
296
+ children: list[str] = []
297
+ candidates: list[FeedCandidate] = []
298
+ if run.page is not None and run.page.ok:
299
+ sitemap_report, children = run.fetch_sitemap()
300
+ candidates = feeds.candidates(platform_report, run.doc, run.base)
301
+ probed: list[FeedCandidate] = []
302
+ linked_probes = 0
303
+ for index, candidate in enumerate(candidates):
304
+ if index == 1 and children:
305
+ hints = _child_hints(platform_report, structured_report.key_types)
306
+ run.fetch_child_sitemap(sitemap_report, children, hints)
307
+ children = []
308
+ generic = candidate.kind in ("rss", "atom", "json_feed", "json_link")
309
+ if generic and linked_probes >= MAX_LINKED_FEED_PROBES:
310
+ probed.append(candidate)
311
+ continue
312
+ if candidate.kind == "shopify_product_js" and _catalog_answered(probed):
313
+ # products.json already returned the products; the per-product JSON
314
+ # would add nothing to the plan, so keep the request (I-009).
315
+ probed.append(
316
+ candidate.model_copy(
317
+ update={"note": "not requested: /products.json already answered"}
318
+ )
319
+ )
320
+ continue
321
+ linked_probes += generic
322
+ probed.append(run.probe(candidate))
323
+ if children:
324
+ hints = _child_hints(platform_report, structured_report.key_types)
325
+ run.fetch_child_sitemap(sitemap_report, children, hints)
326
+ candidates = probed
327
+
328
+ origin = host_key(run.base)
329
+ used_urls = [run.base]
330
+ if sitemap_report.url:
331
+ used_urls.append(sitemap_report.url)
332
+ used_urls.extend(candidate.url for candidate in candidates)
333
+ if sitemap_report.child and sitemap_report.child.sample_urls:
334
+ used_urls.append(sitemap_report.child.sample_urls[0])
335
+ elif sitemap_report.sample_urls and sitemap_report.kind == "urlset":
336
+ used_urls.append(sitemap_report.sample_urls[0])
337
+ robots_report = robots.build_report(
338
+ f"{origin}/robots.txt",
339
+ fetcher.robots.get(origin),
340
+ [u for u in used_urls if host_key(u) == origin],
341
+ )
342
+
343
+ antibot_report = antibot.merge(
344
+ (
345
+ page.final_url,
346
+ antibot.detect(page.status, page.headers, page.cookie_names, page.text),
347
+ )
348
+ for page in fetcher.pages
349
+ )
350
+ stopped = fetcher.stopped or run.error
351
+ plan = build_plan(
352
+ Findings(
353
+ page=run.page_info,
354
+ robots=robots_report,
355
+ sitemap=sitemap_report,
356
+ platform=platform_report,
357
+ antibot=antibot_report,
358
+ structured=structured_report,
359
+ embedded=embedded_report,
360
+ rendering=rendering_report,
361
+ feeds=FeedsReport(candidates=candidates),
362
+ stopped=stopped,
363
+ items=options.items,
364
+ base_delay=options.delay,
365
+ today=started.date(),
366
+ )
367
+ )
368
+ return Report(
369
+ tool=ToolInfo(version=__version__, user_agent=config.USER_AGENT),
370
+ input_url=url,
371
+ site=urlsplit(run.base).netloc,
372
+ started_at=started.isoformat(timespec="seconds"),
373
+ duration_seconds=round(time.perf_counter() - t0, 2),
374
+ max_requests=options.max_requests,
375
+ requests_used=fetcher.requests_used,
376
+ requests=fetcher.log,
377
+ skipped=fetcher.skipped,
378
+ stopped=stopped,
379
+ page=run.page_info,
380
+ robots=robots_report,
381
+ sitemap=sitemap_report,
382
+ platform=platform_report,
383
+ antibot=antibot_report,
384
+ structured=structured_report,
385
+ embedded=embedded_report,
386
+ rendering=rendering_report,
387
+ feeds=FeedsReport(candidates=candidates),
388
+ plan=plan,
389
+ )
scrapeplanner/cli.py ADDED
@@ -0,0 +1,108 @@
1
+ """Command line: ``scrapeplanner URL [--json] [--profile PATH] ...``."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import sys
6
+ from pathlib import Path
7
+ from typing import Annotated
8
+
9
+ import typer
10
+ from rich.console import Console
11
+
12
+ from scrapeplanner import __version__, config
13
+ from scrapeplanner.analyze import InvalidURL, Options, analyze
14
+ from scrapeplanner.output.json_out import to_json
15
+ from scrapeplanner.output.profile import to_yaml
16
+ from scrapeplanner.output.terminal import render
17
+
18
+ app = typer.Typer(
19
+ add_completion=False,
20
+ no_args_is_help=True,
21
+ help=(
22
+ "Report the cheapest reliable, polite way to collect a site's data. "
23
+ f"Makes at most {config.MAX_REQUESTS} requests by default, reads robots.txt "
24
+ "first and stops at the first block."
25
+ ),
26
+ )
27
+
28
+
29
+ def _version(value: bool) -> None:
30
+ if value:
31
+ typer.echo(f"scrapeplanner {__version__}")
32
+ raise typer.Exit()
33
+
34
+
35
+ @app.command()
36
+ def run(
37
+ url: Annotated[str, typer.Argument(help="Page to analyse: a homepage or a public listing.")],
38
+ json_output: Annotated[
39
+ bool, typer.Option("--json", help="Print the full report as JSON instead of text.")
40
+ ] = False,
41
+ profile: Annotated[
42
+ Path | None,
43
+ typer.Option(
44
+ "--profile",
45
+ help="Write a starter profile.yaml to this path.",
46
+ dir_okay=False,
47
+ writable=True,
48
+ ),
49
+ ] = None,
50
+ items: Annotated[
51
+ int, typer.Option("--items", min=1, help="Number of records to estimate for.")
52
+ ] = config.DEFAULT_ITEMS,
53
+ max_requests: Annotated[
54
+ int,
55
+ typer.Option(
56
+ "--max-requests",
57
+ min=1,
58
+ max=config.MAX_REQUESTS_CAP,
59
+ help="Request budget for this run (every redirect hop counts).",
60
+ ),
61
+ ] = config.MAX_REQUESTS,
62
+ delay: Annotated[
63
+ float,
64
+ typer.Option(
65
+ "--delay", min=1.0, help="Seconds between requests (robots crawl-delay wins if larger)."
66
+ ),
67
+ ] = config.DELAY_SECONDS,
68
+ timeout: Annotated[
69
+ float, typer.Option("--timeout", min=1.0, help="Per-request timeout in seconds.")
70
+ ] = config.TIMEOUT_SECONDS,
71
+ record: Annotated[
72
+ Path | None,
73
+ typer.Option(
74
+ "--record",
75
+ file_okay=False,
76
+ help="Save every HTTP exchange to this directory (for tests).",
77
+ ),
78
+ ] = None,
79
+ version: Annotated[
80
+ bool | None,
81
+ typer.Option("--version", callback=_version, is_eager=True, help="Show the version."),
82
+ ] = None,
83
+ ) -> None:
84
+ """Analyse URL and print a collection plan."""
85
+ options = Options(
86
+ items=items, max_requests=max_requests, delay=delay, timeout=timeout, record_dir=record
87
+ )
88
+ err = Console(stderr=True)
89
+ try:
90
+ if not json_output:
91
+ with err.status(f"Analysing {url} (polite: {delay:g}s between requests)"):
92
+ report = analyze(url, options)
93
+ else:
94
+ report = analyze(url, options)
95
+ except InvalidURL as exc:
96
+ err.print(f"[red]error:[/] {exc}")
97
+ raise typer.Exit(2) from exc
98
+ if json_output:
99
+ sys.stdout.write(to_json(report))
100
+ else:
101
+ render(report, Console())
102
+ if profile is not None:
103
+ profile.write_text(to_yaml(report), encoding="utf-8")
104
+ err.print(f"profile written to {profile}")
105
+
106
+
107
+ def main() -> None:
108
+ app()
@@ -0,0 +1,25 @@
1
+ """Defaults that define how polite the tool is. See docs/decisions.md (D-006)."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Final
6
+
7
+ from scrapeplanner import __version__
8
+
9
+ PROJECT_URL: Final = "https://github.com/siddharthkumarx/scrapeplanner"
10
+ UA_TOKEN: Final = "scrapeplanner"
11
+ USER_AGENT: Final = f"{UA_TOKEN}/{__version__} (+{PROJECT_URL})"
12
+
13
+ MAX_REQUESTS: Final = 10
14
+ """Default request budget per site per run (every redirect hop counts)."""
15
+ MAX_REQUESTS_CAP: Final = 50
16
+ """Hard ceiling for ``--max-requests``."""
17
+ DELAY_SECONDS: Final = 1.0
18
+ """Minimum gap between two requests to the same site."""
19
+ MAX_CRAWL_DELAY: Final = 30.0
20
+ """A robots.txt crawl-delay above this stops the run instead of waiting."""
21
+ TIMEOUT_SECONDS: Final = 15.0
22
+ MAX_BODY_BYTES: Final = 5_000_000
23
+ MAX_REDIRECTS: Final = 5
24
+ DEFAULT_ITEMS: Final = 1000
25
+ """Default N for the request estimate."""
File without changes