JumpingSpider 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
JumpingSpider/main.py ADDED
@@ -0,0 +1,279 @@
1
+ from __future__ import annotations
2
+
3
+ import argparse
4
+ import sys
5
+ from collections.abc import Sequence
6
+ from pathlib import Path
7
+
8
+ from rich.console import Console
9
+ from rich.table import Table
10
+
11
+ from JumpingSpider.crawler import SpiderCrawler
12
+ from JumpingSpider.models import CrawlConfig, CrawlSummary, LinkResult
13
+ from JumpingSpider.reporters import export_report
14
+
15
+
16
+ def crawl_site(
17
+ start_url: str,
18
+ output_filename: str | Sequence[str] = "crawl_results.json",
19
+ delay: float = 0.2,
20
+ workers: int = 5,
21
+ timeout: float = 10.0,
22
+ max_depth: int | None = None,
23
+ check_external: bool = True,
24
+ check_assets: bool = True,
25
+ broken_only: bool = False,
26
+ fmt: str | None = None,
27
+ quiet: bool = False,
28
+ ) -> CrawlSummary:
29
+ """
30
+ Crawls a website starting at `start_url`, strictly staying within its path boundary
31
+ for recursion while optionally validating external links and page assets.
32
+ """
33
+ if isinstance(output_filename, str):
34
+ output_files = [output_filename] if output_filename else []
35
+ else:
36
+ output_files = list(output_filename)
37
+
38
+ config = CrawlConfig(
39
+ start_url=start_url,
40
+ output_files=output_files,
41
+ delay=delay,
42
+ workers=workers,
43
+ timeout=timeout,
44
+ max_depth=max_depth,
45
+ check_external=check_external,
46
+ check_assets=check_assets,
47
+ broken_only=broken_only,
48
+ )
49
+
50
+ console = Console()
51
+ if not quiet:
52
+ console.print(
53
+ f"[bold cyan]Starting crawl.[/bold cyan] Strictly locked to paths under: [underline]{config.base_prefix}[/underline]"
54
+ )
55
+
56
+ def on_completed(res: LinkResult, completed: int, total: int) -> None:
57
+ if quiet:
58
+ return
59
+ status_str = str(res.status_code) if res.status_code else (res.error_type or "ERR")
60
+ if res.success == 0:
61
+ badge = f"[bold red]✗ {status_str}[/bold red]"
62
+ elif res.is_redirect:
63
+ badge = f"[bold yellow]↪ {status_str}[/bold yellow]"
64
+ else:
65
+ badge = f"[bold green]✓ {status_str}[/bold green]"
66
+ console.print(f"[{completed}/{total}] {badge} {res.link} [dim]({res.scope}, {res.resource_type}, {res.method_used})[/dim]")
67
+
68
+ crawler = SpiderCrawler(config=config, on_url_completed=on_completed)
69
+ summary = crawler.run()
70
+
71
+ saved_paths: list[Path] = []
72
+ for out_file in output_files:
73
+ saved = export_report(summary, out_file, fmt=fmt, broken_only=broken_only)
74
+ saved_paths.append(saved)
75
+
76
+ if not quiet:
77
+ _print_terminal_summary(console, summary, saved_paths)
78
+
79
+ return summary
80
+
81
+
82
+ def _print_terminal_summary(console: Console, summary: CrawlSummary, saved_paths: list[Path]) -> None:
83
+ console.print()
84
+ score_table = Table(title="Crawl Summary Scorecard", show_header=True, header_style="bold cyan")
85
+ score_table.add_column("Total Checked", justify="right")
86
+ score_table.add_column("Valid", justify="right", style="green")
87
+ score_table.add_column("Broken", justify="right", style="red")
88
+ score_table.add_column("Redirects", justify="right", style="yellow")
89
+ score_table.add_column("External", justify="right")
90
+ score_table.add_column("Docs & Assets", justify="right")
91
+ score_table.add_column("Duration", justify="right")
92
+
93
+ score_table.add_row(
94
+ str(summary.total_checked),
95
+ str(summary.valid_count),
96
+ str(summary.broken_count),
97
+ str(summary.redirect_count),
98
+ str(summary.external_count),
99
+ str(summary.document_count + summary.asset_count),
100
+ f"{summary.duration_seconds:.2f}s",
101
+ )
102
+ console.print(score_table)
103
+
104
+ broken = summary.broken_results
105
+ if broken:
106
+ console.print()
107
+ broken_table = Table(
108
+ title=f"Broken Links ({len(broken)})",
109
+ show_header=True,
110
+ header_style="bold red",
111
+ show_lines=True,
112
+ )
113
+ broken_table.add_column("Status", style="bold red", width=10)
114
+ broken_table.add_column("Broken URL", overflow="fold")
115
+ broken_table.add_column("Type", width=10)
116
+ broken_table.add_column("Found On (Referrer Pages)", overflow="fold")
117
+ broken_table.add_column("Link Text / Alt", overflow="fold")
118
+
119
+ for r in broken:
120
+ status_label = str(r.status_code) if r.status_code else (r.error_type or "ERR")
121
+ referrers = "\n".join(r.found_on) if r.found_on else "(Start URL)"
122
+ texts = ", ".join(r.link_texts) if r.link_texts else "—"
123
+ broken_table.add_row(status_label, r.link, r.resource_type, referrers, texts)
124
+ console.print(broken_table)
125
+
126
+ for path in saved_paths:
127
+ console.print(f"[bold green]Saved report:[/bold green] {path}")
128
+
129
+
130
+ def build_parser() -> argparse.ArgumentParser:
131
+ parser = argparse.ArgumentParser(
132
+ prog="JumpingSpider",
133
+ description="JumpingSpider: Concurrent web crawler, broken link detector, and asset validator.",
134
+ )
135
+ parser.add_argument(
136
+ "start_url",
137
+ nargs="?",
138
+ help="Starting URL to crawl (strictly locked to paths under this directory).",
139
+ )
140
+ parser.add_argument(
141
+ "legacy_output",
142
+ nargs="?",
143
+ help="Optional positional output filename (for backwards compatibility).",
144
+ )
145
+ parser.add_argument(
146
+ "-o",
147
+ "--output",
148
+ action="append",
149
+ dest="outputs",
150
+ help="Output report file path (.json, .csv, .md, .html). Can be specified multiple times.",
151
+ )
152
+ parser.add_argument(
153
+ "-f",
154
+ "--format",
155
+ choices=["json", "csv", "md", "html"],
156
+ default=None,
157
+ help="Explicit report format override (otherwise inferred from file extension).",
158
+ )
159
+ parser.add_argument(
160
+ "--tui",
161
+ action="store_true",
162
+ help="Launch the interactive Terminal User Interface (TUI).",
163
+ )
164
+ parser.add_argument(
165
+ "--check-external",
166
+ action=argparse.BooleanOptionalAction,
167
+ default=True,
168
+ help="Validate external and sibling links without crawling them (default: True).",
169
+ )
170
+ parser.add_argument(
171
+ "--no-external",
172
+ action="store_false",
173
+ dest="check_external",
174
+ help="Disable external and sibling link validation.",
175
+ )
176
+ parser.add_argument(
177
+ "--check-assets",
178
+ action=argparse.BooleanOptionalAction,
179
+ default=True,
180
+ help="Validate page assets (<img>, <script>, <link>, <video>, <iframe>) (default: True).",
181
+ )
182
+ parser.add_argument(
183
+ "--no-assets",
184
+ action="store_false",
185
+ dest="check_assets",
186
+ help="Disable page asset validation.",
187
+ )
188
+ parser.add_argument(
189
+ "-w",
190
+ "--workers",
191
+ type=int,
192
+ default=5,
193
+ help="Number of concurrent worker threads (default: 5).",
194
+ )
195
+ parser.add_argument(
196
+ "-d",
197
+ "--delay",
198
+ type=float,
199
+ default=0.2,
200
+ help="Delay in seconds between requests (default: 0.2).",
201
+ )
202
+ parser.add_argument(
203
+ "--timeout",
204
+ type=float,
205
+ default=10.0,
206
+ help="HTTP request timeout in seconds (default: 10.0).",
207
+ )
208
+ parser.add_argument(
209
+ "--max-depth",
210
+ type=int,
211
+ default=None,
212
+ help="Maximum link depth to crawl for internal pages (default: unlimited).",
213
+ )
214
+ parser.add_argument(
215
+ "--broken-only",
216
+ action="store_true",
217
+ help="Only include broken links in exported reports.",
218
+ )
219
+ parser.add_argument(
220
+ "-q",
221
+ "--quiet",
222
+ action="store_true",
223
+ help="Suppress live console output.",
224
+ )
225
+ return parser
226
+
227
+
228
+ def main(argv: Sequence[str] | None = None) -> int:
229
+ parser = build_parser()
230
+ args = parser.parse_args(argv)
231
+
232
+ output_files: list[str] = []
233
+ if args.legacy_output:
234
+ output_files.append(args.legacy_output)
235
+ if args.outputs:
236
+ output_files.extend(args.outputs)
237
+ if not output_files:
238
+ output_files = ["crawl_results.json"]
239
+
240
+ if args.tui:
241
+ from JumpingSpider.tui import JumpingSpiderApp
242
+
243
+ initial_config = CrawlConfig(
244
+ start_url=args.start_url or "https://weather.ndc.nasa.gov/tempo/",
245
+ output_files=output_files,
246
+ delay=args.delay,
247
+ workers=args.workers,
248
+ timeout=args.timeout,
249
+ max_depth=args.max_depth,
250
+ check_external=args.check_external,
251
+ check_assets=args.check_assets,
252
+ broken_only=args.broken_only,
253
+ )
254
+ app = JumpingSpiderApp(initial_config=initial_config)
255
+ app.run()
256
+ return 0
257
+
258
+ if not args.start_url:
259
+ parser.print_help()
260
+ return 1
261
+
262
+ crawl_site(
263
+ start_url=args.start_url,
264
+ output_filename=output_files,
265
+ delay=args.delay,
266
+ workers=args.workers,
267
+ timeout=args.timeout,
268
+ max_depth=args.max_depth,
269
+ check_external=args.check_external,
270
+ check_assets=args.check_assets,
271
+ broken_only=args.broken_only,
272
+ fmt=args.format,
273
+ quiet=args.quiet,
274
+ )
275
+ return 0
276
+
277
+
278
+ if __name__ == "__main__":
279
+ sys.exit(main())
@@ -0,0 +1,178 @@
1
+ from __future__ import annotations
2
+
3
+ from dataclasses import dataclass, field
4
+ from typing import Any
5
+
6
+
7
+ @dataclass
8
+ class LinkReference:
9
+ """Represents a single occurrence where a URL was referenced on a crawled page."""
10
+
11
+ source_url: str
12
+ tag: str = "a"
13
+ attribute: str = "href"
14
+ text: str = ""
15
+
16
+ def to_dict(self) -> dict[str, str]:
17
+ return {
18
+ "source_url": self.source_url,
19
+ "tag": self.tag,
20
+ "attribute": self.attribute,
21
+ "text": self.text,
22
+ }
23
+
24
+
25
+ @dataclass
26
+ class LinkResult:
27
+ """Stores the validation result and metadata for a single discovered URL."""
28
+
29
+ link: str
30
+ success: int
31
+ status_code: int
32
+ final_url: str = ""
33
+ is_redirect: bool = False
34
+ redirect_chain: list[str] = field(default_factory=list)
35
+ error_type: str | None = None
36
+ error_message: str | None = None
37
+ content_type: str = ""
38
+ scope: str = "internal" # "internal" | "external"
39
+ resource_type: str = "page" # "page" | "document" | "asset"
40
+ method_used: str = "GET" # "GET" | "HEAD"
41
+ response_time_ms: float = 0.0
42
+ depth: int = 0
43
+ references: list[LinkReference] = field(default_factory=list)
44
+
45
+ @property
46
+ def found_on(self) -> list[str]:
47
+ """Unique ordered list of source URLs where this link was found."""
48
+ seen: set[str] = set()
49
+ pages: list[str] = []
50
+ for ref in self.references:
51
+ if ref.source_url and ref.source_url not in seen:
52
+ seen.add(ref.source_url)
53
+ pages.append(ref.source_url)
54
+ return pages
55
+
56
+ @property
57
+ def link_texts(self) -> list[str]:
58
+ """Unique ordered list of non-empty anchor/alt texts associated with this link."""
59
+ seen: set[str] = set()
60
+ texts: list[str] = []
61
+ for ref in self.references:
62
+ cleaned = ref.text.strip()
63
+ if cleaned and cleaned not in seen:
64
+ seen.add(cleaned)
65
+ texts.append(cleaned)
66
+ return texts
67
+
68
+ def add_reference(self, ref: LinkReference) -> None:
69
+ """Adds a reference if an identical one is not already recorded."""
70
+ for existing in self.references:
71
+ if (
72
+ existing.source_url == ref.source_url
73
+ and existing.tag == ref.tag
74
+ and existing.attribute == ref.attribute
75
+ and existing.text == ref.text
76
+ ):
77
+ return
78
+ self.references.append(ref)
79
+
80
+ def to_dict(self) -> dict[str, Any]:
81
+ return {
82
+ "link": self.link,
83
+ "success": self.success,
84
+ "status_code": self.status_code,
85
+ "final_url": self.final_url or self.link,
86
+ "is_redirect": self.is_redirect,
87
+ "redirect_chain": list(self.redirect_chain),
88
+ "error_type": self.error_type,
89
+ "error_message": self.error_message,
90
+ "content_type": self.content_type,
91
+ "scope": self.scope,
92
+ "resource_type": self.resource_type,
93
+ "method_used": self.method_used,
94
+ "response_time_ms": round(self.response_time_ms, 2),
95
+ "depth": self.depth,
96
+ "found_on": self.found_on,
97
+ "link_texts": self.link_texts,
98
+ "references": [r.to_dict() for r in self.references],
99
+ }
100
+
101
+
102
+ @dataclass
103
+ class CrawlConfig:
104
+ """Configuration options for a SpiderCrawler run."""
105
+
106
+ start_url: str
107
+ output_files: list[str] = field(default_factory=lambda: ["crawl_results.json"])
108
+ delay: float = 0.2
109
+ workers: int = 5
110
+ timeout: float = 10.0
111
+ max_depth: int | None = None
112
+ check_external: bool = True
113
+ check_assets: bool = True
114
+ broken_only: bool = False
115
+ user_agent: str = "JumpingSpider/0.2.0 (+https://github.com/rogerallen/JumpingSpider)"
116
+
117
+ @property
118
+ def base_prefix(self) -> str:
119
+ """Strict directory prefix boundary for crawling internal pages."""
120
+ url = self.start_url
121
+ if url.endswith("/"):
122
+ return url
123
+ # If the last path segment looks like a file (e.g. /tempo/index.html), use its directory
124
+ from urllib.parse import urlparse
125
+
126
+ parsed = urlparse(url)
127
+ path = parsed.path
128
+ last_segment = path.rsplit("/", 1)[-1] if "/" in path else path
129
+ if "." in last_segment:
130
+ parent_path = path.rsplit("/", 1)[0] + "/"
131
+ return f"{parsed.scheme}://{parsed.netloc}{parent_path}"
132
+ return url + "/"
133
+
134
+
135
+ @dataclass
136
+ class CrawlSummary:
137
+ """Aggregated results and metrics from a crawl session."""
138
+
139
+ start_url: str
140
+ base_prefix: str
141
+ results: list[LinkResult] = field(default_factory=list)
142
+ duration_seconds: float = 0.0
143
+
144
+ @property
145
+ def total_checked(self) -> int:
146
+ return len(self.results)
147
+
148
+ @property
149
+ def valid_count(self) -> int:
150
+ return sum(1 for r in self.results if r.success == 1)
151
+
152
+ @property
153
+ def broken_count(self) -> int:
154
+ return sum(1 for r in self.results if r.success == 0)
155
+
156
+ @property
157
+ def redirect_count(self) -> int:
158
+ return sum(1 for r in self.results if r.is_redirect)
159
+
160
+ @property
161
+ def external_count(self) -> int:
162
+ return sum(1 for r in self.results if r.scope == "external")
163
+
164
+ @property
165
+ def asset_count(self) -> int:
166
+ return sum(1 for r in self.results if r.resource_type == "asset")
167
+
168
+ @property
169
+ def document_count(self) -> int:
170
+ return sum(1 for r in self.results if r.resource_type == "document")
171
+
172
+ @property
173
+ def broken_results(self) -> list[LinkResult]:
174
+ return [r for r in self.results if r.success == 0]
175
+
176
+ @property
177
+ def redirect_results(self) -> list[LinkResult]:
178
+ return [r for r in self.results if r.is_redirect]
JumpingSpider/py.typed ADDED
File without changes