JumpingSpider 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- JumpingSpider/__init__.py +13 -0
- JumpingSpider/crawler.py +460 -0
- JumpingSpider/main.py +279 -0
- JumpingSpider/models.py +178 -0
- JumpingSpider/py.typed +0 -0
- JumpingSpider/reporters.py +281 -0
- JumpingSpider/tui.py +377 -0
- jumpingspider-0.2.1.dist-info/METADATA +147 -0
- jumpingspider-0.2.1.dist-info/RECORD +12 -0
- jumpingspider-0.2.1.dist-info/WHEEL +4 -0
- jumpingspider-0.2.1.dist-info/entry_points.txt +4 -0
- jumpingspider-0.2.1.dist-info/licenses/LICENSE +21 -0
JumpingSpider/main.py
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import sys
|
|
5
|
+
from collections.abc import Sequence
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
|
|
11
|
+
from JumpingSpider.crawler import SpiderCrawler
|
|
12
|
+
from JumpingSpider.models import CrawlConfig, CrawlSummary, LinkResult
|
|
13
|
+
from JumpingSpider.reporters import export_report
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def crawl_site(
|
|
17
|
+
start_url: str,
|
|
18
|
+
output_filename: str | Sequence[str] = "crawl_results.json",
|
|
19
|
+
delay: float = 0.2,
|
|
20
|
+
workers: int = 5,
|
|
21
|
+
timeout: float = 10.0,
|
|
22
|
+
max_depth: int | None = None,
|
|
23
|
+
check_external: bool = True,
|
|
24
|
+
check_assets: bool = True,
|
|
25
|
+
broken_only: bool = False,
|
|
26
|
+
fmt: str | None = None,
|
|
27
|
+
quiet: bool = False,
|
|
28
|
+
) -> CrawlSummary:
|
|
29
|
+
"""
|
|
30
|
+
Crawls a website starting at `start_url`, strictly staying within its path boundary
|
|
31
|
+
for recursion while optionally validating external links and page assets.
|
|
32
|
+
"""
|
|
33
|
+
if isinstance(output_filename, str):
|
|
34
|
+
output_files = [output_filename] if output_filename else []
|
|
35
|
+
else:
|
|
36
|
+
output_files = list(output_filename)
|
|
37
|
+
|
|
38
|
+
config = CrawlConfig(
|
|
39
|
+
start_url=start_url,
|
|
40
|
+
output_files=output_files,
|
|
41
|
+
delay=delay,
|
|
42
|
+
workers=workers,
|
|
43
|
+
timeout=timeout,
|
|
44
|
+
max_depth=max_depth,
|
|
45
|
+
check_external=check_external,
|
|
46
|
+
check_assets=check_assets,
|
|
47
|
+
broken_only=broken_only,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
console = Console()
|
|
51
|
+
if not quiet:
|
|
52
|
+
console.print(
|
|
53
|
+
f"[bold cyan]Starting crawl.[/bold cyan] Strictly locked to paths under: [underline]{config.base_prefix}[/underline]"
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
def on_completed(res: LinkResult, completed: int, total: int) -> None:
|
|
57
|
+
if quiet:
|
|
58
|
+
return
|
|
59
|
+
status_str = str(res.status_code) if res.status_code else (res.error_type or "ERR")
|
|
60
|
+
if res.success == 0:
|
|
61
|
+
badge = f"[bold red]✗ {status_str}[/bold red]"
|
|
62
|
+
elif res.is_redirect:
|
|
63
|
+
badge = f"[bold yellow]↪ {status_str}[/bold yellow]"
|
|
64
|
+
else:
|
|
65
|
+
badge = f"[bold green]✓ {status_str}[/bold green]"
|
|
66
|
+
console.print(f"[{completed}/{total}] {badge} {res.link} [dim]({res.scope}, {res.resource_type}, {res.method_used})[/dim]")
|
|
67
|
+
|
|
68
|
+
crawler = SpiderCrawler(config=config, on_url_completed=on_completed)
|
|
69
|
+
summary = crawler.run()
|
|
70
|
+
|
|
71
|
+
saved_paths: list[Path] = []
|
|
72
|
+
for out_file in output_files:
|
|
73
|
+
saved = export_report(summary, out_file, fmt=fmt, broken_only=broken_only)
|
|
74
|
+
saved_paths.append(saved)
|
|
75
|
+
|
|
76
|
+
if not quiet:
|
|
77
|
+
_print_terminal_summary(console, summary, saved_paths)
|
|
78
|
+
|
|
79
|
+
return summary
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _print_terminal_summary(console: Console, summary: CrawlSummary, saved_paths: list[Path]) -> None:
|
|
83
|
+
console.print()
|
|
84
|
+
score_table = Table(title="Crawl Summary Scorecard", show_header=True, header_style="bold cyan")
|
|
85
|
+
score_table.add_column("Total Checked", justify="right")
|
|
86
|
+
score_table.add_column("Valid", justify="right", style="green")
|
|
87
|
+
score_table.add_column("Broken", justify="right", style="red")
|
|
88
|
+
score_table.add_column("Redirects", justify="right", style="yellow")
|
|
89
|
+
score_table.add_column("External", justify="right")
|
|
90
|
+
score_table.add_column("Docs & Assets", justify="right")
|
|
91
|
+
score_table.add_column("Duration", justify="right")
|
|
92
|
+
|
|
93
|
+
score_table.add_row(
|
|
94
|
+
str(summary.total_checked),
|
|
95
|
+
str(summary.valid_count),
|
|
96
|
+
str(summary.broken_count),
|
|
97
|
+
str(summary.redirect_count),
|
|
98
|
+
str(summary.external_count),
|
|
99
|
+
str(summary.document_count + summary.asset_count),
|
|
100
|
+
f"{summary.duration_seconds:.2f}s",
|
|
101
|
+
)
|
|
102
|
+
console.print(score_table)
|
|
103
|
+
|
|
104
|
+
broken = summary.broken_results
|
|
105
|
+
if broken:
|
|
106
|
+
console.print()
|
|
107
|
+
broken_table = Table(
|
|
108
|
+
title=f"Broken Links ({len(broken)})",
|
|
109
|
+
show_header=True,
|
|
110
|
+
header_style="bold red",
|
|
111
|
+
show_lines=True,
|
|
112
|
+
)
|
|
113
|
+
broken_table.add_column("Status", style="bold red", width=10)
|
|
114
|
+
broken_table.add_column("Broken URL", overflow="fold")
|
|
115
|
+
broken_table.add_column("Type", width=10)
|
|
116
|
+
broken_table.add_column("Found On (Referrer Pages)", overflow="fold")
|
|
117
|
+
broken_table.add_column("Link Text / Alt", overflow="fold")
|
|
118
|
+
|
|
119
|
+
for r in broken:
|
|
120
|
+
status_label = str(r.status_code) if r.status_code else (r.error_type or "ERR")
|
|
121
|
+
referrers = "\n".join(r.found_on) if r.found_on else "(Start URL)"
|
|
122
|
+
texts = ", ".join(r.link_texts) if r.link_texts else "—"
|
|
123
|
+
broken_table.add_row(status_label, r.link, r.resource_type, referrers, texts)
|
|
124
|
+
console.print(broken_table)
|
|
125
|
+
|
|
126
|
+
for path in saved_paths:
|
|
127
|
+
console.print(f"[bold green]Saved report:[/bold green] {path}")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
131
|
+
parser = argparse.ArgumentParser(
|
|
132
|
+
prog="JumpingSpider",
|
|
133
|
+
description="JumpingSpider: Concurrent web crawler, broken link detector, and asset validator.",
|
|
134
|
+
)
|
|
135
|
+
parser.add_argument(
|
|
136
|
+
"start_url",
|
|
137
|
+
nargs="?",
|
|
138
|
+
help="Starting URL to crawl (strictly locked to paths under this directory).",
|
|
139
|
+
)
|
|
140
|
+
parser.add_argument(
|
|
141
|
+
"legacy_output",
|
|
142
|
+
nargs="?",
|
|
143
|
+
help="Optional positional output filename (for backwards compatibility).",
|
|
144
|
+
)
|
|
145
|
+
parser.add_argument(
|
|
146
|
+
"-o",
|
|
147
|
+
"--output",
|
|
148
|
+
action="append",
|
|
149
|
+
dest="outputs",
|
|
150
|
+
help="Output report file path (.json, .csv, .md, .html). Can be specified multiple times.",
|
|
151
|
+
)
|
|
152
|
+
parser.add_argument(
|
|
153
|
+
"-f",
|
|
154
|
+
"--format",
|
|
155
|
+
choices=["json", "csv", "md", "html"],
|
|
156
|
+
default=None,
|
|
157
|
+
help="Explicit report format override (otherwise inferred from file extension).",
|
|
158
|
+
)
|
|
159
|
+
parser.add_argument(
|
|
160
|
+
"--tui",
|
|
161
|
+
action="store_true",
|
|
162
|
+
help="Launch the interactive Terminal User Interface (TUI).",
|
|
163
|
+
)
|
|
164
|
+
parser.add_argument(
|
|
165
|
+
"--check-external",
|
|
166
|
+
action=argparse.BooleanOptionalAction,
|
|
167
|
+
default=True,
|
|
168
|
+
help="Validate external and sibling links without crawling them (default: True).",
|
|
169
|
+
)
|
|
170
|
+
parser.add_argument(
|
|
171
|
+
"--no-external",
|
|
172
|
+
action="store_false",
|
|
173
|
+
dest="check_external",
|
|
174
|
+
help="Disable external and sibling link validation.",
|
|
175
|
+
)
|
|
176
|
+
parser.add_argument(
|
|
177
|
+
"--check-assets",
|
|
178
|
+
action=argparse.BooleanOptionalAction,
|
|
179
|
+
default=True,
|
|
180
|
+
help="Validate page assets (<img>, <script>, <link>, <video>, <iframe>) (default: True).",
|
|
181
|
+
)
|
|
182
|
+
parser.add_argument(
|
|
183
|
+
"--no-assets",
|
|
184
|
+
action="store_false",
|
|
185
|
+
dest="check_assets",
|
|
186
|
+
help="Disable page asset validation.",
|
|
187
|
+
)
|
|
188
|
+
parser.add_argument(
|
|
189
|
+
"-w",
|
|
190
|
+
"--workers",
|
|
191
|
+
type=int,
|
|
192
|
+
default=5,
|
|
193
|
+
help="Number of concurrent worker threads (default: 5).",
|
|
194
|
+
)
|
|
195
|
+
parser.add_argument(
|
|
196
|
+
"-d",
|
|
197
|
+
"--delay",
|
|
198
|
+
type=float,
|
|
199
|
+
default=0.2,
|
|
200
|
+
help="Delay in seconds between requests (default: 0.2).",
|
|
201
|
+
)
|
|
202
|
+
parser.add_argument(
|
|
203
|
+
"--timeout",
|
|
204
|
+
type=float,
|
|
205
|
+
default=10.0,
|
|
206
|
+
help="HTTP request timeout in seconds (default: 10.0).",
|
|
207
|
+
)
|
|
208
|
+
parser.add_argument(
|
|
209
|
+
"--max-depth",
|
|
210
|
+
type=int,
|
|
211
|
+
default=None,
|
|
212
|
+
help="Maximum link depth to crawl for internal pages (default: unlimited).",
|
|
213
|
+
)
|
|
214
|
+
parser.add_argument(
|
|
215
|
+
"--broken-only",
|
|
216
|
+
action="store_true",
|
|
217
|
+
help="Only include broken links in exported reports.",
|
|
218
|
+
)
|
|
219
|
+
parser.add_argument(
|
|
220
|
+
"-q",
|
|
221
|
+
"--quiet",
|
|
222
|
+
action="store_true",
|
|
223
|
+
help="Suppress live console output.",
|
|
224
|
+
)
|
|
225
|
+
return parser
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
229
|
+
parser = build_parser()
|
|
230
|
+
args = parser.parse_args(argv)
|
|
231
|
+
|
|
232
|
+
output_files: list[str] = []
|
|
233
|
+
if args.legacy_output:
|
|
234
|
+
output_files.append(args.legacy_output)
|
|
235
|
+
if args.outputs:
|
|
236
|
+
output_files.extend(args.outputs)
|
|
237
|
+
if not output_files:
|
|
238
|
+
output_files = ["crawl_results.json"]
|
|
239
|
+
|
|
240
|
+
if args.tui:
|
|
241
|
+
from JumpingSpider.tui import JumpingSpiderApp
|
|
242
|
+
|
|
243
|
+
initial_config = CrawlConfig(
|
|
244
|
+
start_url=args.start_url or "https://weather.ndc.nasa.gov/tempo/",
|
|
245
|
+
output_files=output_files,
|
|
246
|
+
delay=args.delay,
|
|
247
|
+
workers=args.workers,
|
|
248
|
+
timeout=args.timeout,
|
|
249
|
+
max_depth=args.max_depth,
|
|
250
|
+
check_external=args.check_external,
|
|
251
|
+
check_assets=args.check_assets,
|
|
252
|
+
broken_only=args.broken_only,
|
|
253
|
+
)
|
|
254
|
+
app = JumpingSpiderApp(initial_config=initial_config)
|
|
255
|
+
app.run()
|
|
256
|
+
return 0
|
|
257
|
+
|
|
258
|
+
if not args.start_url:
|
|
259
|
+
parser.print_help()
|
|
260
|
+
return 1
|
|
261
|
+
|
|
262
|
+
crawl_site(
|
|
263
|
+
start_url=args.start_url,
|
|
264
|
+
output_filename=output_files,
|
|
265
|
+
delay=args.delay,
|
|
266
|
+
workers=args.workers,
|
|
267
|
+
timeout=args.timeout,
|
|
268
|
+
max_depth=args.max_depth,
|
|
269
|
+
check_external=args.check_external,
|
|
270
|
+
check_assets=args.check_assets,
|
|
271
|
+
broken_only=args.broken_only,
|
|
272
|
+
fmt=args.format,
|
|
273
|
+
quiet=args.quiet,
|
|
274
|
+
)
|
|
275
|
+
return 0
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
if __name__ == "__main__":
|
|
279
|
+
sys.exit(main())
|
JumpingSpider/models.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass, field
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass
|
|
8
|
+
class LinkReference:
|
|
9
|
+
"""Represents a single occurrence where a URL was referenced on a crawled page."""
|
|
10
|
+
|
|
11
|
+
source_url: str
|
|
12
|
+
tag: str = "a"
|
|
13
|
+
attribute: str = "href"
|
|
14
|
+
text: str = ""
|
|
15
|
+
|
|
16
|
+
def to_dict(self) -> dict[str, str]:
|
|
17
|
+
return {
|
|
18
|
+
"source_url": self.source_url,
|
|
19
|
+
"tag": self.tag,
|
|
20
|
+
"attribute": self.attribute,
|
|
21
|
+
"text": self.text,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class LinkResult:
|
|
27
|
+
"""Stores the validation result and metadata for a single discovered URL."""
|
|
28
|
+
|
|
29
|
+
link: str
|
|
30
|
+
success: int
|
|
31
|
+
status_code: int
|
|
32
|
+
final_url: str = ""
|
|
33
|
+
is_redirect: bool = False
|
|
34
|
+
redirect_chain: list[str] = field(default_factory=list)
|
|
35
|
+
error_type: str | None = None
|
|
36
|
+
error_message: str | None = None
|
|
37
|
+
content_type: str = ""
|
|
38
|
+
scope: str = "internal" # "internal" | "external"
|
|
39
|
+
resource_type: str = "page" # "page" | "document" | "asset"
|
|
40
|
+
method_used: str = "GET" # "GET" | "HEAD"
|
|
41
|
+
response_time_ms: float = 0.0
|
|
42
|
+
depth: int = 0
|
|
43
|
+
references: list[LinkReference] = field(default_factory=list)
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def found_on(self) -> list[str]:
|
|
47
|
+
"""Unique ordered list of source URLs where this link was found."""
|
|
48
|
+
seen: set[str] = set()
|
|
49
|
+
pages: list[str] = []
|
|
50
|
+
for ref in self.references:
|
|
51
|
+
if ref.source_url and ref.source_url not in seen:
|
|
52
|
+
seen.add(ref.source_url)
|
|
53
|
+
pages.append(ref.source_url)
|
|
54
|
+
return pages
|
|
55
|
+
|
|
56
|
+
@property
|
|
57
|
+
def link_texts(self) -> list[str]:
|
|
58
|
+
"""Unique ordered list of non-empty anchor/alt texts associated with this link."""
|
|
59
|
+
seen: set[str] = set()
|
|
60
|
+
texts: list[str] = []
|
|
61
|
+
for ref in self.references:
|
|
62
|
+
cleaned = ref.text.strip()
|
|
63
|
+
if cleaned and cleaned not in seen:
|
|
64
|
+
seen.add(cleaned)
|
|
65
|
+
texts.append(cleaned)
|
|
66
|
+
return texts
|
|
67
|
+
|
|
68
|
+
def add_reference(self, ref: LinkReference) -> None:
|
|
69
|
+
"""Adds a reference if an identical one is not already recorded."""
|
|
70
|
+
for existing in self.references:
|
|
71
|
+
if (
|
|
72
|
+
existing.source_url == ref.source_url
|
|
73
|
+
and existing.tag == ref.tag
|
|
74
|
+
and existing.attribute == ref.attribute
|
|
75
|
+
and existing.text == ref.text
|
|
76
|
+
):
|
|
77
|
+
return
|
|
78
|
+
self.references.append(ref)
|
|
79
|
+
|
|
80
|
+
def to_dict(self) -> dict[str, Any]:
|
|
81
|
+
return {
|
|
82
|
+
"link": self.link,
|
|
83
|
+
"success": self.success,
|
|
84
|
+
"status_code": self.status_code,
|
|
85
|
+
"final_url": self.final_url or self.link,
|
|
86
|
+
"is_redirect": self.is_redirect,
|
|
87
|
+
"redirect_chain": list(self.redirect_chain),
|
|
88
|
+
"error_type": self.error_type,
|
|
89
|
+
"error_message": self.error_message,
|
|
90
|
+
"content_type": self.content_type,
|
|
91
|
+
"scope": self.scope,
|
|
92
|
+
"resource_type": self.resource_type,
|
|
93
|
+
"method_used": self.method_used,
|
|
94
|
+
"response_time_ms": round(self.response_time_ms, 2),
|
|
95
|
+
"depth": self.depth,
|
|
96
|
+
"found_on": self.found_on,
|
|
97
|
+
"link_texts": self.link_texts,
|
|
98
|
+
"references": [r.to_dict() for r in self.references],
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass
|
|
103
|
+
class CrawlConfig:
|
|
104
|
+
"""Configuration options for a SpiderCrawler run."""
|
|
105
|
+
|
|
106
|
+
start_url: str
|
|
107
|
+
output_files: list[str] = field(default_factory=lambda: ["crawl_results.json"])
|
|
108
|
+
delay: float = 0.2
|
|
109
|
+
workers: int = 5
|
|
110
|
+
timeout: float = 10.0
|
|
111
|
+
max_depth: int | None = None
|
|
112
|
+
check_external: bool = True
|
|
113
|
+
check_assets: bool = True
|
|
114
|
+
broken_only: bool = False
|
|
115
|
+
user_agent: str = "JumpingSpider/0.2.0 (+https://github.com/rogerallen/JumpingSpider)"
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def base_prefix(self) -> str:
|
|
119
|
+
"""Strict directory prefix boundary for crawling internal pages."""
|
|
120
|
+
url = self.start_url
|
|
121
|
+
if url.endswith("/"):
|
|
122
|
+
return url
|
|
123
|
+
# If the last path segment looks like a file (e.g. /tempo/index.html), use its directory
|
|
124
|
+
from urllib.parse import urlparse
|
|
125
|
+
|
|
126
|
+
parsed = urlparse(url)
|
|
127
|
+
path = parsed.path
|
|
128
|
+
last_segment = path.rsplit("/", 1)[-1] if "/" in path else path
|
|
129
|
+
if "." in last_segment:
|
|
130
|
+
parent_path = path.rsplit("/", 1)[0] + "/"
|
|
131
|
+
return f"{parsed.scheme}://{parsed.netloc}{parent_path}"
|
|
132
|
+
return url + "/"
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@dataclass
|
|
136
|
+
class CrawlSummary:
|
|
137
|
+
"""Aggregated results and metrics from a crawl session."""
|
|
138
|
+
|
|
139
|
+
start_url: str
|
|
140
|
+
base_prefix: str
|
|
141
|
+
results: list[LinkResult] = field(default_factory=list)
|
|
142
|
+
duration_seconds: float = 0.0
|
|
143
|
+
|
|
144
|
+
@property
|
|
145
|
+
def total_checked(self) -> int:
|
|
146
|
+
return len(self.results)
|
|
147
|
+
|
|
148
|
+
@property
|
|
149
|
+
def valid_count(self) -> int:
|
|
150
|
+
return sum(1 for r in self.results if r.success == 1)
|
|
151
|
+
|
|
152
|
+
@property
|
|
153
|
+
def broken_count(self) -> int:
|
|
154
|
+
return sum(1 for r in self.results if r.success == 0)
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def redirect_count(self) -> int:
|
|
158
|
+
return sum(1 for r in self.results if r.is_redirect)
|
|
159
|
+
|
|
160
|
+
@property
|
|
161
|
+
def external_count(self) -> int:
|
|
162
|
+
return sum(1 for r in self.results if r.scope == "external")
|
|
163
|
+
|
|
164
|
+
@property
|
|
165
|
+
def asset_count(self) -> int:
|
|
166
|
+
return sum(1 for r in self.results if r.resource_type == "asset")
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def document_count(self) -> int:
|
|
170
|
+
return sum(1 for r in self.results if r.resource_type == "document")
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def broken_results(self) -> list[LinkResult]:
|
|
174
|
+
return [r for r in self.results if r.success == 0]
|
|
175
|
+
|
|
176
|
+
@property
|
|
177
|
+
def redirect_results(self) -> list[LinkResult]:
|
|
178
|
+
return [r for r in self.results if r.is_redirect]
|
JumpingSpider/py.typed
ADDED
|
File without changes
|