JumpingSpider 0.2.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. jumpingspider-0.2.1/LICENSE +21 -0
  2. jumpingspider-0.2.1/PKG-INFO +147 -0
  3. jumpingspider-0.2.1/README.md +118 -0
  4. jumpingspider-0.2.1/pyproject.toml +75 -0
  5. jumpingspider-0.2.1/pyproject.toml.orig +65 -0
  6. jumpingspider-0.2.1/src/JumpingSpider/__init__.py +13 -0
  7. jumpingspider-0.2.1/src/JumpingSpider/crawler.py +460 -0
  8. jumpingspider-0.2.1/src/JumpingSpider/main.py +279 -0
  9. jumpingspider-0.2.1/src/JumpingSpider/models.py +178 -0
  10. jumpingspider-0.2.1/src/JumpingSpider/py.typed +0 -0
  11. jumpingspider-0.2.1/src/JumpingSpider/reporters.py +281 -0
  12. jumpingspider-0.2.1/src/JumpingSpider/tui.py +377 -0
jumpingspider-0.2.1/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Roger Allen
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
jumpingspider-0.2.1/PKG-INFO ADDED
@@ -0,0 +1,147 @@
1
+ Metadata-Version: 2.4
2
+ Name: JumpingSpider
3
+ Version: 0.2.1
4
+ Summary: Concurrent path-bounded web crawler, broken link detector, and asset validator with multi-format reports and Textual TUI
5
+ Keywords: crawler,spider,broken-links,link-checker,tui,textual,seo,site-audit
6
+ Author: Roger Allen
7
+ Author-email: Roger Allen <roger.allen795@gmail.com>
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Environment :: Console
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Intended Audience :: System Administrators
14
+ Classifier: Operating System :: OS Independent
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking
18
+ Classifier: Topic :: Utilities
19
+ Classifier: Typing :: Typed
20
+ Requires-Dist: beautifulsoup4>=4.15.0
21
+ Requires-Dist: requests>=2.34.2
22
+ Requires-Dist: textual>=8.2.8
23
+ Requires-Python: >=3.13
24
+ Project-URL: Homepage, https://github.com/Hailstone-Labs/JumpingSpider
25
+ Project-URL: Repository, https://github.com/Hailstone-Labs/JumpingSpider
26
+ Project-URL: Issues, https://github.com/Hailstone-Labs/JumpingSpider/issues
27
+ Project-URL: Changelog, https://github.com/Hailstone-Labs/JumpingSpider/blob/main/CHANGELOG.md
28
+ Description-Content-Type: text/markdown
29
+
30
+ # JumpingSpider
31
+
32
+ [![PyPI version](https://img.shields.io/pypi/v/JumpingSpider.svg)](https://pypi.org/project/JumpingSpider/)
33
+ [![CI](https://github.com/Hailstone-Labs/JumpingSpider/actions/workflows/ci.yml/badge.svg)](https://github.com/Hailstone-Labs/JumpingSpider/actions/workflows/ci.yml)
34
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
35
+ [![Python 3.13+](https://img.shields.io/badge/python-3.13%2B-blue.svg)](https://www.python.org/downloads/)
36
+
37
+ **JumpingSpider** (`v0.2.1`) is a concurrent web crawler and broken link/asset validator with multi-format reporting (`.json`, `.csv`, `.md`, `.html`) and an interactive Terminal User Interface (TUI).
38
+
39
+ ## Features
40
+
41
+ - **Strict Path Boundary Crawling**: Recursively crawls HTML pages under the starting URL path while preventing upward or parallel directory escapes.
42
+ - **Referrer & Anchor Text Tracking**: Records every source page (`found_on`), HTML tag (`<a>`, `<img>`, `<script>`, `<link>`, `<video>`, `<iframe>`), and link/alt text for every discovered URL so you know exactly which file to update.
43
+ - **External & Sibling Link Validation**: Validates links outside the crawl boundary without crawling into external sites.
44
+ - **Smart `HEAD` & Streaming Requests**: Uses `HEAD` requests (with automatic streaming `GET` fallback on `405`/`403`) for documents (`.pdf`, `.pptx`, `.xlsx`, `.mp4`, etc.), static assets, and external links to avoid downloading large binary files into memory.
45
+ - **Redirect & Error Diagnostics**: Captures full redirect chains (`301`/`302` -> final URL) and categorizes failures (`Timeout`, `SSLError`, `ConnectionError`, `HTTPError`).
46
+ - **Multi-Format Reports**: Export results to JSON, CSV, Markdown, or styled HTML.
47
+ - **Interactive TUI (`--tui`)**: Built with `textual` for live crawl monitoring, filtering (`All`, `Broken`, `Redirects`, `External`, `Assets/Docs`), referrer inspection, and one-key report exports.
48
+
49
+ ## Installation
50
+
51
+ Requires Python 3.13+.
52
+
53
+ ### From PyPI (Recommended)
54
+
55
+ ```bash
56
+ # Install globally as an isolated CLI tool with uv
57
+ uv tool install JumpingSpider
58
+
59
+ # Or run directly without installing using uvx
60
+ uvx JumpingSpider https://example.com/docs/
61
+
62
+ # Or install with pip / pipx
63
+ pip install JumpingSpider
64
+ ```
65
+
66
+ ### From Source (Development)
67
+
68
+ Requires [uv](https://docs.astral.sh/uv/):
69
+
70
+ ```bash
71
+ git clone https://github.com/Hailstone-Labs/JumpingSpider.git
72
+ cd JumpingSpider
73
+ uv sync --all-groups
74
+ ```
75
+
76
+ ## Usage
77
+
78
+ ### CLI Mode
79
+
80
+ ```bash
81
+ # Basic crawl (outputs crawl_results.json and prints a terminal summary scorecard)
82
+ uv run JumpingSpider https://weather.ndc.nasa.gov/tempo/
83
+
84
+ # Export multiple report formats at once
85
+ uv run JumpingSpider https://weather.ndc.nasa.gov/tempo/ \
86
+ -o tempo_results.json \
87
+ -o tempo_broken.csv \
88
+ -o tempo_report.md \
89
+ -o tempo_report.html
90
+
91
+ # Export only broken links to CSV
92
+ uv run JumpingSpider https://weather.ndc.nasa.gov/tempo/ -o broken_links.csv --broken-only
93
+
94
+ # Customize concurrency, rate limit delay, and depth
95
+ uv run JumpingSpider https://example.com/docs/ -w 8 -d 0.1 --max-depth 3
96
+
97
+ # Only check internal links and skip page assets
98
+ uv run JumpingSpider https://example.com/docs/ --no-external --no-assets
99
+ ```
100
+
101
+ ### CLI Options
102
+
103
+ | Option | Default | Description |
104
+ | :--- | :--- | :--- |
105
+ | `start_url` | — | Starting URL to crawl (strictly locked to paths under this directory). |
106
+ | `-o, --output <path>` | `crawl_results.json` | Output file path (`.json`, `.csv`, `.md`, `.html`). Can be passed multiple times. |
107
+ | `-f, --format <fmt>` | Auto from extension | Explicit report format override (`json`, `csv`, `md`, `html`). |
108
+ | `--tui` | `False` | Launch the interactive Terminal User Interface (TUI). |
109
+ | `--check-external` / `--no-external` | `True` | Validate external and sibling links without crawling them. |
110
+ | `--check-assets` / `--no-assets` | `True` | Validate page assets (`<img>`, `<script>`, `<link>`, `<video>`, `<iframe>`). |
111
+ | `-w, --workers <int>` | `5` | Number of concurrent worker threads. |
112
+ | `-d, --delay <float>` | `0.2` | Delay in seconds between HTTP requests. |
113
+ | `--timeout <float>` | `10.0` | HTTP request timeout in seconds. |
114
+ | `--max-depth <int>` | Unlimited | Maximum link depth to crawl for internal pages. |
115
+ | `--broken-only` | `False` | Only include broken links in exported reports. |
116
+ | `-q, --quiet` | `False` | Suppress live console progress and summary tables. |
117
+
118
+ ### Interactive TUI Mode
119
+
120
+ ```bash
121
+ uv run JumpingSpider --tui https://weather.ndc.nasa.gov/tempo/
122
+ ```
123
+
124
+ **TUI Keyboard Shortcuts:**
125
+ - `Ctrl+S`: Start crawl
126
+ - `Ctrl+X`: Stop crawl
127
+ - `1`–`5`: Filter table (`1` All, `2` Broken, `3` Redirects, `4` External, `5` Assets/Docs)
128
+ - `e`: Export all report formats (`crawl_results.{json,csv,md,html}`)
129
+ - `q`: Quit
130
+
131
+ ## Development & Tests
132
+
133
+ ```bash
134
+ # Run unit tests
135
+ uv run pytest -v
136
+
137
+ # Run Ruff linter
138
+ uv run ruff check .
139
+ ```
140
+
141
+ ## Contributing
142
+
143
+ Contributions are welcome! Please read our [Contributing Guide](CONTRIBUTING.md) and [Code of Conduct](CODE_OF_CONDUCT.md) before submitting a pull request. For security issues, please refer to our [Security Policy](SECURITY.md).
144
+
145
+ ## License
146
+
147
+ Distributed under the MIT License. See [`LICENSE`](LICENSE) for full details.
jumpingspider-0.2.1/README.md ADDED
@@ -0,0 +1,118 @@
1
+ # JumpingSpider
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/JumpingSpider.svg)](https://pypi.org/project/JumpingSpider/)
4
+ [![CI](https://github.com/Hailstone-Labs/JumpingSpider/actions/workflows/ci.yml/badge.svg)](https://github.com/Hailstone-Labs/JumpingSpider/actions/workflows/ci.yml)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
6
+ [![Python 3.13+](https://img.shields.io/badge/python-3.13%2B-blue.svg)](https://www.python.org/downloads/)
7
+
8
+ **JumpingSpider** (`v0.2.1`) is a concurrent web crawler and broken link/asset validator with multi-format reporting (`.json`, `.csv`, `.md`, `.html`) and an interactive Terminal User Interface (TUI).
9
+
10
+ ## Features
11
+
12
+ - **Strict Path Boundary Crawling**: Recursively crawls HTML pages under the starting URL path while preventing upward or parallel directory escapes.
13
+ - **Referrer & Anchor Text Tracking**: Records every source page (`found_on`), HTML tag (`<a>`, `<img>`, `<script>`, `<link>`, `<video>`, `<iframe>`), and link/alt text for every discovered URL so you know exactly which file to update.
14
+ - **External & Sibling Link Validation**: Validates links outside the crawl boundary without crawling into external sites.
15
+ - **Smart `HEAD` & Streaming Requests**: Uses `HEAD` requests (with automatic streaming `GET` fallback on `405`/`403`) for documents (`.pdf`, `.pptx`, `.xlsx`, `.mp4`, etc.), static assets, and external links to avoid downloading large binary files into memory.
16
+ - **Redirect & Error Diagnostics**: Captures full redirect chains (`301`/`302` -> final URL) and categorizes failures (`Timeout`, `SSLError`, `ConnectionError`, `HTTPError`).
17
+ - **Multi-Format Reports**: Export results to JSON, CSV, Markdown, or styled HTML.
18
+ - **Interactive TUI (`--tui`)**: Built with `textual` for live crawl monitoring, filtering (`All`, `Broken`, `Redirects`, `External`, `Assets/Docs`), referrer inspection, and one-key report exports.
19
+
20
+ ## Installation
21
+
22
+ Requires Python 3.13+.
23
+
24
+ ### From PyPI (Recommended)
25
+
26
+ ```bash
27
+ # Install globally as an isolated CLI tool with uv
28
+ uv tool install JumpingSpider
29
+
30
+ # Or run directly without installing using uvx
31
+ uvx JumpingSpider https://example.com/docs/
32
+
33
+ # Or install with pip / pipx
34
+ pip install JumpingSpider
35
+ ```
36
+
37
+ ### From Source (Development)
38
+
39
+ Requires [uv](https://docs.astral.sh/uv/):
40
+
41
+ ```bash
42
+ git clone https://github.com/Hailstone-Labs/JumpingSpider.git
43
+ cd JumpingSpider
44
+ uv sync --all-groups
45
+ ```
46
+
47
+ ## Usage
48
+
49
+ ### CLI Mode
50
+
51
+ ```bash
52
+ # Basic crawl (outputs crawl_results.json and prints a terminal summary scorecard)
53
+ uv run JumpingSpider https://weather.ndc.nasa.gov/tempo/
54
+
55
+ # Export multiple report formats at once
56
+ uv run JumpingSpider https://weather.ndc.nasa.gov/tempo/ \
57
+ -o tempo_results.json \
58
+ -o tempo_broken.csv \
59
+ -o tempo_report.md \
60
+ -o tempo_report.html
61
+
62
+ # Export only broken links to CSV
63
+ uv run JumpingSpider https://weather.ndc.nasa.gov/tempo/ -o broken_links.csv --broken-only
64
+
65
+ # Customize concurrency, rate limit delay, and depth
66
+ uv run JumpingSpider https://example.com/docs/ -w 8 -d 0.1 --max-depth 3
67
+
68
+ # Only check internal links and skip page assets
69
+ uv run JumpingSpider https://example.com/docs/ --no-external --no-assets
70
+ ```
71
+
72
+ ### CLI Options
73
+
74
+ | Option | Default | Description |
75
+ | :--- | :--- | :--- |
76
+ | `start_url` | — | Starting URL to crawl (strictly locked to paths under this directory). |
77
+ | `-o, --output <path>` | `crawl_results.json` | Output file path (`.json`, `.csv`, `.md`, `.html`). Can be passed multiple times. |
78
+ | `-f, --format <fmt>` | Auto from extension | Explicit report format override (`json`, `csv`, `md`, `html`). |
79
+ | `--tui` | `False` | Launch the interactive Terminal User Interface (TUI). |
80
+ | `--check-external` / `--no-external` | `True` | Validate external and sibling links without crawling them. |
81
+ | `--check-assets` / `--no-assets` | `True` | Validate page assets (`<img>`, `<script>`, `<link>`, `<video>`, `<iframe>`). |
82
+ | `-w, --workers <int>` | `5` | Number of concurrent worker threads. |
83
+ | `-d, --delay <float>` | `0.2` | Delay in seconds between HTTP requests. |
84
+ | `--timeout <float>` | `10.0` | HTTP request timeout in seconds. |
85
+ | `--max-depth <int>` | Unlimited | Maximum link depth to crawl for internal pages. |
86
+ | `--broken-only` | `False` | Only include broken links in exported reports. |
87
+ | `-q, --quiet` | `False` | Suppress live console progress and summary tables. |
88
+
89
+ ### Interactive TUI Mode
90
+
91
+ ```bash
92
+ uv run JumpingSpider --tui https://weather.ndc.nasa.gov/tempo/
93
+ ```
94
+
95
+ **TUI Keyboard Shortcuts:**
96
+ - `Ctrl+S`: Start crawl
97
+ - `Ctrl+X`: Stop crawl
98
+ - `1`–`5`: Filter table (`1` All, `2` Broken, `3` Redirects, `4` External, `5` Assets/Docs)
99
+ - `e`: Export all report formats (`crawl_results.{json,csv,md,html}`)
100
+ - `q`: Quit
101
+
102
+ ## Development & Tests
103
+
104
+ ```bash
105
+ # Run unit tests
106
+ uv run pytest -v
107
+
108
+ # Run Ruff linter
109
+ uv run ruff check .
110
+ ```
111
+
112
+ ## Contributing
113
+
114
+ Contributions are welcome! Please read our [Contributing Guide](CONTRIBUTING.md) and [Code of Conduct](CODE_OF_CONDUCT.md) before submitting a pull request. For security issues, please refer to our [Security Policy](SECURITY.md).
115
+
116
+ ## License
117
+
118
+ Distributed under the MIT License. See [`LICENSE`](LICENSE) for full details.
jumpingspider-0.2.1/pyproject.toml ADDED
@@ -0,0 +1,75 @@
1
+ [project]
2
+ name = "JumpingSpider"
3
+ version = "0.2.1"
4
+ description = "Concurrent path-bounded web crawler, broken link detector, and asset validator with multi-format reports and Textual TUI"
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ license-files = ["LICENSE"]
8
+ keywords = [
9
+ "crawler",
10
+ "spider",
11
+ "broken-links",
12
+ "link-checker",
13
+ "tui",
14
+ "textual",
15
+ "seo",
16
+ "site-audit",
17
+ ]
18
+ classifiers = [
19
+ "Development Status :: 4 - Beta",
20
+ "Environment :: Console",
21
+ "Intended Audience :: Developers",
22
+ "Intended Audience :: System Administrators",
23
+ "Operating System :: OS Independent",
24
+ "Programming Language :: Python :: 3",
25
+ "Programming Language :: Python :: 3.13",
26
+ "Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking",
27
+ "Topic :: Utilities",
28
+ "Typing :: Typed",
29
+ ]
30
+ requires-python = ">=3.13"
31
+ dependencies = [
32
+ "beautifulsoup4>=4.15.0",
33
+ "requests>=2.34.2",
34
+ "textual>=8.2.8",
35
+ ]
36
+
37
+ [[project.authors]]
38
+ name = "Roger Allen"
39
+ email = "roger.allen795@gmail.com"
40
+
41
+ [project.urls]
42
+ Homepage = "https://github.com/Hailstone-Labs/JumpingSpider"
43
+ Repository = "https://github.com/Hailstone-Labs/JumpingSpider"
44
+ Issues = "https://github.com/Hailstone-Labs/JumpingSpider/issues"
45
+ Changelog = "https://github.com/Hailstone-Labs/JumpingSpider/blob/main/CHANGELOG.md"
46
+
47
+ [project.scripts]
48
+ JumpingSpider = "JumpingSpider:main"
49
+ jumping-spider = "JumpingSpider:main"
50
+
51
+ [tool.pytest.ini_options]
52
+ asyncio_mode = "auto"
53
+ asyncio_default_fixture_loop_scope = "function"
54
+ testpaths = ["tests"]
55
+
56
+ [tool.ruff]
57
+ line-length = 120
58
+ target-version = "py313"
59
+
60
+ [tool.ruff.lint]
61
+ ignore = ["N999"]
62
+
63
+ [tool.uv.build-backend]
64
+ module-name = "JumpingSpider"
65
+
66
+ [build-system]
67
+ requires = ["uv_build>=0.13.7,<0.14.0"]
68
+ build-backend = "uv_build"
69
+
70
+ [dependency-groups]
71
+ dev = [
72
+ "pytest>=9.2.0",
73
+ "pytest-asyncio>=1.4.1.post1",
74
+ "ruff>=0.17.4",
75
+ ]
jumpingspider-0.2.1/pyproject.toml.orig ADDED
@@ -0,0 +1,65 @@
1
+ [project]
2
+ name = "JumpingSpider"
3
+ version = "0.2.1"
4
+ description = "Concurrent path-bounded web crawler, broken link detector, and asset validator with multi-format reports and Textual TUI"
5
+ readme = "README.md"
6
+ license = "MIT"
7
+ license-files = ["LICENSE"]
8
+ authors = [
9
+ { name = "Roger Allen", email = "roger.allen795@gmail.com" }
10
+ ]
11
+ keywords = ["crawler", "spider", "broken-links", "link-checker", "tui", "textual", "seo", "site-audit"]
12
+ classifiers = [
13
+ "Development Status :: 4 - Beta",
14
+ "Environment :: Console",
15
+ "Intended Audience :: Developers",
16
+ "Intended Audience :: System Administrators",
17
+ "Operating System :: OS Independent",
18
+ "Programming Language :: Python :: 3",
19
+ "Programming Language :: Python :: 3.13",
20
+ "Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking",
21
+ "Topic :: Utilities",
22
+ "Typing :: Typed",
23
+ ]
24
+ requires-python = ">=3.13"
25
+ dependencies = [
26
+ "beautifulsoup4>=4.15.0",
27
+ "requests>=2.34.2",
28
+ "textual>=8.2.8",
29
+ ]
30
+
31
+ [project.urls]
32
+ Homepage = "https://github.com/Hailstone-Labs/JumpingSpider"
33
+ Repository = "https://github.com/Hailstone-Labs/JumpingSpider"
34
+ Issues = "https://github.com/Hailstone-Labs/JumpingSpider/issues"
35
+ Changelog = "https://github.com/Hailstone-Labs/JumpingSpider/blob/main/CHANGELOG.md"
36
+
37
+ [project.scripts]
38
+ JumpingSpider = "JumpingSpider:main"
39
+ jumping-spider = "JumpingSpider:main"
40
+
41
+ [tool.pytest.ini_options]
42
+ asyncio_mode = "auto"
43
+ asyncio_default_fixture_loop_scope = "function"
44
+ testpaths = ["tests"]
45
+
46
+ [tool.ruff]
47
+ line-length = 120
48
+ target-version = "py313"
49
+
50
+ [tool.ruff.lint]
51
+ ignore = ["N999"]
52
+
53
+ [build-system]
54
+ requires = ["uv_build>=0.13.7,<0.14.0"]
55
+ build-backend = "uv_build"
56
+
57
+ [tool.uv.build-backend]
58
+ module-name = "JumpingSpider"
59
+
60
+ [dependency-groups]
61
+ dev = [
62
+ "pytest>=9.2.0",
63
+ "pytest-asyncio>=1.4.1.post1",
64
+ "ruff>=0.17.4",
65
+ ]
jumpingspider-0.2.1/src/JumpingSpider/__init__.py ADDED
@@ -0,0 +1,13 @@
1
+ from JumpingSpider.crawler import SpiderCrawler
2
+ from JumpingSpider.main import crawl_site, main
3
+ from JumpingSpider.models import CrawlConfig, CrawlSummary, LinkReference, LinkResult
4
+
5
+ __all__ = [
6
+ "CrawlConfig",
7
+ "CrawlSummary",
8
+ "LinkReference",
9
+ "LinkResult",
10
+ "SpiderCrawler",
11
+ "crawl_site",
12
+ "main",
13
+ ]