webtools-cli 1.1.2__tar.gz → 1.1.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- webtools_cli-1.1.4/PKG-INFO +170 -0
- webtools_cli-1.1.4/README.md +135 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/pyproject.toml +2 -2
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools/core.py +2694 -2602
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools/web/index.html +17 -18
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools/web/script.js +75 -107
- webtools_cli-1.1.4/webtools/web/style.css +410 -0
- webtools_cli-1.1.4/webtools_cli.egg-info/PKG-INFO +170 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools_cli.egg-info/SOURCES.txt +0 -1
- webtools_cli-1.1.2/PKG-INFO +0 -109
- webtools_cli-1.1.2/README.md +0 -74
- webtools_cli-1.1.2/webtools/web/Web_Tools.png +0 -0
- webtools_cli-1.1.2/webtools/web/style.css +0 -71
- webtools_cli-1.1.2/webtools_cli.egg-info/PKG-INFO +0 -109
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/LICENSE +0 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/setup.cfg +0 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools/__init__.py +0 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools/__main__.py +0 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools/cli.py +0 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools_cli.egg-info/dependency_links.txt +0 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools_cli.egg-info/entry_points.txt +0 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools_cli.egg-info/requires.txt +0 -0
- {webtools_cli-1.1.2 → webtools_cli-1.1.4}/webtools_cli.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: webtools-cli
|
|
3
|
+
Version: 1.1.4
|
|
4
|
+
Summary: Advanced Web Intelligence & Scraping Toolkit with CLI and Web UI
|
|
5
|
+
Author: Abhinav Adarsh
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://webtoolscli.pages.dev
|
|
8
|
+
Keywords: web-scraping,osint,seo,intelligence,cli
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Environment :: Console
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Topic :: Internet :: WWW/HTTP
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: flask
|
|
22
|
+
Requires-Dist: requests
|
|
23
|
+
Requires-Dist: beautifulsoup4
|
|
24
|
+
Requires-Dist: qrcode
|
|
25
|
+
Requires-Dist: opencv-python
|
|
26
|
+
Requires-Dist: numpy
|
|
27
|
+
Requires-Dist: textblob
|
|
28
|
+
Requires-Dist: Pillow
|
|
29
|
+
Requires-Dist: mtranslate
|
|
30
|
+
Requires-Dist: colorama
|
|
31
|
+
Requires-Dist: pyreadline3; platform_system == "Windows"
|
|
32
|
+
Provides-Extra: playwright
|
|
33
|
+
Requires-Dist: playwright; extra == "playwright"
|
|
34
|
+
Dynamic: license-file
|
|
35
|
+
|
|
36
|
+
# WebTools CLI
|
|
37
|
+
|
|
38
|
+
[](https://pypi.org/project/webtools-cli/)
|
|
39
|
+
[](https://github.com/abhinavgautam08/webtools-cli/blob/main/LICENSE)
|
|
40
|
+
[](https://pypi.org/project/webtools-cli/)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
try : [](https://colab.research.google.com/drive/1l4-ZChKGYhOFrPwp2_JPLJ1X0ZYSVrZk?usp=sharing)
|
|
45
|
+
|
|
46
|
+
WebTools CLI is an advanced web intelligence suite for researchers, OSINT enthusiasts, and developers. It brings the power of deep web analysis and automated scraping directly into your terminal.
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## 🚀 Why WebTools CLI?
|
|
51
|
+
|
|
52
|
+
- **🎯 Stealth & Speed**: Smart proxy rotation and Turbo-Fetch logic for evasion and performance.
|
|
53
|
+
- **🧠 AI-Powered**: Automated content summarization, sentiment analysis, and readability scoring.
|
|
54
|
+
- **🔧 Security-Centric**: Built-in honeypot detection, threat leveling, and image forensic analysis.
|
|
55
|
+
- **💻 Terminal-First**: Designed for power users who live in the command line.
|
|
56
|
+
- **🛡️ Cross-Platform**: Works seamlessly on Windows, Linux, and macOS (with auto-download for Windows tunnels).
|
|
57
|
+
- **🔌 SPA Ready**: Automatic Playwright fallback for JavaScript-heavy sites like LinkedIn/Instagram.
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## 📦 Installation
|
|
62
|
+
|
|
63
|
+
See the installation guide for recommended system specifications.
|
|
64
|
+
|
|
65
|
+
### Quick Install
|
|
66
|
+
|
|
67
|
+
Install globally via pip:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
pip install webtools-cli
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
To upgrade to the latest version:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
pip install webtools-cli --upgrade
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Optional Dependencies
|
|
80
|
+
|
|
81
|
+
For Single Page Application (SPA) support:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
playwright install chromium
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## 📋 Key Features
|
|
90
|
+
|
|
91
|
+
### Advanced Scraping & Stealth
|
|
92
|
+
- **Smart Proxy Rotation**: Automatically rotates User-Agents and Proxies to evade detection.
|
|
93
|
+
- **Turbo-Fetch**: Parallel chunk downloads for large media (Videos/Images).
|
|
94
|
+
- **Deep Crawl**: Recursive link mapping up to 3 levels deep.
|
|
95
|
+
- **Headless Fallback**: Integrated Playwright support for auth-walled or SPA environments.
|
|
96
|
+
|
|
97
|
+
### Intelligence & Security Analysis
|
|
98
|
+
- **OSINT Toolkit**: Auto-extract emails, phones, locations, social media, and tech stacks.
|
|
99
|
+
- **SEO Auditor**: Page score, heading hierarchy, link integrity, and image alt-text auditing.
|
|
100
|
+
- **Image Forensics**: CLI-based Error Level Analysis (ELA) and AI-likelihood detection.
|
|
101
|
+
- **Honeypot Detector**: Identifies hidden traps and anti-bot measures (Cloudflare/CAPTCHAs).
|
|
102
|
+
|
|
103
|
+
### Modern Experience
|
|
104
|
+
- **Premium Visual Engine**: Sleek glassmorphism, fluid gradients, and premium Motion One animations.
|
|
105
|
+
- **Responsive Preview**: Live rendering scaling for desktop and mobile viewpoints.
|
|
106
|
+
- **History & Stats**: Phase-by-phase performance tracking and historical session management.
|
|
107
|
+
|
|
108
|
+
---
|
|
109
|
+
|
|
110
|
+
## 🚀 Getting Started
|
|
111
|
+
|
|
112
|
+
### Basic Usage
|
|
113
|
+
|
|
114
|
+
#### Launch Interactive Menu
|
|
115
|
+
```bash
|
|
116
|
+
webtools
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
#### Non-Interactive Script Mode
|
|
120
|
+
```bash
|
|
121
|
+
python -m webtools
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
### Slash Commands Reference
|
|
125
|
+
|
|
126
|
+
Navigate the suite using quick terminal commands:
|
|
127
|
+
|
|
128
|
+
| Command | Alias | Description |
|
|
129
|
+
|---------|-------|-------------|
|
|
130
|
+
| `/web` | `/w` | Launch **Web UI** (Cloudflare Tunnel + QR) |
|
|
131
|
+
| `/cli` | `/c` | Launch **CLI Intelligence** scan |
|
|
132
|
+
| `/image` | `/i` | **Image Forensics** & AI Likelihood |
|
|
133
|
+
| `/history`| `/hi`| View and manage scan history |
|
|
134
|
+
| `/help` | `/h` | Show full command documentation |
|
|
135
|
+
| `/clear` | - | Purge all locally scraped data |
|
|
136
|
+
| `/quit` | `/q` | Exit the application |
|
|
137
|
+
|
|
138
|
+
---
|
|
139
|
+
|
|
140
|
+
## ☁️ Deployment Options
|
|
141
|
+
|
|
142
|
+
- **Local Development**: Run on your machine with a generated QR code for mobile access.
|
|
143
|
+
- **Cloud Tunnels**: Automatic `cloudflared` integration to expose your UI globally.
|
|
144
|
+
- **Google Colab**: Compatible with Colab for cloud-based scraping (see badge above).
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
## 🤝 Resources & Support
|
|
149
|
+
|
|
150
|
+
- **[GitHub Repository](https://github.com/abhinavgautam08/webtools-cli)** - Source code and updates.
|
|
151
|
+
- **[Issue Tracker](https://github.com/abhinavgautam08/webtools-cli/issues)** - Report bugs or request features.
|
|
152
|
+
- **[License](./LICENSE)** - MIT License.
|
|
153
|
+
|
|
154
|
+
---
|
|
155
|
+
|
|
156
|
+
## ⚖️ Legal
|
|
157
|
+
|
|
158
|
+
This tool is for **educational and testing purposes only**. Always respect `robots.txt` and the Terms of Service of the websites you scrape. Neither the author nor the contributors are responsible for any misuse of this tool.
|
|
159
|
+
|
|
160
|
+
---
|
|
161
|
+
|
|
162
|
+
<p align="center">
|
|
163
|
+
Built with ❤️ by
|
|
164
|
+
<strong>
|
|
165
|
+
<a href="https://abhinavgautam08.pages.dev" target="_blank" rel="noopener noreferrer">
|
|
166
|
+
Abhinav Adarsh
|
|
167
|
+
</a>
|
|
168
|
+
</strong>
|
|
169
|
+
and the open source community
|
|
170
|
+
</p>
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# WebTools CLI
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/webtools-cli/)
|
|
4
|
+
[](https://github.com/abhinavgautam08/webtools-cli/blob/main/LICENSE)
|
|
5
|
+
[](https://pypi.org/project/webtools-cli/)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
try : [](https://colab.research.google.com/drive/1l4-ZChKGYhOFrPwp2_JPLJ1X0ZYSVrZk?usp=sharing)
|
|
10
|
+
|
|
11
|
+
WebTools CLI is an advanced web intelligence suite for researchers, OSINT enthusiasts, and developers. It brings the power of deep web analysis and automated scraping directly into your terminal.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## 🚀 Why WebTools CLI?
|
|
16
|
+
|
|
17
|
+
- **🎯 Stealth & Speed**: Smart proxy rotation and Turbo-Fetch logic for evasion and performance.
|
|
18
|
+
- **🧠 AI-Powered**: Automated content summarization, sentiment analysis, and readability scoring.
|
|
19
|
+
- **🔧 Security-Centric**: Built-in honeypot detection, threat leveling, and image forensic analysis.
|
|
20
|
+
- **💻 Terminal-First**: Designed for power users who live in the command line.
|
|
21
|
+
- **🛡️ Cross-Platform**: Works seamlessly on Windows, Linux, and macOS (with auto-download for Windows tunnels).
|
|
22
|
+
- **🔌 SPA Ready**: Automatic Playwright fallback for JavaScript-heavy sites like LinkedIn/Instagram.
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## 📦 Installation
|
|
27
|
+
|
|
28
|
+
See the installation guide for recommended system specifications.
|
|
29
|
+
|
|
30
|
+
### Quick Install
|
|
31
|
+
|
|
32
|
+
Install globally via pip:
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
pip install webtools-cli
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
To upgrade to the latest version:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
pip install webtools-cli --upgrade
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
### Optional Dependencies
|
|
45
|
+
|
|
46
|
+
For Single Page Application (SPA) support:
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
playwright install chromium
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## 📋 Key Features
|
|
55
|
+
|
|
56
|
+
### Advanced Scraping & Stealth
|
|
57
|
+
- **Smart Proxy Rotation**: Automatically rotates User-Agents and Proxies to evade detection.
|
|
58
|
+
- **Turbo-Fetch**: Parallel chunk downloads for large media (Videos/Images).
|
|
59
|
+
- **Deep Crawl**: Recursive link mapping up to 3 levels deep.
|
|
60
|
+
- **Headless Fallback**: Integrated Playwright support for auth-walled or SPA environments.
|
|
61
|
+
|
|
62
|
+
### Intelligence & Security Analysis
|
|
63
|
+
- **OSINT Toolkit**: Auto-extract emails, phones, locations, social media, and tech stacks.
|
|
64
|
+
- **SEO Auditor**: Page score, heading hierarchy, link integrity, and image alt-text auditing.
|
|
65
|
+
- **Image Forensics**: CLI-based Error Level Analysis (ELA) and AI-likelihood detection.
|
|
66
|
+
- **Honeypot Detector**: Identifies hidden traps and anti-bot measures (Cloudflare/CAPTCHAs).
|
|
67
|
+
|
|
68
|
+
### Modern Experience
|
|
69
|
+
- **Premium Visual Engine**: Sleek glassmorphism, fluid gradients, and premium Motion One animations.
|
|
70
|
+
- **Responsive Preview**: Live rendering scaling for desktop and mobile viewpoints.
|
|
71
|
+
- **History & Stats**: Phase-by-phase performance tracking and historical session management.
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
## 🚀 Getting Started
|
|
76
|
+
|
|
77
|
+
### Basic Usage
|
|
78
|
+
|
|
79
|
+
#### Launch Interactive Menu
|
|
80
|
+
```bash
|
|
81
|
+
webtools
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
#### Non-Interactive Script Mode
|
|
85
|
+
```bash
|
|
86
|
+
python -m webtools
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
### Slash Commands Reference
|
|
90
|
+
|
|
91
|
+
Navigate the suite using quick terminal commands:
|
|
92
|
+
|
|
93
|
+
| Command | Alias | Description |
|
|
94
|
+
|---------|-------|-------------|
|
|
95
|
+
| `/web` | `/w` | Launch **Web UI** (Cloudflare Tunnel + QR) |
|
|
96
|
+
| `/cli` | `/c` | Launch **CLI Intelligence** scan |
|
|
97
|
+
| `/image` | `/i` | **Image Forensics** & AI Likelihood |
|
|
98
|
+
| `/history`| `/hi`| View and manage scan history |
|
|
99
|
+
| `/help` | `/h` | Show full command documentation |
|
|
100
|
+
| `/clear` | - | Purge all locally scraped data |
|
|
101
|
+
| `/quit` | `/q` | Exit the application |
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## ☁️ Deployment Options
|
|
106
|
+
|
|
107
|
+
- **Local Development**: Run on your machine with a generated QR code for mobile access.
|
|
108
|
+
- **Cloud Tunnels**: Automatic `cloudflared` integration to expose your UI globally.
|
|
109
|
+
- **Google Colab**: Compatible with Colab for cloud-based scraping (see badge above).
|
|
110
|
+
|
|
111
|
+
---
|
|
112
|
+
|
|
113
|
+
## 🤝 Resources & Support
|
|
114
|
+
|
|
115
|
+
- **[GitHub Repository](https://github.com/abhinavgautam08/webtools-cli)** - Source code and updates.
|
|
116
|
+
- **[Issue Tracker](https://github.com/abhinavgautam08/webtools-cli/issues)** - Report bugs or request features.
|
|
117
|
+
- **[License](./LICENSE)** - MIT License.
|
|
118
|
+
|
|
119
|
+
---
|
|
120
|
+
|
|
121
|
+
## ⚖️ Legal
|
|
122
|
+
|
|
123
|
+
This tool is for **educational and testing purposes only**. Always respect `robots.txt` and the Terms of Service of the websites you scrape. Neither the author nor the contributors are responsible for any misuse of this tool.
|
|
124
|
+
|
|
125
|
+
---
|
|
126
|
+
|
|
127
|
+
<p align="center">
|
|
128
|
+
Built with ❤️ by
|
|
129
|
+
<strong>
|
|
130
|
+
<a href="https://abhinavgautam08.pages.dev" target="_blank" rel="noopener noreferrer">
|
|
131
|
+
Abhinav Adarsh
|
|
132
|
+
</a>
|
|
133
|
+
</strong>
|
|
134
|
+
and the open source community
|
|
135
|
+
</p>
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "webtools-cli"
|
|
7
|
-
version = "1.1.
|
|
7
|
+
version = "1.1.4"
|
|
8
8
|
description = "Advanced Web Intelligence & Scraping Toolkit with CLI and Web UI"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -45,7 +45,7 @@ playwright = ["playwright"]
|
|
|
45
45
|
webtools = "webtools.cli:main"
|
|
46
46
|
|
|
47
47
|
[project.urls]
|
|
48
|
-
Homepage = "https://
|
|
48
|
+
Homepage = "https://webtoolscli.pages.dev"
|
|
49
49
|
|
|
50
50
|
[tool.setuptools.packages.find]
|
|
51
51
|
include = ["webtools*"]
|