webtools-cli 1.0.3__tar.gz → 1.0.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {webtools_cli-1.0.3/webtools_cli.egg-info → webtools_cli-1.0.4}/PKG-INFO +2 -2
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/README.md +1 -1
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/pyproject.toml +1 -1
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools/core.py +9 -3
- {webtools_cli-1.0.3 → webtools_cli-1.0.4/webtools_cli.egg-info}/PKG-INFO +2 -2
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/LICENSE +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/setup.cfg +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools/__init__.py +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools/__main__.py +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools/cli.py +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools/web/Web_Tools.png +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools/web/index.html +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools/web/script.js +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools/web/style.css +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools_cli.egg-info/SOURCES.txt +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools_cli.egg-info/dependency_links.txt +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools_cli.egg-info/entry_points.txt +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools_cli.egg-info/requires.txt +0 -0
- {webtools_cli-1.0.3 → webtools_cli-1.0.4}/webtools_cli.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: webtools-cli
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.4
|
|
4
4
|
Summary: Advanced Web Intelligence & Scraping Toolkit with CLI and Web UI
|
|
5
5
|
Author: Abhinav Adarsh
|
|
6
6
|
License-Expression: MIT
|
|
@@ -39,7 +39,7 @@ Dynamic: license-file
|
|
|
39
39
|
[](https://github.com/abhinavgautam08/webtools-cli/blob/main/LICENSE)
|
|
40
40
|
[](https://pypi.org/project/webtools-cli/)
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
|
|
43
43
|
|
|
44
44
|
WebTools CLI is an advanced web intelligence suite for researchers, OSINT enthusiasts, and developers. It brings the power of deep web analysis and automated scraping directly into your terminal, bridging the gap between a high-speed **Terminal UI** and a feature-rich **Cyber-themed Dashboard**.
|
|
45
45
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
[](https://github.com/abhinavgautam08/webtools-cli/blob/main/LICENSE)
|
|
5
5
|
[](https://pypi.org/project/webtools-cli/)
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
|
|
8
8
|
|
|
9
9
|
WebTools CLI is an advanced web intelligence suite for researchers, OSINT enthusiasts, and developers. It brings the power of deep web analysis and automated scraping directly into your terminal, bridging the gap between a high-speed **Terminal UI** and a feature-rich **Cyber-themed Dashboard**.
|
|
10
10
|
|
|
@@ -799,8 +799,9 @@ def execute_scrape_logic(url, fetch_images=False, fetch_videos=False, crawl_dept
|
|
|
799
799
|
response = requests.get(url, headers=headers, timeout=30)
|
|
800
800
|
else:
|
|
801
801
|
# Direct Request (SPA Check ke sath)
|
|
802
|
-
|
|
803
|
-
|
|
802
|
+
block_domains = ['linkedin.com', 'instagram.com', 'facebook.com', 'twitter.com', 'x.com', 'xhamster', 'pornhub', 'xnxx']
|
|
803
|
+
if any(domain in url.lower() for domain in block_domains):
|
|
804
|
+
print(f"⚠️ High-Security Domain detected. Skipping direct requests...")
|
|
804
805
|
response = None # Playwright force karo
|
|
805
806
|
else:
|
|
806
807
|
response = requests.get(url, headers=headers, timeout=30)
|
|
@@ -833,7 +834,12 @@ def execute_scrape_logic(url, fetch_images=False, fetch_videos=False, crawl_dept
|
|
|
833
834
|
else:
|
|
834
835
|
return {'success': False, 'error': f'Request failed and Playwright fallback failed: {str(e)}'}
|
|
835
836
|
else:
|
|
836
|
-
|
|
837
|
+
error_msg = f"Request failed: {str(e)}"
|
|
838
|
+
if response and response.status_code == 404:
|
|
839
|
+
error_msg += "\n💡 Tip: URL might be expired or the site is blocking direct requests. Install Playwright for better results: pip install webtools-cli[playwright]"
|
|
840
|
+
else:
|
|
841
|
+
error_msg += "\n💡 Tip: Try installing Playwright for better bypassing: pip install webtools-cli[playwright]"
|
|
842
|
+
return {'success': False, 'error': error_msg}
|
|
837
843
|
|
|
838
844
|
soup = BeautifulSoup(response.text, 'html.parser')
|
|
839
845
|
perf_tracker.record_phase("HTML Parsing")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: webtools-cli
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.4
|
|
4
4
|
Summary: Advanced Web Intelligence & Scraping Toolkit with CLI and Web UI
|
|
5
5
|
Author: Abhinav Adarsh
|
|
6
6
|
License-Expression: MIT
|
|
@@ -39,7 +39,7 @@ Dynamic: license-file
|
|
|
39
39
|
[](https://github.com/abhinavgautam08/webtools-cli/blob/main/LICENSE)
|
|
40
40
|
[](https://pypi.org/project/webtools-cli/)
|
|
41
41
|
|
|
42
|
-
|
|
42
|
+
|
|
43
43
|
|
|
44
44
|
WebTools CLI is an advanced web intelligence suite for researchers, OSINT enthusiasts, and developers. It brings the power of deep web analysis and automated scraping directly into your terminal, bridging the gap between a high-speed **Terminal UI** and a feature-rich **Cyber-themed Dashboard**.
|
|
45
45
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|