waymore 5.0__tar.gz → 5.1__tar.gz

{waymore-5.0/waymore.egg-info → waymore-5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: waymore
-Version: 5.0
+Version: 5.1
 Summary: Find way more from the Wayback Machine, Common Crawl, Alien Vault OTX, URLScan & VirusTotal!
 Home-page: https://github.com/xnl-h4ck3r/waymore
 Author: @xnl-h4ck3r
@@ -15,7 +15,7 @@ Requires-Dist: tldextract
 
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v5.0
+## About - v5.1
 
 The idea behind **waymore** is to find even more links from the Wayback Machine than other existing tools.
 

{waymore-5.0 → waymore-5.1}/README.md

@@ -1,6 +1,6 @@
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v5.0
+## About - v5.1
 
 The idea behind **waymore** is to find even more links from the Wayback Machine than other existing tools.
 

waymore-5.1/waymore/__init__.py

@@ -0,0 +1 @@
+__version__="5.1"

{waymore-5.0 → waymore-5.1}/waymore/waymore.py

@@ -1316,7 +1316,7 @@ def validateArgProviders(x):
             invalid = True
             break
     if invalid:
-        raise argparse.ArgumentTypeError('Pass providers separated by a comma, e.g. wayback,commoncrawl,otx,urlscan,virustotal')     
+        raise argparse.ArgumentTypeError('Pass providers separated by a comma, e.g. wayback,commoncrawl,otx,urlscan,virustotal,intelx')     
     return x
 
 def processAlienVaultPage(url):
@@ -1638,7 +1638,10 @@ def getURLScanUrls():
         # Get the first page from urlscan.io
         try:
             # Choose a random user agent string to use for any requests
-            userAgent = random.choice(USER_AGENT)
+            # For other sources we would use `random.choice(USER_AGENT)` to asignn a random user-agent, but it seems 
+            # that there are a handful of those that ALWAYS return 429. Passing a specific one all the time seems to
+            # be successful all the time 
+            userAgent = "waymore by xnl-h4ck3r" 
             session = requests.Session()
             session.mount('https://', HTTP_ADAPTER)
             session.mount('http://', HTTP_ADAPTER)
@@ -1789,7 +1792,6 @@ def getURLScanUrls():
                             # Get the next page from urlscan.io
                             try:
                                 # Choose a random user agent string to use for any requests
-                                userAgent = random.choice(USER_AGENT)
                                 session = requests.Session()
                                 session.mount('https://', HTTP_ADAPTER)
                                 session.mount('http://', HTTP_ADAPTER)

{waymore-5.0 → waymore-5.1/waymore.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: waymore
-Version: 5.0
+Version: 5.1
 Summary: Find way more from the Wayback Machine, Common Crawl, Alien Vault OTX, URLScan & VirusTotal!
 Home-page: https://github.com/xnl-h4ck3r/waymore
 Author: @xnl-h4ck3r
@@ -15,7 +15,7 @@ Requires-Dist: tldextract
 
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v5.0
+## About - v5.1
 
 The idea behind **waymore** is to find even more links from the Wayback Machine than other existing tools.
 

waymore-5.0/waymore/__init__.py

@@ -1 +0,0 @@
-__version__="5.0"