waymore 4.0__tar.gz → 4.2__tar.gz

{waymore-4.0/waymore.egg-info → waymore-4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: waymore
-Version: 4.0
+Version: 4.2
 Summary: Find way more from the Wayback Machine, Common Crawl, Alien Vault OTX, URLScan & VirusTotal!
 Home-page: https://github.com/xnl-h4ck3r/waymore
 Author: @xnl-h4ck3r
@@ -16,7 +16,7 @@ Requires-Dist: tldextract
 
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v4.0
+## About - v4.2
 
 The idea behind **waymore** is to find even more links from the Wayback Machine than other existing tools.
 
@@ -47,12 +47,24 @@ Now **waymore** gets URL's from ALL of those sources too (with ability to filter
 
 `waymore` supports **Python 3**.
 
-Install `waymore` in default(global) python environment.
+Install `waymore` in default (global) python environment.
+
+```bash
+pip install waymore
+```
+
+OR
 
 ```bash
 pip install git+https://github.com/xnl-h4ck3r/waymore.git -v
 ```
 
+You can upgrade with
+
+```bash
+pip install --upgrade waymore
+```
+
 ### pipx
 
 Quick setup in isolated python environment using [pipx](https://pypa.github.io/pipx/)

{waymore-4.0 → waymore-4.2}/README.md

@@ -1,6 +1,6 @@
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v4.0
+## About - v4.2
 
 The idea behind **waymore** is to find even more links from the Wayback Machine than other existing tools.
 
@@ -31,12 +31,24 @@ Now **waymore** gets URL's from ALL of those sources too (with ability to filter
 
 `waymore` supports **Python 3**.
 
-Install `waymore` in default(global) python environment.
+Install `waymore` in default (global) python environment.
+
+```bash
+pip install waymore
+```
+
+OR
 
 ```bash
 pip install git+https://github.com/xnl-h4ck3r/waymore.git -v
 ```
 
+You can upgrade with
+
+```bash
+pip install --upgrade waymore
+```
+
 ### pipx
 
 Quick setup in isolated python environment using [pipx](https://pypa.github.io/pipx/)

waymore-4.2/waymore/__init__.py

@@ -0,0 +1 @@
+__version__="4.2"

{waymore-4.0 → waymore-4.2}/waymore/waymore.py

@@ -30,7 +30,6 @@ try:
     from . import __version__
 except:
     pass
-from tqdm import tqdm
 
 # Try to import psutil to show memory usage
 try:
@@ -1781,23 +1780,31 @@ def processWayBackPage(url):
                     return
             
             # Get the URLs and MIME types. Each line is a separate JSON string
-            for line in resp.iter_lines():
-                results = line.decode("utf-8")
-                # Only get MIME Types if --verbose option was selected
-                if verbose():
-                    try:
-                        linkMimes.add(str(results).split(' ')[2])
-                    except Exception as e:
-                        if verbose():
-                            writerr(colored(getSPACER('ERROR processWayBackPage 2: Cannot get MIME type from line: ' + str(line)),'red'))
-                            write(resp.text)
-                try:
+            try:
+                for line in resp.iter_lines():
+                    results = line.decode("utf-8")
                     foundUrl = fixArchiveOrgUrl(str(results).split(' ')[1])
-                    linksFoundAdd(foundUrl)
-                except Exception as e:
-                    if verbose():
-                        writerr(colored(getSPACER('ERROR processWayBackPage 3: Cannot get link from line: ' + str(line)),'red'))         
-                        write(resp.text)       
+                    
+                    # Check the URL exclusions
+                    match = re.search(r'('+re.escape(FILTER_URL).replace(',','|')+')', foundUrl, flags=re.IGNORECASE)
+                    if match is None:
+                        # Only get MIME Types if --verbose option was selected
+                        if verbose():
+                            try:
+                                linkMimes.add(str(results).split(' ')[2])
+                            except Exception as e:
+                                if verbose():
+                                    writerr(colored(getSPACER('ERROR processWayBackPage 2: Cannot get MIME type from line: ' + str(line)),'red'))
+                                    write(resp.text)
+                        try:
+                            linksFoundAdd(foundUrl)
+                        except Exception as e:
+                            if verbose():
+                                writerr(colored(getSPACER('ERROR processWayBackPage 3: Cannot get link from line: ' + str(line)),'red'))         
+                                write(resp.text)       
+            except Exception as e:
+                if verbose():
+                    writerr(colored(getSPACER('ERROR processWayBackPage 4: ' + str(line)),'red'))   
         else:
             pass
     except Exception as e:
@@ -2423,12 +2430,12 @@ def processResponses():
             # This is useful for filtering out captures that are 'too dense' or when looking for unique captures."
             if args.capture_interval == 'none': # get all
                 collapse = ''
-            elif args.capture_interval == 'h': # get at most 1 capture per hour
-                collapse = 'timestamp:10'
-            elif args.capture_interval == 'd': # get at most 1 capture per day
-                collapse = 'timestamp:8'
-            elif args.capture_interval == 'm': # get at most 1 capture per month
-                collapse = 'timestamp:6'
+            elif args.capture_interval == 'h': # get at most 1 capture per URL per hour
+                collapse = 'timestamp:10,original'
+            elif args.capture_interval == 'd': # get at most 1 capture per URL per day
+                collapse = 'timestamp:8,original'
+            elif args.capture_interval == 'm': # get at most 1 capture per URL per month
+                collapse = 'timestamp:6,original'
 
             url = WAYBACK_URL.replace('{DOMAIN}',subs + quote(argsInput) + path).replace('{COLLAPSE}',collapse) + filterMIME + filterCode + filterLimit + filterFrom + filterTo + filterKeywords
                 

{waymore-4.0 → waymore-4.2/waymore.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: waymore
-Version: 4.0
+Version: 4.2
 Summary: Find way more from the Wayback Machine, Common Crawl, Alien Vault OTX, URLScan & VirusTotal!
 Home-page: https://github.com/xnl-h4ck3r/waymore
 Author: @xnl-h4ck3r
@@ -16,7 +16,7 @@ Requires-Dist: tldextract
 
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v4.0
+## About - v4.2
 
 The idea behind **waymore** is to find even more links from the Wayback Machine than other existing tools.
 
@@ -47,12 +47,24 @@ Now **waymore** gets URL's from ALL of those sources too (with ability to filter
 
 `waymore` supports **Python 3**.
 
-Install `waymore` in default(global) python environment.
+Install `waymore` in default (global) python environment.
+
+```bash
+pip install waymore
+```
+
+OR
 
 ```bash
 pip install git+https://github.com/xnl-h4ck3r/waymore.git -v
 ```
 
+You can upgrade with
+
+```bash
+pip install --upgrade waymore
+```
+
 ### pipx
 
 Quick setup in isolated python environment using [pipx](https://pypa.github.io/pipx/)

waymore-4.0/waymore/__init__.py

@@ -1 +0,0 @@
-__version__="4.0"