waymore 8.2__tar.gz → 8.4__tar.gz

{waymore-8.2/waymore.egg-info → waymore-8.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: waymore
-Version: 8.2
+Version: 8.4
 Summary: Find way more from the Wayback Machine, Common Crawl, Alien Vault OTX, URLScan, VirusTotal & Intelligence X!
 Home-page: https://github.com/xnl-h4ck3r/waymore
 Author: xnl-h4ck3r
@@ -21,7 +21,7 @@ Dynamic: license-file
 
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v8.2
+## About - v8.4
 
 The idea behind **waymore** is to find even more links from the Wayback Machine (plus other sources) than other existing tools.
 

{waymore-8.2 → waymore-8.4}/README.md

@@ -1,6 +1,6 @@
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v8.2
+## About - v8.4
 
 The idea behind **waymore** is to find even more links from the Wayback Machine (plus other sources) than other existing tools.
 

waymore-8.4/setup.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+import os
+import re
+import shutil
+import sys
+
+from setuptools import find_packages, setup
+
+# Read version from __init__.py without importing
+
+
+def get_version():
+    init_path = os.path.join(os.path.dirname(__file__), "waymore", "__init__.py")
+    with open(init_path, encoding="utf-8") as f:
+        content = f.read()
+        match = re.search(r'__version__\s*=\s*["\']([^"\']+)["\']', content)
+        if match:
+            return match.group(1)
+    raise RuntimeError("Unable to find version string.")
+
+
+target_directory = (
+    os.path.join(os.getenv("APPDATA", ""), "waymore")
+    if os.name == "nt"
+    else (
+        os.path.join(os.path.expanduser("~"), "Library", "Application Support", "waymore")
+        if sys.platform == "darwin"
+        else os.path.join(os.path.expanduser("~"), ".config", "waymore")
+    )
+)
+
+# Copy the config.yml file to the target directory if it exists
+configNew = False
+if target_directory and os.path.isfile("config.yml"):
+    os.makedirs(target_directory, exist_ok=True)
+    # If file already exists, create a new one
+    if os.path.isfile(target_directory + "/config.yml"):
+        configNew = True
+        os.rename(target_directory + "/config.yml", target_directory + "/config.yml.OLD")
+        shutil.copy("config.yml", target_directory)
+        os.rename(target_directory + "/config.yml", target_directory + "/config.yml.NEW")
+        os.rename(target_directory + "/config.yml.OLD", target_directory + "/config.yml")
+    else:
+        shutil.copy("config.yml", target_directory)
+
+setup(
+    name="waymore",
+    packages=find_packages(),
+    version=get_version(),
+    description="Find way more from the Wayback Machine, Common Crawl, Alien Vault OTX, URLScan & VirusTotal!",
+    long_description=open("README.md", encoding="utf-8").read(),
+    long_description_content_type="text/markdown",
+    author="@xnl-h4ck3r",
+    url="https://github.com/xnl-h4ck3r/waymore",
+    install_requires=[
+        "requests",
+        "pyyaml",
+        "termcolor",
+        "psutil",
+        "urlparse3",
+        "tldextract",
+    ],
+    entry_points={
+        "console_scripts": [
+            "waymore = waymore.waymore:main",
+        ],
+    },
+)
+
+if configNew:
+    print(
+        "\n\033[33mIMPORTANT: The file "
+        + target_directory
+        + "/config.yml already exists.\nCreating config.yml.NEW but leaving existing config.\nIf you need the new file, then remove the current one and rename config.yml.NEW to config.yml\n\033[0m"
+    )
+else:
+    print("\n\033[92mThe file " + target_directory + "/config.yml has been created.\n\033[0m")

waymore-8.4/waymore/__init__.py

@@ -0,0 +1 @@
+__version__ = "8.4"

{waymore-8.2 → waymore-8.4}/waymore/waymore.py

@@ -3188,13 +3188,21 @@ def getAlienVaultUrls():
                 )
 
         if not args.check_only:
-            linkCountAlienVault = len(linksFoundAlienVault)
-            write(
-                colored("AlienVault - [ INFO ] Links found on alienvault.com: ", "cyan")
-                + colored(str(linkCountAlienVault), "white")
-            )
-            linksFound.update(linksFoundAlienVault)
-            linksFoundAlienVault.clear()
+            if linksFoundAlienVault is not None:
+                linkCountAlienVault = len(linksFoundAlienVault)
+                write(
+                    colored("AlienVault - [ INFO ] Links found on alienvault.com: ", "cyan")
+                    + colored(str(linkCountAlienVault), "white")
+                )
+                if linksFound is not None:
+                    linksFound.update(linksFoundAlienVault)
+                linksFoundAlienVault.clear()
+            else:
+                linkCountAlienVault = 0
+                write(
+                    colored("AlienVault - [ INFO ] Links found on alienvault.com: ", "cyan")
+                    + colored("0", "white")
+                )
 
     except Exception as e:
         writerr(colored("ERROR getAlienVaultUrls 1: " + str(e), "red"))
@@ -3294,7 +3302,8 @@ def processURLScanUrl(url, httpCode, mimeType, urlscanID=""):
                 if verbose():
                     if mimeType.strip() != "":
                         with links_lock:
-                            linkMimes.add(mimeType)
+                            if linkMimes is not None:
+                                linkMimes.add(mimeType)
 
         # Add link if it passed filters
         if addLink:
@@ -3314,6 +3323,9 @@ def processURLScanUrl(url, httpCode, mimeType, urlscanID=""):
             )
             if match is not None:
                 if args.mode in ("U", "B"):
+                    # Ensure linksFoundURLScan is initialized (can be None during concurrent execution)
+                    if linksFoundURLScan is None:
+                        linksFoundURLScan = set()
                     linksFoundAdd(url, linksFoundURLScan)
                 # If Response mode is requested then add the DOM ID to try later, for the number of responses wanted
                 if urlscanID != "" and args.mode in ("R", "B"):
@@ -4494,7 +4506,7 @@ def getURLScanUrls():
                                 stopSourceURLScan = True
 
             # Show the MIME types found (in case user wants to exclude more)
-            if verbose() and len(linkMimes) > 0 and args.mode != "R":
+            if verbose() and linkMimes is not None and len(linkMimes) > 0 and args.mode != "R":
                 linkMimes.discard("warc/revisit")
                 write(
                     colored("URLScan - [ INFO ] MIME types found: ", "magenta")
@@ -4503,13 +4515,21 @@ def getURLScanUrls():
                 )
 
             if args.mode != "R":
-                linkCountURLScan = len(linksFoundURLScan)
-                write(
-                    colored("URLScan - [ INFO ] Links found on urlscan.io: ", "cyan")
-                    + colored(str(linkCountURLScan), "white")
-                )
-                linksFound.update(linksFoundURLScan)
-                linksFoundURLScan.clear()
+                if linksFoundURLScan is not None:
+                    linkCountURLScan = len(linksFoundURLScan)
+                    write(
+                        colored("URLScan - [ INFO ] Links found on urlscan.io: ", "cyan")
+                        + colored(str(linkCountURLScan), "white")
+                    )
+                    if linksFound is not None:
+                        linksFound.update(linksFoundURLScan)
+                    linksFoundURLScan.clear()
+                else:
+                    linkCountURLScan = 0
+                    write(
+                        colored("URLScan - [ INFO ] Links found on urlscan.io: ", "cyan")
+                        + colored("0", "white")
+                    )
 
     except Exception as e:
         writerr(colored("ERROR getURLScanUrls 1: " + str(e), "red"))
@@ -4973,7 +4993,7 @@ def getWaybackUrls():
                     p.join()
 
             # Show the MIME types found (in case user wants to exclude more)
-            if verbose() and len(linkMimes) > 0:
+            if verbose() and linkMimes is not None and len(linkMimes) > 0:
                 linkMimes.discard("warc/revisit")
                 write(
                     colored("Wayback - [ INFO ] MIME types found: ", "magenta")
@@ -4983,15 +5003,27 @@ def getWaybackUrls():
                 linkMimes = None
 
             if not args.xwm:
-                linkCountWayback = len(linksFoundWayback)
-                write(
-                    colored(
-                        "Wayback - [ INFO ] Links found on Wayback Machine (archive.org): ", "cyan"
+                if linksFoundWayback is not None:
+                    linkCountWayback = len(linksFoundWayback)
+                    write(
+                        colored(
+                            "Wayback - [ INFO ] Links found on Wayback Machine (archive.org): ",
+                            "cyan",
+                        )
+                        + colored(str(linkCountWayback), "white")
+                    )
+                    if linksFound is not None:
+                        linksFound.update(linksFoundWayback)
+                    linksFoundWayback.clear()
+                else:
+                    linkCountWayback = 0
+                    write(
+                        colored(
+                            "Wayback - [ INFO ] Links found on Wayback Machine (archive.org): ",
+                            "cyan",
+                        )
+                        + colored("0", "white")
                     )
-                    + colored(str(linkCountWayback), "white")
-                )
-                linksFound.update(linksFoundWayback)
-                linksFoundWayback.clear()
 
     except Exception as e:
         writerr(colored("ERROR getWaybackUrls 1: " + str(e), "red"))
@@ -5449,7 +5481,7 @@ def getCommonCrawlUrls():
                     p.join()
 
                 # Show the MIME types found (in case user wants to exclude more)
-                if verbose() and len(linkMimes) > 0:
+                if verbose() and linkMimes is not None and len(linkMimes) > 0:
                     linkMimes.discard("warc/revisit")
                     write(
                         colored("CommonCrawl - [ INFO ] MIME types found: ", "magenta")
@@ -5457,13 +5489,21 @@ def getCommonCrawlUrls():
                         + "\n"
                     )
 
-                linkCountCommonCrawl = len(linksFoundCommonCrawl)
-                write(
-                    colored("CommonCrawl - [ INFO ] Links found on commoncrawl.org: ", "cyan")
-                    + colored(str(linkCountCommonCrawl), "white")
-                )
-                linksFound.update(linksFoundCommonCrawl)
-                linksFoundCommonCrawl.clear()
+                if linksFoundCommonCrawl is not None:
+                    linkCountCommonCrawl = len(linksFoundCommonCrawl)
+                    write(
+                        colored("CommonCrawl - [ INFO ] Links found on commoncrawl.org: ", "cyan")
+                        + colored(str(linkCountCommonCrawl), "white")
+                    )
+                    if linksFound is not None:
+                        linksFound.update(linksFoundCommonCrawl)
+                    linksFoundCommonCrawl.clear()
+                else:
+                    linkCountCommonCrawl = 0
+                    write(
+                        colored("CommonCrawl - [ INFO ] Links found on commoncrawl.org: ", "cyan")
+                        + colored("0", "white")
+                    )
 
     except Exception as e:
         writerr(colored("ERROR getCommonCrawlUrls 1: " + str(e), "red"))
@@ -5678,13 +5718,21 @@ def getVirusTotalUrls():
                 processVirusTotalUrl(url)
 
             # Show links found
-            linkCountVirusTotal = len(linksFoundVirusTotal)
-            write(
-                colored("VirusTotal - [ INFO ] Links found on virustotal.com: ", "cyan")
-                + colored(str(linkCountVirusTotal), "white")
-            )
-            linksFound.update(linksFoundVirusTotal)
-            linksFoundVirusTotal.clear()
+            if linksFoundVirusTotal is not None:
+                linkCountVirusTotal = len(linksFoundVirusTotal)
+                write(
+                    colored("VirusTotal - [ INFO ] Links found on virustotal.com: ", "cyan")
+                    + colored(str(linkCountVirusTotal), "white")
+                )
+                if linksFound is not None:
+                    linksFound.update(linksFoundVirusTotal)
+                linksFoundVirusTotal.clear()
+            else:
+                linkCountVirusTotal = 0
+                write(
+                    colored("VirusTotal - [ INFO ] Links found on virustotal.com: ", "cyan")
+                    + colored("0", "white")
+                )
 
     except Exception as e:
         writerr(colored(f"ERROR getVirusTotalUrls: {e}", "red"))
@@ -6001,13 +6049,21 @@ def getIntelxUrls():
         if not intelxAPIIssue:
             processIntelxType(3, credits)
 
-        linkCountIntelx = len(linksFoundIntelx)
-        write(
-            colored("IntelX - [ INFO ] Links found on intelx.io: ", "cyan")
-            + colored(str(linkCountIntelx), "white")
-        )
-        linksFound.update(linksFoundIntelx)
-        linksFoundIntelx.clear()
+        if linksFoundIntelx is not None:
+            linkCountIntelx = len(linksFoundIntelx)
+            write(
+                colored("IntelX - [ INFO ] Links found on intelx.io: ", "cyan")
+                + colored(str(linkCountIntelx), "white")
+            )
+            if linksFound is not None:
+                linksFound.update(linksFoundIntelx)
+            linksFoundIntelx.clear()
+        else:
+            linkCountIntelx = 0
+            write(
+                colored("IntelX - [ INFO ] Links found on intelx.io: ", "cyan")
+                + colored("0", "white")
+            )
 
     except Exception as e:
         writerr(colored("ERROR getIntelxUrls 1: " + str(e), "red"))
@@ -6292,15 +6348,23 @@ def getGhostArchiveUrls():
         if not args.check_only:
             # Count links based on mode - in R mode, count response links; in U/B mode, count URL links
             if args.mode == "R":
-                linkCountGhostArchive = len(ghostArchiveRequestLinks)
+                if ghostArchiveRequestLinks is not None:
+                    linkCountGhostArchive = len(ghostArchiveRequestLinks)
+                else:
+                    linkCountGhostArchive = 0
             else:
-                linkCountGhostArchive = len(linksFoundGhostArchive)
+                if linksFoundGhostArchive is not None:
+                    linkCountGhostArchive = len(linksFoundGhostArchive)
+                else:
+                    linkCountGhostArchive = 0
             write(
                 colored("GhostArchive - [ INFO ] Links found on ghostarchive.org: ", "cyan")
                 + colored(str(linkCountGhostArchive), "white")
             )
-            linksFound.update(linksFoundGhostArchive)
-            linksFoundGhostArchive.clear()
+            if linksFoundGhostArchive is not None:
+                if linksFound is not None:
+                    linksFound.update(linksFoundGhostArchive)
+                linksFoundGhostArchive.clear()
 
     except Exception as e:
         writerr(colored("ERROR getGhostArchiveUrls 1: " + str(e), "red"))

{waymore-8.2 → waymore-8.4/waymore.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: waymore
-Version: 8.2
+Version: 8.4
 Summary: Find way more from the Wayback Machine, Common Crawl, Alien Vault OTX, URLScan, VirusTotal & Intelligence X!
 Home-page: https://github.com/xnl-h4ck3r/waymore
 Author: xnl-h4ck3r
@@ -21,7 +21,7 @@ Dynamic: license-file
 
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v8.2
+## About - v8.4
 
 The idea behind **waymore** is to find even more links from the Wayback Machine (plus other sources) than other existing tools.
 

waymore-8.2/setup.py

@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-import os
-import re
-
-from setuptools import find_packages, setup
-
-# Read version from __init__.py without importing
-
-
-def get_version():
-    init_path = os.path.join(os.path.dirname(__file__), "waymore", "__init__.py")
-    with open(init_path, encoding="utf-8") as f:
-        content = f.read()
-        match = re.search(r'__version__\s*=\s*["\']([^"\']+)["\']', content)
-        if match:
-            return match.group(1)
-    raise RuntimeError("Unable to find version string.")
-
-
-setup(
-    name="waymore",
-    packages=find_packages(),
-    version=get_version(),
-    description="Find way more from the Wayback Machine, Common Crawl, Alien Vault OTX, URLScan & VirusTotal!",
-    long_description=open("README.md", encoding="utf-8").read(),
-    long_description_content_type="text/markdown",
-    author="@xnl-h4ck3r",
-    url="https://github.com/xnl-h4ck3r/waymore",
-    install_requires=[
-        "requests",
-        "pyyaml",
-        "termcolor",
-        "psutil",
-        "urlparse3",
-        "tldextract",
-    ],
-    entry_points={
-        "console_scripts": [
-            "waymore = waymore.waymore:main",
-        ],
-    },
-)

waymore-8.2/waymore/__init__.py

@@ -1 +0,0 @@
-__version__ = "8.2"