waymore 7.4__py3-none-any.whl → 7.5__py3-none-any.whl

waymore/__init__.py

@@ -1 +1 @@
-__version__ = "7.4"
+__version__ = "7.5"

waymore/waymore.py

@@ -19,6 +19,7 @@ import threading
 from datetime import datetime, timedelta
 from pathlib import Path
 from signal import SIGINT, signal
+from typing import Optional
 from urllib.parse import urlparse
 
 import requests
@@ -129,9 +130,64 @@ ALIENVAULT_URL = "https://otx.alienvault.com/api/v1/indicators/{TYPE}/{DOMAIN}/u
 URLSCAN_URL = "https://urlscan.io/api/v1/search/?q=domain:{DOMAIN}{DATERANGE}&size=10000"
 URLSCAN_DOM_URL = "https://urlscan.io/dom/"
 VIRUSTOTAL_URL = "https://www.virustotal.com/vtapi/v2/domain/report?apikey={APIKEY}&domain={DOMAIN}"
-INTELX_SEARCH_URL = "https://2.intelx.io/phonebook/search"
-INTELX_RESULTS_URL = "https://2.intelx.io/phonebook/search/result?id="
-INTELX_ACCOUNT_URL = "https://2.intelx.io/authenticate/info"
+# Paid endpoint first, free endpoint as fallback
+INTELX_BASES = ["https://2.intelx.io", "https://free.intelx.io"]
+
+intelx_tls = threading.local()
+
+
+def initIntelxTls():
+    """Initialize thread-local storage for IntelX if not already done."""
+    if not hasattr(intelx_tls, "INTELX_BASE"):
+        intelx_tls.INTELX_BASE = INTELX_BASES[0]
+        intelx_tls.INTELX_SEARCH_URL = f"{intelx_tls.INTELX_BASE}/phonebook/search"
+        intelx_tls.INTELX_RESULTS_URL = f"{intelx_tls.INTELX_BASE}/phonebook/search/result?id="
+        intelx_tls.INTELX_ACCOUNT_URL = f"{intelx_tls.INTELX_BASE}/authenticate/info"
+
+
+def setIntelxBase(base: str):
+    """Update IntelX URLs to use the provided base (thread-local)."""
+    initIntelxTls()
+    intelx_tls.INTELX_BASE = base
+    intelx_tls.INTELX_SEARCH_URL = f"{intelx_tls.INTELX_BASE}/phonebook/search"
+    intelx_tls.INTELX_RESULTS_URL = f"{intelx_tls.INTELX_BASE}/phonebook/search/result?id="
+    intelx_tls.INTELX_ACCOUNT_URL = f"{intelx_tls.INTELX_BASE}/authenticate/info"
+
+
+def chooseIntelxBase(api_key: str) -> Optional[requests.Response]:
+    """
+    Probe IntelX endpoints in order (paid, then free) and set the first that works.
+    Returns the last response (or None) so callers can inspect status/JSON.
+    """
+    initIntelxTls()
+    try:
+        session = requests.Session()
+        session.mount("https://", HTTP_ADAPTER)
+        session.mount("http://", HTTP_ADAPTER)
+        last_resp = None
+        for base in INTELX_BASES:
+            userAgent = random.choice(USER_AGENT)
+            try:
+                resp = session.get(
+                    f"{base}/authenticate/info",
+                    headers={"User-Agent": userAgent, "X-Key": api_key},
+                )
+                last_resp = resp
+                if resp.status_code == 200:
+                    setIntelxBase(base)
+                    return resp
+                if resp.status_code in [401, 403]:
+                    # Try next base
+                    continue
+            except Exception as e:
+                writerr(colored(f"IntelX - [ ERR ] Problem probing {base}: {e}", "red"))
+            # For other codes or exceptions, try next base anyway instead of breaking prematurely
+            continue
+        return last_resp
+    except Exception as e:
+        writerr(colored(f"IntelX - [ ERR ] Unexpected error in chooseIntelxBase: {e}", "red"))
+        return None
+
 
 # User Agents to use when making requests, chosen at random
 USER_AGENT = [
@@ -559,7 +615,7 @@ def showOptions():
             write(
                 colored("Intelligence X API Key:", "magenta")
                 + colored(
-                    " {none} - You require a paid API Key from https://intelx.io/product",
+                    " {none} - You require a Academia or Paid API Key from https://intelx.io/product",
                     "white",
                 )
             )
@@ -4600,7 +4656,10 @@ def processIntelxUrl(url):
 
         # Add link if it passed filters
         if addLink:
-            linksFoundAdd(url, linksFoundIntelx)
+            # Clean the link to remove any █ (\u2588) characters from the link. These can be present in the IntelX results when the Academia plan is used
+            url = url.replace("\u2588", "").strip()
+            if url != "":
+                linksFoundAdd(url, linksFoundIntelx)
 
     except Exception as e:
         writerr(colored("ERROR processIntelxUrl 1: " + str(e), "red"))
@@ -4611,86 +4670,107 @@ def processIntelxType(target, credits):
     target: 1 - Domains
     target: 3 - URLs
     """
+    initIntelxTls()
     global intelxAPIIssue
     try:
-        try:
-            requestsMade = 0
+        attempts = 0
+        resp = None
+        # Choose a random user agent string to use for any requests and reuse session
+        userAgent = random.choice(USER_AGENT)
+        session = requests.Session()
+        session.mount("https://", HTTP_ADAPTER)
+        session.mount("http://", HTTP_ADAPTER)
 
-            # Choose a random user agent string to use for any requests
-            userAgent = random.choice(USER_AGENT)
-            session = requests.Session()
-            session.mount("https://", HTTP_ADAPTER)
-            session.mount("http://", HTTP_ADAPTER)
-            # Pass the API key in the X-Key header too.
-            resp = session.post(
-                INTELX_SEARCH_URL,
-                data='{"term":"' + quote(argsInputHostname) + '","target":' + str(target) + "}",
-                headers={"User-Agent": userAgent, "X-Key": INTELX_API_KEY},
-            )
-            requestsMade = requestsMade + 1
-        except Exception as e:
-            write(
-                colored(
-                    "IntelX - [ ERR ] Unable to get links from intelx.io: " + str(e),
-                    "red",
+        while attempts < 2:
+            attempts += 1
+            try:
+                requestsMade = 0
+                # Pass the API key in the X-Key header too.
+                resp = session.post(
+                    intelx_tls.INTELX_SEARCH_URL,
+                    data='{"term":"' + quote(argsInputHostname) + '","target":' + str(target) + "}",
+                    headers={"User-Agent": userAgent, "X-Key": INTELX_API_KEY},
                 )
-            )
-            return
+                requestsMade = requestsMade + 1
+            except Exception as e:
+                write(
+                    colored(
+                        "IntelX - [ ERR ] Unable to get links from intelx.io: " + str(e),
+                        "red",
+                    )
+                )
+                return
 
-        # Deal with any errors
-        if resp.status_code == 429:
-            intelxAPIIssue = True
-            writerr(
-                colored(
-                    "IntelX - [ 429 ] Rate limit reached so unable to get links.",
-                    "red",
+            # Deal with any errors
+            if resp.status_code == 200:
+                break
+            elif resp.status_code == 429:
+                intelxAPIIssue = True
+                writerr(
+                    colored(
+                        "IntelX - [ 429 ] Rate limit reached so unable to get links.",
+                        "red",
+                    )
                 )
-            )
-            return
-        elif resp.status_code == 401:
-            intelxAPIIssue = True
-            writerr(
-                colored(
-                    "IntelX - [ 401 ] Not authorized. The source requires a paid API key. Check your API key is correct.",
-                    "red",
+                return
+            elif resp.status_code == 401:
+                # Retry with free endpoint if paid endpoint was used and auth failed
+                if intelx_tls.INTELX_BASE != INTELX_BASES[-1]:
+                    setIntelxBase(INTELX_BASES[-1])
+                    continue
+                intelxAPIIssue = True
+                writerr(
+                    colored(
+                        "IntelX - [ 401 ] Not authorized. Check your API key is correct.",
+                        "red",
+                    )
                 )
-            )
-            return
-        elif resp.status_code == 402:
-            intelxAPIIssue = True
-            if credits.startswith("0/"):
+                return
+            elif resp.status_code == 402:
+                # If we were on paid, fall back to free and retry once
+                if intelx_tls.INTELX_BASE != INTELX_BASES[-1]:
+                    setIntelxBase(INTELX_BASES[-1])
+                    continue
+                intelxAPIIssue = True
+                if credits.startswith("0/"):
+                    writerr(
+                        colored(
+                            "IntelX - [ 402 ] You have run out of daily credits on Intelx ("
+                            + credits
+                            + ").",
+                            "red",
+                        )
+                    )
+                else:
+                    writerr(
+                        colored(
+                            "IntelX - [ 402 ] It appears you have run out of daily credits on Intelx.",
+                            "red",
+                        )
+                    )
+                return
+            elif resp.status_code == 403:
+                intelxAPIIssue = True
                 writerr(
                     colored(
-                        "IntelX - [ 402 ] You have run out of daily credits on Intelx ("
-                        + credits
-                        + ").",
+                        "IntelX - [ 403 ] Permission denied. Check your API key is correct.",
                         "red",
                     )
                 )
+                return
             else:
                 writerr(
                     colored(
-                        "IntelX - [ 402 ] It appears you have run out of daily credits on Intelx.",
+                        "IntelX - [ "
+                        + str(resp.status_code)
+                        + " ] Unable to get links from intelx.io",
                         "red",
                     )
                 )
-            return
-        elif resp.status_code == 403:
-            intelxAPIIssue = True
-            writerr(
-                colored(
-                    "IntelX - [ 403 ] Permission denied. Check your API key is correct.",
-                    "red",
-                )
-            )
-            return
-        elif resp.status_code != 200:
-            writerr(
-                colored(
-                    "IntelX - [ " + str(resp.status_code) + " ] Unable to get links from intelx.io",
-                    "red",
-                )
-            )
+                return
+
+        # Double check we have a valid response
+        if resp is None or resp.status_code != 200:
             return
 
         # Get the JSON response
@@ -4714,7 +4794,7 @@ def processIntelxType(target, credits):
                 break
             try:
                 resp = session.get(
-                    INTELX_RESULTS_URL + id,
+                    intelx_tls.INTELX_RESULTS_URL + id,
                     headers={"User-Agent": userAgent, "X-Key": INTELX_API_KEY},
                 )
                 requestsMade = requestsMade + 1
@@ -4769,19 +4849,13 @@ def processIntelxType(target, credits):
 
 def getIntelxAccountInfo() -> str:
     """
-    Get the account info and return the number of Credits remainiing from the /phonebook/search
+    Get the account info and return the number of Credits remaining from the /phonebook/search
     """
+    initIntelxTls()
     try:
-        # Choose a random user agent string to use for any requests
-        userAgent = random.choice(USER_AGENT)
-        session = requests.Session()
-        session.mount("https://", HTTP_ADAPTER)
-        session.mount("http://", HTTP_ADAPTER)
-        # Pass the API key in the X-Key header too.
-        resp = session.get(
-            INTELX_ACCOUNT_URL,
-            headers={"User-Agent": userAgent, "X-Key": INTELX_API_KEY},
-        )
+        resp = chooseIntelxBase(INTELX_API_KEY)
+        if resp is None or resp.status_code != 200:
+            return "Unknown"
         jsonResp = json.loads(resp.text.strip())
         credits = str(
             jsonResp.get("paths", {}).get("/phonebook/search", {}).get("Credit", "Unknown")
@@ -4812,6 +4886,7 @@ def getIntelxUrls():
 
         stopSourceIntelx = False
         linksFoundIntelx = set()
+        initIntelxTls()
 
         credits = getIntelxAccountInfo()
         if verbose():
@@ -4822,7 +4897,7 @@ def getIntelxUrls():
                     + "): ",
                     "magenta",
                 )
-                + colored(INTELX_SEARCH_URL + "\n", "white")
+                + colored(intelx_tls.INTELX_SEARCH_URL + "\n", "white")
             )
 
         if not args.check_only:

{waymore-7.4.dist-info → waymore-7.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: waymore
-Version: 7.4
+Version: 7.5
 Summary: Find way more from the Wayback Machine, Common Crawl, Alien Vault OTX, URLScan, VirusTotal & Intelligence X!
 Home-page: https://github.com/xnl-h4ck3r/waymore
 Author: xnl-h4ck3r
@@ -21,12 +21,12 @@ Dynamic: license-file
 
 <center><img src="https://github.com/xnl-h4ck3r/waymore/blob/main/waymore/images/title.png"></center>
 
-## About - v7.4
+## About - v7.5
 
-The idea behind **waymore** is to find even more links from the Wayback Machine than other existing tools.
+The idea behind **waymore** is to find even more links from the Wayback Machine (plus other sources) than other existing tools.
 
-👉 The biggest difference between **waymore** and other tools is that it can also **download the archived responses** for URLs on wayback machine so that you can then search these for even more links, developer comments, extra parameters, etc. etc.
-👉 Also, other tools do not currenrtly deal with the rate limiting now in place by the sources, and will often just stop with incomplete results and not let you know they are incomplete.
+👉 The biggest difference between **waymore** and other tools is that it can also **download the archived responses** for URLs on wayback machine (and URLScan) so that you can then search these for even more links, developer comments, extra parameters, etc. etc.
+👉 Also, other tools do not currently deal with the rate limiting now in place by the sources, and will often just stop with incomplete results and not let you know they are incomplete.
 
 Anyone who does bug bounty will have likely used the amazing [waybackurls](https://github.com/tomnomnom/waybackurls) by @TomNomNoms. This tool gets URLs from [web.archive.org](https://web.archive.org) and additional links (if any) from one of the index collections on [index.commoncrawl.org](http://index.commoncrawl.org/).
 You would have also likely used the amazing [gau](https://github.com/lc/gau) by @hacker\_ which also finds URL's from wayback archive, Common Crawl, but also from Alien Vault, URLScan, Virus Total and Intelligence X.
@@ -37,7 +37,7 @@ Now **waymore** gets URL's from ALL of those sources too (with ability to filter
 - Alien Vault OTX (otx.alienvault.com)
 - URLScan (urlscan.io)
 - Virus Total (virustotal.com)
-- Intelligence X (intelx.io) - PAID SOURCE ONLY
+- Intelligence X (intelx.io) - ACADEMIA OR PAID TIERS ONLY
 
 👉 It's a point that many seem to miss, so I'll just add it again :) ... The biggest difference between **waymore** and other tools is that it can also **download the archived responses** for URLs on wayback machine so that you can then search these for even more links, developer comments, extra parameters, etc. etc.
 
@@ -175,7 +175,7 @@ The `config.yml` file (typically in `~/.config/waymore/`) have values that can b
 - `TELEGRAM_BOT_TOKEN` - If the `--notify-telegram` argument is passed, `waymore` will use this token to send a notification to Telegram.
 - `TELEGRAM_CHAT_ID` - If the `--notify-telegram` argument is passed, `waymore` will send the notification to this chat ID.
 - `DEFAULT_OUTPUT_DIR` - This is the default location of any output files written if the `-oU` and `-oR` arguments are not used. If the value of this key is blank, then it will default to the location of the `config.yml` file.
-- `INTELX_API_KEY` - You can sign up to [intelx.io here](https://intelx.io/product). It requires a paid API key to do the `/phonebook/search` through their API (as of 2024-09-01, the Phonebook service has been restricted to paid users due to constant abuse by spam accounts).
+- `INTELX_API_KEY` - You can sign up to [intelx.io here](https://intelx.io/product). It requires an academia or paid API key to do the `/phonebook/search` through their API (as of 2024-09-01, the Phonebook service has been restricted to academia or paid users due to constant abuse by spam accounts). You can get a free API key for academic use if you sign up with a valid academic email address.
 
   **NOTE: The MIME types cannot be filtered for Alien Vault OTX, Virus Total and Intelligence X because they don't have the ability to filter on MIME Type. Sometimes URLScan does not have a MIME Type defined for a URL. In these cases, URLs will be included regardless of filter or match. Bear this in mind and consider excluding certain providers if this is important.**
 

waymore-7.5.dist-info/RECORD

@@ -0,0 +1,8 @@
+waymore/__init__.py,sha256=Bj0SHUhuftOvJsN9gwdGRcvQ8yI3Mk5ytfgq6DQVu74,20
+waymore/waymore.py,sha256=4GnaBKY0bcDhHQsIUc258IG5IeuPolsP8OfwL90AUUQ,280938
+waymore-7.5.dist-info/licenses/LICENSE,sha256=o_jq62xZ1YxI8tqzQKbNtqr3RW2i5sh0rk6ixCJEroU,1068
+waymore-7.5.dist-info/METADATA,sha256=QAIem6CNbsbyY4lgbwQoy4-qVWFV-qsSOxs7OZRU__Y,52640
+waymore-7.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+waymore-7.5.dist-info/entry_points.txt,sha256=YHy5EUf3r_7OTkt9jvylLjNeg7Z5yvIVm5RUAyfNcN4,49
+waymore-7.5.dist-info/top_level.txt,sha256=RFTphkWaRu1N7lUWIPUjabgCPQ3ETmNllF7qze4JJ_s,8
+waymore-7.5.dist-info/RECORD,,

waymore-7.4.dist-info/RECORD

@@ -1,8 +0,0 @@
-waymore/__init__.py,sha256=ewDGl3zNlo7l0-Jn6yNrqHtDpC0IF0FjF6rbWzh5_YA,20
-waymore/waymore.py,sha256=GT4WU-jse7nbFcODS-Ss0jUdq0iGOSLo4K7JjS3iulY,277670
-waymore-7.4.dist-info/licenses/LICENSE,sha256=o_jq62xZ1YxI8tqzQKbNtqr3RW2i5sh0rk6ixCJEroU,1068
-waymore-7.4.dist-info/METADATA,sha256=-q0twpS_Y1Dag-gkPMyNSuM1dwkpdTYZ1U8CaDNNkkQ,52474
-waymore-7.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-waymore-7.4.dist-info/entry_points.txt,sha256=YHy5EUf3r_7OTkt9jvylLjNeg7Z5yvIVm5RUAyfNcN4,49
-waymore-7.4.dist-info/top_level.txt,sha256=RFTphkWaRu1N7lUWIPUjabgCPQ3ETmNllF7qze4JJ_s,8
-waymore-7.4.dist-info/RECORD,,