opensquat 2.3.0__tar.gz → 2.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {opensquat-2.3.0 → opensquat-2.3.1}/PKG-INFO +26 -5
- {opensquat-2.3.0 → opensquat-2.3.1}/README.md +25 -4
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/__init__.py +1 -1
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/arg_parser.py +29 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/cli.py +5 -1
- opensquat-2.3.1/opensquat/phishing.py +337 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat.egg-info/PKG-INFO +26 -5
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat.egg-info/SOURCES.txt +1 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/pyproject.toml +1 -1
- {opensquat-2.3.0 → opensquat-2.3.1}/tests/test_arg_parser.py +27 -0
- opensquat-2.3.1/tests/test_phishing.py +395 -0
- opensquat-2.3.0/opensquat/phishing.py +0 -164
- {opensquat-2.3.0 → opensquat-2.3.1}/LICENSE +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/api_client.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/app.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/auth.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/check_update.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/ct.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/dns_resolvers.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/dns_validator.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/feed_manager.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/file_input.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/homograph.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/output.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/port_check.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/squatting_detector.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/validations.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat/vt.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat.egg-info/dependency_links.txt +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat.egg-info/entry_points.txt +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat.egg-info/requires.txt +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/opensquat.egg-info/top_level.txt +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/setup.cfg +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/tests/test_api_client.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/tests/test_auth.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/tests/test_ct.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/tests/test_feed_manager.py +0 -0
- {opensquat-2.3.0 → opensquat-2.3.1}/tests/test_validations.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opensquat
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.1
|
|
4
4
|
Summary: openSquat - Detection of domain squatting, typosquatting, IDN homograph attacks, and phishing threats
|
|
5
5
|
Author-email: Andre Tenreiro <andre@opensquat.com>
|
|
6
6
|
License-Expression: GPL-3.0-only
|
|
@@ -36,7 +36,8 @@ Dynamic: license-file
|
|
|
36
36
|
<h1 align="center">openSquat Core</h1>
|
|
37
37
|
|
|
38
38
|
<p align="center">
|
|
39
|
-
<a href="
|
|
39
|
+
<a href="https://pypi.org/project/opensquat/"><img src="https://img.shields.io/pypi/v/opensquat" alt="PyPI"></a>
|
|
40
|
+
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10+-blue.svg" alt="Python 3.10+"></a>
|
|
40
41
|
<a href="https://github.com/atenreiro/opensquat/blob/master/LICENSE"><img src="https://img.shields.io/badge/License-GPLv3-blue.svg" alt="License: GPL v3"></a>
|
|
41
42
|
<a href="https://github.com/atenreiro/opensquat/issues"><img src="https://img.shields.io/github/issues/atenreiro/opensquat" alt="GitHub issues"></a>
|
|
42
43
|
<a href="https://github.com/atenreiro/opensquat/stargazers"><img src="https://img.shields.io/github/stars/atenreiro/opensquat" alt="GitHub stars"></a>
|
|
@@ -54,6 +55,7 @@ Dynamic: license-file
|
|
|
54
55
|
- [Requirements](#-requirements)
|
|
55
56
|
- [Usage](#-usage)
|
|
56
57
|
- [Premium and API Modes](#-premium-and-api-modes)
|
|
58
|
+
- [MCP Server](#-mcp-server)
|
|
57
59
|
- [Configuration](#%EF%B8%8F-configuration)
|
|
58
60
|
- [Automation](#-automation)
|
|
59
61
|
- [CLI Reference](#-cli-reference)
|
|
@@ -82,9 +84,6 @@ openSquat is an **Open Source Intelligence (OSINT)** security tool that identifi
|
|
|
82
84
|
> **"A powerful swiss army knife for brand protection"**
|
|
83
85
|
> — [WhoisXML API Blog](https://www.whoisxmlapi.com/blog/orchestrating-open-source-software-and-whois-newly-registered-domain-data-feeds-to-fight-the-typosquatting-plague), August 2022
|
|
84
86
|
|
|
85
|
-
> **"A tool with insane power to fight typosquatting and all related types of cyber mischief."**
|
|
86
|
-
> — [WhoisXML API Blog](https://www.whoisxmlapi.com/blog/orchestrating-open-source-software-and-whois-newly-registered-domain-data-feeds-to-fight-the-typosquatting-plague), August 2022
|
|
87
|
-
|
|
88
87
|
> **"A handy tool for collecting information on newly registered domains."** — ranked Top 5 phishing detection tool
|
|
89
88
|
> — [SOCRadar Blog](https://socradar.io/blog/top-5-tools-for-phishing-domain-detection/), July 2022
|
|
90
89
|
|
|
@@ -141,6 +140,12 @@ python3 opensquat.py -k keywords.txt
|
|
|
141
140
|
|
|
142
141
|
> **Repo users:** in all the examples below, replace `opensquat` with `python3 opensquat.py` to run from a cloned checkout.
|
|
143
142
|
|
|
143
|
+
### See it in action
|
|
144
|
+
|
|
145
|
+
<p align="center">
|
|
146
|
+
<img src="screenshots/openSquat_v2.3.0.png" alt="openSquat 2.3.0 scanning the daily NRD feed for lookalikes of google, facebook, amazon and paypal" width="650"/>
|
|
147
|
+
</p>
|
|
148
|
+
|
|
144
149
|
---
|
|
145
150
|
|
|
146
151
|
## 📦 Requirements
|
|
@@ -316,6 +321,21 @@ Premium API (`--api`) is incompatible with `--doppelganger` and `-d/--domains`.
|
|
|
316
321
|
|
|
317
322
|
---
|
|
318
323
|
|
|
324
|
+
## 🔌 MCP Server
|
|
325
|
+
|
|
326
|
+
openSquat is also available as a **Model Context Protocol (MCP) server**, so AI assistants and agents (Claude, and any other MCP-compatible client) can query lookalike-domain intelligence directly:
|
|
327
|
+
|
|
328
|
+
| Tool | What it does | Quota cost |
|
|
329
|
+
|------|--------------|------------|
|
|
330
|
+
| `search_lookalike_domains` | Find newly registered domains impersonating a brand or keyword (phishing, brand abuse, typosquatting) | 1 query per search |
|
|
331
|
+
| `validate_api_key` | Confirm the connection is authenticated | Free |
|
|
332
|
+
|
|
333
|
+
Authentication uses the same openSquat account as the Premium modes: either sign in with OAuth, or supply your API key (`os_...`) via the `Authorization: Bearer` header (or `X-OpenSquat-Key`). Searches draw from the same plan quota as the Premium API.
|
|
334
|
+
|
|
335
|
+
See [opensquat.com](https://opensquat.com) for the MCP endpoint and client setup instructions.
|
|
336
|
+
|
|
337
|
+
---
|
|
338
|
+
|
|
319
339
|
## ⚙️ Configuration
|
|
320
340
|
|
|
321
341
|
### Keywords File (`keywords.txt`)
|
|
@@ -378,6 +398,7 @@ Run daily via crontab:
|
|
|
378
398
|
| `--doppelganger` | — | Doppelganger-only mode (keyword in domain + reachability check) |
|
|
379
399
|
| `--ct` | — | Search Certificate Transparency logs |
|
|
380
400
|
| `--phishing` | — | Cross-reference phishing database |
|
|
401
|
+
| `--phishing-fallback` | — | Optional `https` list of phishing domains/URLs, used only when the default database cannot be downloaded |
|
|
381
402
|
| `--subdomains` | — | Fetch subdomains via VirusTotal |
|
|
382
403
|
| `--portcheck` | — | Check for open ports 80/443 |
|
|
383
404
|
| `--vt` | — | Validate against VirusTotal |
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
<h1 align="center">openSquat Core</h1>
|
|
6
6
|
|
|
7
7
|
<p align="center">
|
|
8
|
-
<a href="
|
|
8
|
+
<a href="https://pypi.org/project/opensquat/"><img src="https://img.shields.io/pypi/v/opensquat" alt="PyPI"></a>
|
|
9
|
+
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10+-blue.svg" alt="Python 3.10+"></a>
|
|
9
10
|
<a href="https://github.com/atenreiro/opensquat/blob/master/LICENSE"><img src="https://img.shields.io/badge/License-GPLv3-blue.svg" alt="License: GPL v3"></a>
|
|
10
11
|
<a href="https://github.com/atenreiro/opensquat/issues"><img src="https://img.shields.io/github/issues/atenreiro/opensquat" alt="GitHub issues"></a>
|
|
11
12
|
<a href="https://github.com/atenreiro/opensquat/stargazers"><img src="https://img.shields.io/github/stars/atenreiro/opensquat" alt="GitHub stars"></a>
|
|
@@ -23,6 +24,7 @@
|
|
|
23
24
|
- [Requirements](#-requirements)
|
|
24
25
|
- [Usage](#-usage)
|
|
25
26
|
- [Premium and API Modes](#-premium-and-api-modes)
|
|
27
|
+
- [MCP Server](#-mcp-server)
|
|
26
28
|
- [Configuration](#%EF%B8%8F-configuration)
|
|
27
29
|
- [Automation](#-automation)
|
|
28
30
|
- [CLI Reference](#-cli-reference)
|
|
@@ -51,9 +53,6 @@ openSquat is an **Open Source Intelligence (OSINT)** security tool that identifi
|
|
|
51
53
|
> **"A powerful swiss army knife for brand protection"**
|
|
52
54
|
> — [WhoisXML API Blog](https://www.whoisxmlapi.com/blog/orchestrating-open-source-software-and-whois-newly-registered-domain-data-feeds-to-fight-the-typosquatting-plague), August 2022
|
|
53
55
|
|
|
54
|
-
> **"A tool with insane power to fight typosquatting and all related types of cyber mischief."**
|
|
55
|
-
> — [WhoisXML API Blog](https://www.whoisxmlapi.com/blog/orchestrating-open-source-software-and-whois-newly-registered-domain-data-feeds-to-fight-the-typosquatting-plague), August 2022
|
|
56
|
-
|
|
57
56
|
> **"A handy tool for collecting information on newly registered domains."** — ranked Top 5 phishing detection tool
|
|
58
57
|
> — [SOCRadar Blog](https://socradar.io/blog/top-5-tools-for-phishing-domain-detection/), July 2022
|
|
59
58
|
|
|
@@ -110,6 +109,12 @@ python3 opensquat.py -k keywords.txt
|
|
|
110
109
|
|
|
111
110
|
> **Repo users:** in all the examples below, replace `opensquat` with `python3 opensquat.py` to run from a cloned checkout.
|
|
112
111
|
|
|
112
|
+
### See it in action
|
|
113
|
+
|
|
114
|
+
<p align="center">
|
|
115
|
+
<img src="screenshots/openSquat_v2.3.0.png" alt="openSquat 2.3.0 scanning the daily NRD feed for lookalikes of google, facebook, amazon and paypal" width="650"/>
|
|
116
|
+
</p>
|
|
117
|
+
|
|
113
118
|
---
|
|
114
119
|
|
|
115
120
|
## 📦 Requirements
|
|
@@ -285,6 +290,21 @@ Premium API (`--api`) is incompatible with `--doppelganger` and `-d/--domains`.
|
|
|
285
290
|
|
|
286
291
|
---
|
|
287
292
|
|
|
293
|
+
## 🔌 MCP Server
|
|
294
|
+
|
|
295
|
+
openSquat is also available as a **Model Context Protocol (MCP) server**, so AI assistants and agents (Claude, and any other MCP-compatible client) can query lookalike-domain intelligence directly:
|
|
296
|
+
|
|
297
|
+
| Tool | What it does | Quota cost |
|
|
298
|
+
|------|--------------|------------|
|
|
299
|
+
| `search_lookalike_domains` | Find newly registered domains impersonating a brand or keyword (phishing, brand abuse, typosquatting) | 1 query per search |
|
|
300
|
+
| `validate_api_key` | Confirm the connection is authenticated | Free |
|
|
301
|
+
|
|
302
|
+
Authentication uses the same openSquat account as the Premium modes: either sign in with OAuth, or supply your API key (`os_...`) via the `Authorization: Bearer` header (or `X-OpenSquat-Key`). Searches draw from the same plan quota as the Premium API.
|
|
303
|
+
|
|
304
|
+
See [opensquat.com](https://opensquat.com) for the MCP endpoint and client setup instructions.
|
|
305
|
+
|
|
306
|
+
---
|
|
307
|
+
|
|
288
308
|
## ⚙️ Configuration
|
|
289
309
|
|
|
290
310
|
### Keywords File (`keywords.txt`)
|
|
@@ -347,6 +367,7 @@ Run daily via crontab:
|
|
|
347
367
|
| `--doppelganger` | — | Doppelganger-only mode (keyword in domain + reachability check) |
|
|
348
368
|
| `--ct` | — | Search Certificate Transparency logs |
|
|
349
369
|
| `--phishing` | — | Cross-reference phishing database |
|
|
370
|
+
| `--phishing-fallback` | — | Optional `https` list of phishing domains/URLs, used only when the default database cannot be downloaded |
|
|
350
371
|
| `--subdomains` | — | Fetch subdomains via VirusTotal |
|
|
351
372
|
| `--portcheck` | — | Check for open ports 80/443 |
|
|
352
373
|
| `--vt` | — | Validate against VirusTotal |
|
|
@@ -11,6 +11,7 @@ software licensed under GNU version 3
|
|
|
11
11
|
|
|
12
12
|
"""
|
|
13
13
|
import argparse
|
|
14
|
+
from urllib.parse import urlparse
|
|
14
15
|
|
|
15
16
|
from colorama import Fore, Style
|
|
16
17
|
|
|
@@ -37,6 +38,26 @@ def validate_type(file_type):
|
|
|
37
38
|
return file_type
|
|
38
39
|
|
|
39
40
|
|
|
41
|
+
def validate_https_url(url):
|
|
42
|
+
"""
|
|
43
|
+
Validate a URL given on the command line.
|
|
44
|
+
|
|
45
|
+
Args:
|
|
46
|
+
url: string containing the URL, must be https with a host.
|
|
47
|
+
|
|
48
|
+
Return:
|
|
49
|
+
url
|
|
50
|
+
|
|
51
|
+
Raise:
|
|
52
|
+
If value is not valid, raise an exception to argparse
|
|
53
|
+
"""
|
|
54
|
+
parsed = urlparse(str(url))
|
|
55
|
+
|
|
56
|
+
if parsed.scheme != "https" or not parsed.hostname:
|
|
57
|
+
raise argparse.ArgumentTypeError("URL must start with https://")
|
|
58
|
+
return url
|
|
59
|
+
|
|
60
|
+
|
|
40
61
|
def validate_confidence(confidence_level):
|
|
41
62
|
"""
|
|
42
63
|
Validate confidence_level.
|
|
@@ -194,6 +215,14 @@ def get_args():
|
|
|
194
215
|
default="",
|
|
195
216
|
help="search known and active Phishing sites (arg: output.txt)",
|
|
196
217
|
)
|
|
218
|
+
parser.add_argument(
|
|
219
|
+
"--phishing-fallback",
|
|
220
|
+
type=validate_https_url,
|
|
221
|
+
default=None,
|
|
222
|
+
metavar="URL",
|
|
223
|
+
help="https list of phishing domains/URLs (one per line) used only "
|
|
224
|
+
"when the default Phishing Database cannot be downloaded",
|
|
225
|
+
)
|
|
197
226
|
parser.add_argument(
|
|
198
227
|
"--subdomains",
|
|
199
228
|
action="store_true",
|
|
@@ -301,7 +301,10 @@ def main():
|
|
|
301
301
|
|
|
302
302
|
# Check for phishing
|
|
303
303
|
if (args.phishing != ""):
|
|
304
|
-
|
|
304
|
+
phishing_check = phishing.Phishing(
|
|
305
|
+
fallback_db=args.phishing_fallback
|
|
306
|
+
)
|
|
307
|
+
file_phishing = phishing_check.main(args.keywords)
|
|
305
308
|
output.SaveFile().main(args.phishing, "txt", file_phishing)
|
|
306
309
|
|
|
307
310
|
# Check if domain has webserver port opened
|
|
@@ -353,6 +356,7 @@ def main():
|
|
|
353
356
|
|
|
354
357
|
if (args.phishing != ""):
|
|
355
358
|
print("[*] Phishing results:", args.phishing)
|
|
359
|
+
print("[*] Phishing source:", phishing_check.source)
|
|
356
360
|
print("[*] Active Phishing sites:", len(file_phishing))
|
|
357
361
|
|
|
358
362
|
print("[*] Running time: %s seconds" % end_time_squatting)
|
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
# Module: phishing.py
|
|
3
|
+
"""
|
|
4
|
+
openSquat
|
|
5
|
+
|
|
6
|
+
(c) Andre Tenreiro
|
|
7
|
+
|
|
8
|
+
* https://github.com/atenreiro/opensquat
|
|
9
|
+
|
|
10
|
+
software licensed under GNU version 3
|
|
11
|
+
"""
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import sys
|
|
15
|
+
import tempfile
|
|
16
|
+
import time
|
|
17
|
+
from urllib.parse import urlparse
|
|
18
|
+
|
|
19
|
+
import requests
|
|
20
|
+
from opensquat import file_input
|
|
21
|
+
|
|
22
|
+
from colorama import Fore, Style
|
|
23
|
+
|
|
24
|
+
# Phishing lists come from third parties and are untrusted input. These
|
|
25
|
+
# limits keep a compromised or misbehaving source from exhausting memory
|
|
26
|
+
# or disk, or from smuggling anything but hostnames/URLs into the local DB.
|
|
27
|
+
MAX_DB_BYTES = 256 * 1024 * 1024
|
|
28
|
+
MAX_LINE_LENGTH = 2048
|
|
29
|
+
CHUNK_SIZE = 64 * 1024
|
|
30
|
+
# Minimum share of non-comment lines that must look like a host or URL.
|
|
31
|
+
# Below this the source is treated as broken (HTML page, wrong dataset...).
|
|
32
|
+
MIN_VALID_RATIO = 0.5
|
|
33
|
+
# Content types that can never be a plain-text list.
|
|
34
|
+
REJECTED_CONTENT_TYPES = ("html", "xml", "json")
|
|
35
|
+
|
|
36
|
+
# One entry per line: a hostname (optionally with a port) or an http(s)
|
|
37
|
+
# URL. Strict allow-list of characters: no whitespace, control characters,
|
|
38
|
+
# quotes or backslashes can pass. Matched against the lowercased line.
|
|
39
|
+
LINE_RE = re.compile(
|
|
40
|
+
r"(?:https?://)?[a-z0-9][a-z0-9_.-]*"
|
|
41
|
+
r"(?::[0-9]{1,5})?"
|
|
42
|
+
r"(?:[/?#][a-z0-9._~:/?#@!$&()*+,;=%-]*)?"
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def redact_url(url):
|
|
47
|
+
"""
|
|
48
|
+
Return scheme://host[:port]/path without userinfo, query or fragment.
|
|
49
|
+
|
|
50
|
+
A user-supplied list URL may carry credentials or an API token; only
|
|
51
|
+
this form is ever printed or kept for display.
|
|
52
|
+
"""
|
|
53
|
+
parsed = urlparse(str(url))
|
|
54
|
+
host = parsed.hostname or ""
|
|
55
|
+
try:
|
|
56
|
+
port = parsed.port
|
|
57
|
+
except ValueError:
|
|
58
|
+
port = None
|
|
59
|
+
if port:
|
|
60
|
+
host = host + ":" + str(port)
|
|
61
|
+
return parsed.scheme + "://" + host + parsed.path
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class Phishing:
|
|
65
|
+
|
|
66
|
+
"""
|
|
67
|
+
Class Phishing.
|
|
68
|
+
|
|
69
|
+
To use:
|
|
70
|
+
Phishing().main(keyword)
|
|
71
|
+
|
|
72
|
+
Attribute:
|
|
73
|
+
keyword: list of keywords
|
|
74
|
+
"""
|
|
75
|
+
def __init__(self, fallback_db=None):
|
|
76
|
+
self.phishing_db = "https://phish.co.za" \
|
|
77
|
+
"/latest/" \
|
|
78
|
+
"phishing-domains-ACTIVE.txt"
|
|
79
|
+
# Optional second https list, tried only when the Phishing Database
|
|
80
|
+
# cannot be downloaded. There is no default: it is chosen by the
|
|
81
|
+
# user with --phishing-fallback.
|
|
82
|
+
self.fallback_db = fallback_db
|
|
83
|
+
# URL of the list that produced the local database in this run.
|
|
84
|
+
self.source = None
|
|
85
|
+
self.phishing_filename = "phishing.db"
|
|
86
|
+
self.keyword = ""
|
|
87
|
+
self.keywords_filename = ""
|
|
88
|
+
self.list_domains = []
|
|
89
|
+
self.keywords_total = 0
|
|
90
|
+
|
|
91
|
+
def set_keywords(self, keywords):
|
|
92
|
+
self.keywords_filename = keywords
|
|
93
|
+
|
|
94
|
+
@staticmethod
|
|
95
|
+
def URL_contains(keyword, phishing):
|
|
96
|
+
if keyword in phishing:
|
|
97
|
+
return True
|
|
98
|
+
|
|
99
|
+
return False
|
|
100
|
+
|
|
101
|
+
def count_keywords(self):
|
|
102
|
+
self.keywords_total = file_input.InputFile().main(
|
|
103
|
+
self.keywords_filename,
|
|
104
|
+
None
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
def check_phishing(self):
|
|
108
|
+
# keyword iteration
|
|
109
|
+
i = 0
|
|
110
|
+
|
|
111
|
+
# Open phishing DB
|
|
112
|
+
with open(self.keywords_filename, mode='r', encoding='utf-8') as f_key:
|
|
113
|
+
for raw_keyword in f_key:
|
|
114
|
+
keyword = raw_keyword.strip().lower()
|
|
115
|
+
|
|
116
|
+
# Skip blank, whitespace-only, and comment lines. Replaces
|
|
117
|
+
# the old line[0] indexing pattern that crashed on empty
|
|
118
|
+
# input and also miscounted whitespace-only lines.
|
|
119
|
+
if not keyword or keyword.startswith("#"):
|
|
120
|
+
continue
|
|
121
|
+
|
|
122
|
+
i += 1
|
|
123
|
+
print(
|
|
124
|
+
Fore.WHITE + "\n[*] Verifying keyword:",
|
|
125
|
+
keyword,
|
|
126
|
+
"[",
|
|
127
|
+
i,
|
|
128
|
+
"/",
|
|
129
|
+
self.keywords_total,
|
|
130
|
+
"]" + Style.RESET_ALL,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
with open(self.phishing_filename, mode='r', encoding='utf-8') as f_phishing:
|
|
134
|
+
for site in f_phishing:
|
|
135
|
+
# Lowercase AND strip newline in one pass. The old
|
|
136
|
+
# version assigned site.lower() then overwrote it
|
|
137
|
+
# with site.replace("\n", ""), producing a case-
|
|
138
|
+
# sensitive comparison that missed mixed-case hits.
|
|
139
|
+
phishing_site = site.replace("\n", "").lower()
|
|
140
|
+
|
|
141
|
+
# A list may start with a "#" header line, which is
|
|
142
|
+
# not a site and must not match a keyword.
|
|
143
|
+
if phishing_site.startswith("#"):
|
|
144
|
+
continue
|
|
145
|
+
|
|
146
|
+
if self.URL_contains(keyword, phishing_site):
|
|
147
|
+
print(
|
|
148
|
+
Style.BRIGHT + Fore.YELLOW +
|
|
149
|
+
" \\_ Similarity detected between",
|
|
150
|
+
keyword,
|
|
151
|
+
"and",
|
|
152
|
+
phishing_site,
|
|
153
|
+
"" + Style.RESET_ALL
|
|
154
|
+
)
|
|
155
|
+
self.list_domains.append(phishing_site)
|
|
156
|
+
|
|
157
|
+
return self.list_domains
|
|
158
|
+
|
|
159
|
+
@staticmethod
|
|
160
|
+
def sanitize_db(data):
|
|
161
|
+
"""
|
|
162
|
+
Reduce a downloaded list to safe lines.
|
|
163
|
+
|
|
164
|
+
Return (lines, rejected): the lowercased lines that look like a
|
|
165
|
+
hostname or an http(s) URL, and how many non-comment lines were
|
|
166
|
+
dropped. Bad bytes, control characters, ANSI escape sequences,
|
|
167
|
+
spaces and over-long lines never pass, so they cannot reach the
|
|
168
|
+
terminal, the output file or the matching code.
|
|
169
|
+
"""
|
|
170
|
+
text = data.decode("utf-8", errors="replace")
|
|
171
|
+
lines = []
|
|
172
|
+
rejected = 0
|
|
173
|
+
|
|
174
|
+
for raw in text.split("\n"):
|
|
175
|
+
line = raw.strip().lower()
|
|
176
|
+
if not line or line.startswith("#"):
|
|
177
|
+
continue
|
|
178
|
+
if len(line) > MAX_LINE_LENGTH or not LINE_RE.fullmatch(line):
|
|
179
|
+
rejected += 1
|
|
180
|
+
continue
|
|
181
|
+
lines.append(line)
|
|
182
|
+
|
|
183
|
+
return lines, rejected
|
|
184
|
+
|
|
185
|
+
def download_db(self, url):
|
|
186
|
+
"""
|
|
187
|
+
Download one phishing list.
|
|
188
|
+
|
|
189
|
+
Return the raw body, or None when the source is not usable: not
|
|
190
|
+
https, unreachable, not a 200, an HTML/XML/JSON page, empty or
|
|
191
|
+
larger than MAX_DB_BYTES. A server that is down answers with an
|
|
192
|
+
error page (phish.co.za returns a short 503 or 403 HTML page), and
|
|
193
|
+
that page must not be saved as the database: every keyword would
|
|
194
|
+
then silently match nothing.
|
|
195
|
+
"""
|
|
196
|
+
print("[*] Downloading fresh Phishing DB from", redact_url(url))
|
|
197
|
+
|
|
198
|
+
if urlparse(url).scheme != "https":
|
|
199
|
+
print("[!] Phishing DB must be served over https:", redact_url(url))
|
|
200
|
+
return None
|
|
201
|
+
|
|
202
|
+
r = None
|
|
203
|
+
try:
|
|
204
|
+
# Timeout so a hung phishing DB server cannot freeze the CLI.
|
|
205
|
+
r = requests.get(url, timeout=60, stream=True)
|
|
206
|
+
|
|
207
|
+
if r.status_code != 200:
|
|
208
|
+
print("[!] Phishing DB unavailable: HTTP", r.status_code)
|
|
209
|
+
return None
|
|
210
|
+
|
|
211
|
+
# No hop of a redirect chain may use plain http, even when the
|
|
212
|
+
# chain ends on https: that hop could have been tampered with.
|
|
213
|
+
hops = list(r.history) + [r]
|
|
214
|
+
if any(urlparse(str(h.url)).scheme != "https" for h in hops):
|
|
215
|
+
print("[!] Phishing DB redirected through a non-https URL")
|
|
216
|
+
return None
|
|
217
|
+
|
|
218
|
+
content_type = str(r.headers.get("content-type", "")).lower()
|
|
219
|
+
if any(t in content_type for t in REJECTED_CONTENT_TYPES):
|
|
220
|
+
# The header is server-controlled: never echo it.
|
|
221
|
+
print("[!] Phishing DB is not a text list")
|
|
222
|
+
return None
|
|
223
|
+
|
|
224
|
+
try:
|
|
225
|
+
declared = int(r.headers.get("content-length", 0))
|
|
226
|
+
except (TypeError, ValueError):
|
|
227
|
+
declared = 0
|
|
228
|
+
if declared > MAX_DB_BYTES:
|
|
229
|
+
print("[!] Phishing DB is too large:", declared, "bytes")
|
|
230
|
+
return None
|
|
231
|
+
|
|
232
|
+
body = bytearray()
|
|
233
|
+
for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
|
|
234
|
+
body.extend(chunk)
|
|
235
|
+
if len(body) > MAX_DB_BYTES:
|
|
236
|
+
print("[!] Phishing DB is too large, aborting download")
|
|
237
|
+
return None
|
|
238
|
+
except requests.exceptions.RequestException as e:
|
|
239
|
+
print("[!] Phishing DB not reachable:", type(e).__name__)
|
|
240
|
+
return None
|
|
241
|
+
finally:
|
|
242
|
+
if r is not None:
|
|
243
|
+
r.close()
|
|
244
|
+
|
|
245
|
+
if len(body) == 0:
|
|
246
|
+
print("[!] Phishing DB is empty")
|
|
247
|
+
return None
|
|
248
|
+
|
|
249
|
+
return bytes(body)
|
|
250
|
+
|
|
251
|
+
def fetch_list(self, url):
|
|
252
|
+
"""
|
|
253
|
+
Download one list and return its sanitized lines, or None when the
|
|
254
|
+
source is unusable.
|
|
255
|
+
"""
|
|
256
|
+
data = self.download_db(url)
|
|
257
|
+
if data is None:
|
|
258
|
+
return None
|
|
259
|
+
|
|
260
|
+
lines, rejected = self.sanitize_db(data)
|
|
261
|
+
total = len(lines) + rejected
|
|
262
|
+
|
|
263
|
+
if not lines or len(lines) / total < MIN_VALID_RATIO:
|
|
264
|
+
print(
|
|
265
|
+
"[!] Phishing DB rejected: only", len(lines), "of", total,
|
|
266
|
+
"lines are valid hosts or URLs"
|
|
267
|
+
)
|
|
268
|
+
return None
|
|
269
|
+
|
|
270
|
+
if rejected:
|
|
271
|
+
print("[*] Ignored", rejected, "invalid lines from the list")
|
|
272
|
+
|
|
273
|
+
return lines
|
|
274
|
+
|
|
275
|
+
def save_db(self, lines):
|
|
276
|
+
"""
|
|
277
|
+
Write the database atomically: a failed run never leaves a half
|
|
278
|
+
written phishing.db, nor replaces the previous one.
|
|
279
|
+
"""
|
|
280
|
+
target = os.path.abspath(self.phishing_filename)
|
|
281
|
+
fd, tmp_path = tempfile.mkstemp(
|
|
282
|
+
dir=os.path.dirname(target), prefix=".phishing.db."
|
|
283
|
+
)
|
|
284
|
+
try:
|
|
285
|
+
with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as f:
|
|
286
|
+
f.write("\n".join(lines) + "\n")
|
|
287
|
+
os.replace(tmp_path, target)
|
|
288
|
+
except BaseException:
|
|
289
|
+
if os.path.exists(tmp_path):
|
|
290
|
+
os.remove(tmp_path)
|
|
291
|
+
raise
|
|
292
|
+
|
|
293
|
+
def update_db(self):
|
|
294
|
+
|
|
295
|
+
sources = [self.phishing_db]
|
|
296
|
+
if self.fallback_db:
|
|
297
|
+
sources.append(self.fallback_db)
|
|
298
|
+
|
|
299
|
+
lines = None
|
|
300
|
+
for url in sources:
|
|
301
|
+
lines = self.fetch_list(url)
|
|
302
|
+
if lines is not None:
|
|
303
|
+
self.source = redact_url(url)
|
|
304
|
+
break
|
|
305
|
+
if url != sources[-1]:
|
|
306
|
+
print("[*] Trying the fallback list")
|
|
307
|
+
|
|
308
|
+
if lines is None:
|
|
309
|
+
print(
|
|
310
|
+
"[ERROR] No phishing list could be downloaded. " +
|
|
311
|
+
"Try again later. Exiting...\n",
|
|
312
|
+
)
|
|
313
|
+
sys.exit(-1)
|
|
314
|
+
|
|
315
|
+
print("[*] Phishing list source:", self.source)
|
|
316
|
+
print("[*] Phishing entries:", len(lines))
|
|
317
|
+
self.save_db(lines)
|
|
318
|
+
|
|
319
|
+
return True
|
|
320
|
+
|
|
321
|
+
def main(self, keywords):
|
|
322
|
+
"""
|
|
323
|
+
main function that will call other functions.
|
|
324
|
+
|
|
325
|
+
Args:
|
|
326
|
+
keyword: keyword to search for(duh)
|
|
327
|
+
|
|
328
|
+
Return:
|
|
329
|
+
none
|
|
330
|
+
"""
|
|
331
|
+
print("")
|
|
332
|
+
print("+---------- Checking Phishing sites ----------+")
|
|
333
|
+
time.sleep(2)
|
|
334
|
+
self.set_keywords(keywords)
|
|
335
|
+
self.update_db()
|
|
336
|
+
self.count_keywords()
|
|
337
|
+
return self.check_phishing()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: opensquat
|
|
3
|
-
Version: 2.3.
|
|
3
|
+
Version: 2.3.1
|
|
4
4
|
Summary: openSquat - Detection of domain squatting, typosquatting, IDN homograph attacks, and phishing threats
|
|
5
5
|
Author-email: Andre Tenreiro <andre@opensquat.com>
|
|
6
6
|
License-Expression: GPL-3.0-only
|
|
@@ -36,7 +36,8 @@ Dynamic: license-file
|
|
|
36
36
|
<h1 align="center">openSquat Core</h1>
|
|
37
37
|
|
|
38
38
|
<p align="center">
|
|
39
|
-
<a href="
|
|
39
|
+
<a href="https://pypi.org/project/opensquat/"><img src="https://img.shields.io/pypi/v/opensquat" alt="PyPI"></a>
|
|
40
|
+
<a href="https://www.python.org/downloads/"><img src="https://img.shields.io/badge/python-3.10+-blue.svg" alt="Python 3.10+"></a>
|
|
40
41
|
<a href="https://github.com/atenreiro/opensquat/blob/master/LICENSE"><img src="https://img.shields.io/badge/License-GPLv3-blue.svg" alt="License: GPL v3"></a>
|
|
41
42
|
<a href="https://github.com/atenreiro/opensquat/issues"><img src="https://img.shields.io/github/issues/atenreiro/opensquat" alt="GitHub issues"></a>
|
|
42
43
|
<a href="https://github.com/atenreiro/opensquat/stargazers"><img src="https://img.shields.io/github/stars/atenreiro/opensquat" alt="GitHub stars"></a>
|
|
@@ -54,6 +55,7 @@ Dynamic: license-file
|
|
|
54
55
|
- [Requirements](#-requirements)
|
|
55
56
|
- [Usage](#-usage)
|
|
56
57
|
- [Premium and API Modes](#-premium-and-api-modes)
|
|
58
|
+
- [MCP Server](#-mcp-server)
|
|
57
59
|
- [Configuration](#%EF%B8%8F-configuration)
|
|
58
60
|
- [Automation](#-automation)
|
|
59
61
|
- [CLI Reference](#-cli-reference)
|
|
@@ -82,9 +84,6 @@ openSquat is an **Open Source Intelligence (OSINT)** security tool that identifi
|
|
|
82
84
|
> **"A powerful swiss army knife for brand protection"**
|
|
83
85
|
> — [WhoisXML API Blog](https://www.whoisxmlapi.com/blog/orchestrating-open-source-software-and-whois-newly-registered-domain-data-feeds-to-fight-the-typosquatting-plague), August 2022
|
|
84
86
|
|
|
85
|
-
> **"A tool with insane power to fight typosquatting and all related types of cyber mischief."**
|
|
86
|
-
> — [WhoisXML API Blog](https://www.whoisxmlapi.com/blog/orchestrating-open-source-software-and-whois-newly-registered-domain-data-feeds-to-fight-the-typosquatting-plague), August 2022
|
|
87
|
-
|
|
88
87
|
> **"A handy tool for collecting information on newly registered domains."** — ranked Top 5 phishing detection tool
|
|
89
88
|
> — [SOCRadar Blog](https://socradar.io/blog/top-5-tools-for-phishing-domain-detection/), July 2022
|
|
90
89
|
|
|
@@ -141,6 +140,12 @@ python3 opensquat.py -k keywords.txt
|
|
|
141
140
|
|
|
142
141
|
> **Repo users:** in all the examples below, replace `opensquat` with `python3 opensquat.py` to run from a cloned checkout.
|
|
143
142
|
|
|
143
|
+
### See it in action
|
|
144
|
+
|
|
145
|
+
<p align="center">
|
|
146
|
+
<img src="screenshots/openSquat_v2.3.0.png" alt="openSquat 2.3.0 scanning the daily NRD feed for lookalikes of google, facebook, amazon and paypal" width="650"/>
|
|
147
|
+
</p>
|
|
148
|
+
|
|
144
149
|
---
|
|
145
150
|
|
|
146
151
|
## 📦 Requirements
|
|
@@ -316,6 +321,21 @@ Premium API (`--api`) is incompatible with `--doppelganger` and `-d/--domains`.
|
|
|
316
321
|
|
|
317
322
|
---
|
|
318
323
|
|
|
324
|
+
## 🔌 MCP Server
|
|
325
|
+
|
|
326
|
+
openSquat is also available as a **Model Context Protocol (MCP) server**, so AI assistants and agents (Claude, and any other MCP-compatible client) can query lookalike-domain intelligence directly:
|
|
327
|
+
|
|
328
|
+
| Tool | What it does | Quota cost |
|
|
329
|
+
|------|--------------|------------|
|
|
330
|
+
| `search_lookalike_domains` | Find newly registered domains impersonating a brand or keyword (phishing, brand abuse, typosquatting) | 1 query per search |
|
|
331
|
+
| `validate_api_key` | Confirm the connection is authenticated | Free |
|
|
332
|
+
|
|
333
|
+
Authentication uses the same openSquat account as the Premium modes: either sign in with OAuth, or supply your API key (`os_...`) via the `Authorization: Bearer` header (or `X-OpenSquat-Key`). Searches draw from the same plan quota as the Premium API.
|
|
334
|
+
|
|
335
|
+
See [opensquat.com](https://opensquat.com) for the MCP endpoint and client setup instructions.
|
|
336
|
+
|
|
337
|
+
---
|
|
338
|
+
|
|
319
339
|
## ⚙️ Configuration
|
|
320
340
|
|
|
321
341
|
### Keywords File (`keywords.txt`)
|
|
@@ -378,6 +398,7 @@ Run daily via crontab:
|
|
|
378
398
|
| `--doppelganger` | — | Doppelganger-only mode (keyword in domain + reachability check) |
|
|
379
399
|
| `--ct` | — | Search Certificate Transparency logs |
|
|
380
400
|
| `--phishing` | — | Cross-reference phishing database |
|
|
401
|
+
| `--phishing-fallback` | — | Optional `https` list of phishing domains/URLs, used only when the default database cannot be downloaded |
|
|
381
402
|
| `--subdomains` | — | Fetch subdomains via VirusTotal |
|
|
382
403
|
| `--portcheck` | — | Check for open ports 80/443 |
|
|
383
404
|
| `--vt` | — | Validate against VirusTotal |
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "opensquat"
|
|
7
|
-
version = "2.3.
|
|
7
|
+
version = "2.3.1"
|
|
8
8
|
description = "openSquat - Detection of domain squatting, typosquatting, IDN homograph attacks, and phishing threats"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "GPL-3.0-only"
|