nethound 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nethound/__init__.py +7 -0
- nethound/cli.py +211 -0
- nethound/config.py +27 -0
- nethound/fetchers/__init__.py +4 -0
- nethound/fetchers/registry_in.py +79 -0
- nethound/fetchers/whoisds.py +118 -0
- nethound/search.py +68 -0
- nethound-1.0.0.dist-info/METADATA +210 -0
- nethound-1.0.0.dist-info/RECORD +13 -0
- nethound-1.0.0.dist-info/WHEEL +5 -0
- nethound-1.0.0.dist-info/entry_points.txt +2 -0
- nethound-1.0.0.dist-info/licenses/LICENSE +21 -0
- nethound-1.0.0.dist-info/top_level.txt +1 -0
nethound/__init__.py
ADDED
nethound/cli.py
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import json
|
|
3
|
+
import sys
|
|
4
|
+
from datetime import datetime, timedelta
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
from nethound import __version__
|
|
8
|
+
from nethound.config import DEFAULT_DATA_DIR, parse_date
|
|
9
|
+
from nethound.fetchers.registry_in import fetch_in_domains
|
|
10
|
+
from nethound.fetchers.whoisds import fetch_whoisds_domains
|
|
11
|
+
from nethound.search import search_domains, find_matching_files
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def cmd_fetch(args) -> int:
|
|
15
|
+
try:
|
|
16
|
+
target_date = parse_date(args.date)
|
|
17
|
+
except ValueError as e:
|
|
18
|
+
print(f"[!] {e}", file=sys.stderr)
|
|
19
|
+
return 1
|
|
20
|
+
|
|
21
|
+
tld_arg = args.tld.lower()
|
|
22
|
+
output_dir = Path(args.output_dir) if args.output_dir and not args.stdout else None
|
|
23
|
+
|
|
24
|
+
# If not writing to stdout and no explicit output_dir, default to local directory
|
|
25
|
+
if not args.stdout and not output_dir:
|
|
26
|
+
output_dir = Path(".")
|
|
27
|
+
|
|
28
|
+
all_domains: dict[str, list[str]] = {}
|
|
29
|
+
|
|
30
|
+
# 1. Handle .IN TLD
|
|
31
|
+
if tld_arg in ("in", "all", "*"):
|
|
32
|
+
if not args.stdout:
|
|
33
|
+
print(f"[*] Fetching .IN domains for {target_date.strftime('%Y-%m-%d')}...", file=sys.stderr)
|
|
34
|
+
try:
|
|
35
|
+
in_domains = fetch_in_domains(target_date, output_dir=output_dir)
|
|
36
|
+
all_domains["in"] = in_domains
|
|
37
|
+
if not args.stdout:
|
|
38
|
+
print(f"[✓] Extracted {len(in_domains)} .IN domains.", file=sys.stderr)
|
|
39
|
+
except Exception as e:
|
|
40
|
+
if not args.stdout:
|
|
41
|
+
print(f"[!] Failed to fetch .IN domains: {e}", file=sys.stderr)
|
|
42
|
+
|
|
43
|
+
# 2. Handle WhoisDS TLDs (.com, others, specific)
|
|
44
|
+
if tld_arg != "in":
|
|
45
|
+
whois_target = "all" if tld_arg == "all" else tld_arg
|
|
46
|
+
if not args.stdout:
|
|
47
|
+
print(f"[*] Fetching WhoisDS domains ({whois_target}) for {target_date.strftime('%Y-%m-%d')}...", file=sys.stderr)
|
|
48
|
+
try:
|
|
49
|
+
whois_results = fetch_whoisds_domains(target_date, tld=whois_target, output_dir=output_dir)
|
|
50
|
+
for k, v in whois_results.items():
|
|
51
|
+
all_domains[k] = v
|
|
52
|
+
if not args.stdout:
|
|
53
|
+
print(f"[✓] Extracted {len(v)} .{k} domains.", file=sys.stderr)
|
|
54
|
+
except Exception as e:
|
|
55
|
+
if not args.stdout:
|
|
56
|
+
print(f"[!] Failed to fetch WhoisDS domains: {e}", file=sys.stderr)
|
|
57
|
+
|
|
58
|
+
# Output formatting
|
|
59
|
+
if args.json:
|
|
60
|
+
payload = {
|
|
61
|
+
"date": target_date.strftime("%Y-%m-%d"),
|
|
62
|
+
"total_domains": sum(len(d) for d in all_domains.values()),
|
|
63
|
+
"tlds": {k: len(v) for k, v in all_domains.items()},
|
|
64
|
+
"domains": all_domains,
|
|
65
|
+
}
|
|
66
|
+
print(json.dumps(payload, indent=2))
|
|
67
|
+
elif args.stdout:
|
|
68
|
+
for tld_name, dom_list in all_domains.items():
|
|
69
|
+
for dom in dom_list:
|
|
70
|
+
print(dom)
|
|
71
|
+
else:
|
|
72
|
+
total = sum(len(d) for d in all_domains.values())
|
|
73
|
+
print(f"[*] Done. Total {total} domains saved into {output_dir.resolve()}/", file=sys.stderr)
|
|
74
|
+
|
|
75
|
+
return 0
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def cmd_search(args) -> int:
|
|
79
|
+
data_dir = Path(args.data_dir)
|
|
80
|
+
if not data_dir.exists():
|
|
81
|
+
print(f"[!] Data directory '{data_dir}' does not exist.", file=sys.stderr)
|
|
82
|
+
return 1
|
|
83
|
+
|
|
84
|
+
results = []
|
|
85
|
+
for domain, file_path in search_domains(
|
|
86
|
+
data_dir,
|
|
87
|
+
pattern=args.pattern,
|
|
88
|
+
regex=args.regex,
|
|
89
|
+
tld=args.tld,
|
|
90
|
+
date_str=args.date,
|
|
91
|
+
):
|
|
92
|
+
if args.json:
|
|
93
|
+
results.append({"domain": domain, "file": str(file_path)})
|
|
94
|
+
else:
|
|
95
|
+
if args.verbose:
|
|
96
|
+
print(f"{domain}\t({file_path})")
|
|
97
|
+
else:
|
|
98
|
+
print(domain)
|
|
99
|
+
|
|
100
|
+
if args.json:
|
|
101
|
+
print(json.dumps(results, indent=2))
|
|
102
|
+
|
|
103
|
+
return 0
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def cmd_stats(args) -> int:
|
|
107
|
+
data_dir = Path(args.data_dir)
|
|
108
|
+
if not data_dir.exists():
|
|
109
|
+
print(f"[!] Data directory '{data_dir}' does not exist.", file=sys.stderr)
|
|
110
|
+
return 1
|
|
111
|
+
|
|
112
|
+
files = find_matching_files(data_dir)
|
|
113
|
+
print(f"[*] NetHound Database Summary: {data_dir.resolve()}")
|
|
114
|
+
print(f" Total domain files: {len(files)}")
|
|
115
|
+
|
|
116
|
+
tld_counts: dict[str, int] = {}
|
|
117
|
+
for f in files:
|
|
118
|
+
rel = f.relative_to(data_dir)
|
|
119
|
+
tld_key = rel.parts[0]
|
|
120
|
+
if tld_key == "others" and len(rel.parts) > 1:
|
|
121
|
+
tld_key = f"others/{rel.parts[1]}"
|
|
122
|
+
try:
|
|
123
|
+
with open(f, "r", encoding="utf-8", errors="ignore") as fp:
|
|
124
|
+
count = sum(1 for line in fp if line.strip())
|
|
125
|
+
tld_counts[tld_key] = tld_counts.get(tld_key, 0) + count
|
|
126
|
+
except Exception:
|
|
127
|
+
pass
|
|
128
|
+
|
|
129
|
+
for tld_key, count in sorted(tld_counts.items(), key=lambda x: x[1], reverse=True)[:20]:
|
|
130
|
+
print(f" - {tld_key:<15}: {count:,} domains")
|
|
131
|
+
|
|
132
|
+
print(f" Total domains tracked: {sum(tld_counts.values()):,}")
|
|
133
|
+
return 0
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
137
|
+
parser = argparse.ArgumentParser(
|
|
138
|
+
prog="nethound",
|
|
139
|
+
description="NetHound - Newly Registered Domains (NRDs) Tracker & Threat Intelligence Extractor",
|
|
140
|
+
)
|
|
141
|
+
parser.add_argument("-v", "--version", action="version", version=f"%(prog)s {__version__}")
|
|
142
|
+
|
|
143
|
+
subparsers = parser.add_subparsers(dest="command", help="Available subcommands")
|
|
144
|
+
|
|
145
|
+
# Fetch subcommand
|
|
146
|
+
fetch_parser = subparsers.add_parser("fetch", help="Fetch daily newly registered domains")
|
|
147
|
+
fetch_parser.add_argument(
|
|
148
|
+
"--tld",
|
|
149
|
+
default="all",
|
|
150
|
+
help="Target TLD ('in', 'com', 'all', 'others', or specific e.g. 'cn', 'shop', 'net'). Default: all",
|
|
151
|
+
)
|
|
152
|
+
fetch_parser.add_argument(
|
|
153
|
+
"--date",
|
|
154
|
+
default=None,
|
|
155
|
+
help="Date in YYYY-MM-DD or DD-MM-YYYY format (default: yesterday)",
|
|
156
|
+
)
|
|
157
|
+
fetch_parser.add_argument(
|
|
158
|
+
"-o",
|
|
159
|
+
"--output-dir",
|
|
160
|
+
default=None,
|
|
161
|
+
help="Directory to save domains (default: current directory structure)",
|
|
162
|
+
)
|
|
163
|
+
fetch_parser.add_argument(
|
|
164
|
+
"-s",
|
|
165
|
+
"--stdout",
|
|
166
|
+
action="store_true",
|
|
167
|
+
help="Stream domains directly to stdout (useful for piping into httpx, subfinder, dnsx)",
|
|
168
|
+
)
|
|
169
|
+
fetch_parser.add_argument(
|
|
170
|
+
"--json",
|
|
171
|
+
action="store_true",
|
|
172
|
+
help="Output results in JSON format",
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
# Search subcommand
|
|
176
|
+
search_parser = subparsers.add_parser("search", help="Search downloaded NRDs for keywords or regex")
|
|
177
|
+
search_parser.add_argument("pattern", help="String or regular expression pattern to match")
|
|
178
|
+
search_parser.add_argument("-r", "--regex", action="store_true", help="Treat pattern as regular expression")
|
|
179
|
+
search_parser.add_argument("--tld", default=None, help="Filter by specific TLD")
|
|
180
|
+
search_parser.add_argument("--date", default=None, help="Filter by specific date (YYYY-MM-DD)")
|
|
181
|
+
search_parser.add_argument("-d", "--data-dir", default=".", help="Root directory containing domain files (default: .)")
|
|
182
|
+
search_parser.add_argument("-V", "--verbose", action="store_true", help="Print file source alongside matching domain")
|
|
183
|
+
search_parser.add_argument("--json", action="store_true", help="Output results in JSON format")
|
|
184
|
+
|
|
185
|
+
# Stats subcommand
|
|
186
|
+
stats_parser = subparsers.add_parser("stats", help="Display statistics of local NRD database")
|
|
187
|
+
stats_parser.add_argument("-d", "--data-dir", default=".", help="Root directory containing domain files (default: .)")
|
|
188
|
+
|
|
189
|
+
return parser
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def main() -> int:
|
|
193
|
+
parser = build_parser()
|
|
194
|
+
args = parser.parse_args()
|
|
195
|
+
|
|
196
|
+
if not args.command:
|
|
197
|
+
parser.print_help()
|
|
198
|
+
return 0
|
|
199
|
+
|
|
200
|
+
if args.command == "fetch":
|
|
201
|
+
return cmd_fetch(args)
|
|
202
|
+
elif args.command == "search":
|
|
203
|
+
return cmd_search(args)
|
|
204
|
+
elif args.command == "stats":
|
|
205
|
+
return cmd_stats(args)
|
|
206
|
+
|
|
207
|
+
return 0
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
if __name__ == "__main__":
|
|
211
|
+
sys.exit(main())
|
nethound/config.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
DEFAULT_DATA_DIR = Path(os.environ.get("NETHOUND_DATA_DIR", "nethound_data"))
|
|
6
|
+
|
|
7
|
+
DEFAULT_HEADERS = {
|
|
8
|
+
"User-Agent": (
|
|
9
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
10
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
11
|
+
"Chrome/124.0.0.0 Safari/537.36 NetHound/1.0.0"
|
|
12
|
+
),
|
|
13
|
+
"Accept": "*/*",
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
def parse_date(date_str: str | None) -> datetime:
|
|
17
|
+
"""Parse a date string in YYYY-MM-DD or DD-MM-YYYY format, or default to yesterday."""
|
|
18
|
+
if not date_str:
|
|
19
|
+
from datetime import timedelta
|
|
20
|
+
return datetime.now() - timedelta(days=1)
|
|
21
|
+
|
|
22
|
+
for fmt in ("%Y-%m-%d", "%d-%m-%Y"):
|
|
23
|
+
try:
|
|
24
|
+
return datetime.strptime(date_str, fmt)
|
|
25
|
+
except ValueError:
|
|
26
|
+
pass
|
|
27
|
+
raise ValueError(f"Invalid date format '{date_str}'. Expected YYYY-MM-DD or DD-MM-YYYY.")
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import io
|
|
2
|
+
import re
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
from nethound.config import DEFAULT_HEADERS
|
|
8
|
+
|
|
9
|
+
REGISTRY_IN_URL = "https://registry.in/system/files/domain-creates_{date}.pdf"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _extract_domains_pypdf(content: bytes) -> list[str]:
|
|
13
|
+
import pypdf
|
|
14
|
+
reader = pypdf.PdfReader(io.BytesIO(content))
|
|
15
|
+
domains = set()
|
|
16
|
+
for page in reader.pages:
|
|
17
|
+
text = page.extract_text()
|
|
18
|
+
if not text:
|
|
19
|
+
continue
|
|
20
|
+
for line in text.splitlines():
|
|
21
|
+
line = line.strip()
|
|
22
|
+
if not line or line.startswith("Domain User Form") or line.startswith("Page"):
|
|
23
|
+
continue
|
|
24
|
+
parts = line.split()
|
|
25
|
+
if parts:
|
|
26
|
+
dom = parts[0].strip().lower()
|
|
27
|
+
if "." in dom and not dom.startswith("page") and not dom.startswith("created"):
|
|
28
|
+
domains.add(dom)
|
|
29
|
+
return sorted(domains)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _extract_domains_pdfplumber(content: bytes) -> list[str]:
|
|
33
|
+
import pdfplumber
|
|
34
|
+
domains = set()
|
|
35
|
+
with pdfplumber.open(io.BytesIO(content)) as pdf:
|
|
36
|
+
for page in pdf.pages:
|
|
37
|
+
table = page.extract_table()
|
|
38
|
+
if table:
|
|
39
|
+
for row in table[1:]:
|
|
40
|
+
if row and row[0]:
|
|
41
|
+
domain = row[0].strip().lower()
|
|
42
|
+
if domain and domain != "domain user form" and "." in domain:
|
|
43
|
+
domains.add(domain)
|
|
44
|
+
return sorted(domains)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def extract_domains_from_pdf_bytes(content: bytes) -> list[str]:
|
|
48
|
+
"""Extract domains from registry.in PDF bytes using pypdf or pdfplumber."""
|
|
49
|
+
try:
|
|
50
|
+
return _extract_domains_pypdf(content)
|
|
51
|
+
except ImportError:
|
|
52
|
+
pass
|
|
53
|
+
|
|
54
|
+
try:
|
|
55
|
+
return _extract_domains_pdfplumber(content)
|
|
56
|
+
except ImportError:
|
|
57
|
+
raise RuntimeError("No PDF extractor available. Please install 'python3-pypdf' or 'pdfplumber'.")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def fetch_in_domains(target_date: datetime, output_dir: Path | None = None) -> list[str]:
|
|
61
|
+
"""Download and extract .IN newly registered domains for target_date."""
|
|
62
|
+
date_formatted = target_date.strftime("%Y-%m-%d")
|
|
63
|
+
url = REGISTRY_IN_URL.format(date=date_formatted)
|
|
64
|
+
|
|
65
|
+
response = requests.get(url, headers=DEFAULT_HEADERS, timeout=45)
|
|
66
|
+
response.raise_for_status()
|
|
67
|
+
|
|
68
|
+
domains = extract_domains_from_pdf_bytes(response.content)
|
|
69
|
+
|
|
70
|
+
if output_dir:
|
|
71
|
+
year = target_date.strftime("%Y")
|
|
72
|
+
month = target_date.strftime("%m")
|
|
73
|
+
dest_dir = output_dir / "in" / year / month
|
|
74
|
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
75
|
+
dest_file = dest_dir / f"domains_{date_formatted}.txt"
|
|
76
|
+
with open(dest_file, "w", encoding="utf-8") as f:
|
|
77
|
+
f.write("\n".join(domains) + ("\n" if domains else ""))
|
|
78
|
+
|
|
79
|
+
return domains
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
from collections import defaultdict
|
|
3
|
+
from datetime import datetime
|
|
4
|
+
import io
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import tempfile
|
|
7
|
+
import zipfile
|
|
8
|
+
import requests
|
|
9
|
+
|
|
10
|
+
from nethound.config import DEFAULT_HEADERS
|
|
11
|
+
|
|
12
|
+
WHOISDS_URL = "https://whoisds.com/whois-database/newly-registered-domains/{b64_date}/nrd"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def download_nrd_archive(target_date: datetime) -> bytes | None:
|
|
16
|
+
"""Download daily NRD zip archive from WhoisDS, with temporary caching."""
|
|
17
|
+
date_formatted = target_date.strftime("%Y-%m-%d")
|
|
18
|
+
cache_file = Path(tempfile.gettempdir()) / f"nethound_{date_formatted}.zip"
|
|
19
|
+
|
|
20
|
+
if cache_file.exists() and cache_file.stat().st_size > 1024:
|
|
21
|
+
try:
|
|
22
|
+
with open(cache_file, "rb") as f:
|
|
23
|
+
data = f.read()
|
|
24
|
+
with zipfile.ZipFile(io.BytesIO(data)) as z:
|
|
25
|
+
if z.namelist():
|
|
26
|
+
return data
|
|
27
|
+
except Exception:
|
|
28
|
+
cache_file.unlink(missing_ok=True)
|
|
29
|
+
|
|
30
|
+
b64_filename = base64.b64encode(f"{date_formatted}.zip".encode()).decode().rstrip("=")
|
|
31
|
+
url = WHOISDS_URL.format(b64_date=b64_filename)
|
|
32
|
+
|
|
33
|
+
response = requests.get(url, headers=DEFAULT_HEADERS, timeout=60)
|
|
34
|
+
if response.status_code != 200:
|
|
35
|
+
return None
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
with zipfile.ZipFile(io.BytesIO(response.content)) as z:
|
|
39
|
+
if not z.namelist():
|
|
40
|
+
return None
|
|
41
|
+
except zipfile.BadZipFile:
|
|
42
|
+
return None
|
|
43
|
+
|
|
44
|
+
with open(cache_file, "wb") as f:
|
|
45
|
+
f.write(response.content)
|
|
46
|
+
|
|
47
|
+
return response.content
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def fetch_whoisds_domains(
|
|
51
|
+
target_date: datetime,
|
|
52
|
+
tld: str = "com",
|
|
53
|
+
output_dir: Path | None = None,
|
|
54
|
+
) -> dict[str, list[str]]:
|
|
55
|
+
"""
|
|
56
|
+
Extract domains from WhoisDS NRD archive.
|
|
57
|
+
tld can be 'com', 'all', 'others', or a specific TLD like 'net', 'shop', 'cn'.
|
|
58
|
+
Returns a dict mapping TLD -> list of domains.
|
|
59
|
+
"""
|
|
60
|
+
archive_bytes = download_nrd_archive(target_date)
|
|
61
|
+
if not archive_bytes:
|
|
62
|
+
return {}
|
|
63
|
+
|
|
64
|
+
date_formatted = target_date.strftime("%Y-%m-%d")
|
|
65
|
+
year = target_date.strftime("%Y")
|
|
66
|
+
month = target_date.strftime("%m")
|
|
67
|
+
|
|
68
|
+
tld_lower = tld.lower()
|
|
69
|
+
target_is_all = tld_lower in ("all", "*")
|
|
70
|
+
target_is_others = tld_lower == "others"
|
|
71
|
+
target_is_com = tld_lower == "com"
|
|
72
|
+
|
|
73
|
+
results: dict[str, list[str]] = defaultdict(list)
|
|
74
|
+
|
|
75
|
+
with zipfile.ZipFile(io.BytesIO(archive_bytes)) as z:
|
|
76
|
+
for filename in z.namelist():
|
|
77
|
+
with z.open(filename) as f:
|
|
78
|
+
for line in f:
|
|
79
|
+
domain = line.decode("utf-8", errors="ignore").strip().lower()
|
|
80
|
+
if not domain or "." not in domain:
|
|
81
|
+
continue
|
|
82
|
+
|
|
83
|
+
if domain.endswith(".com"):
|
|
84
|
+
if target_is_all or target_is_com:
|
|
85
|
+
results["com"].append(domain)
|
|
86
|
+
else:
|
|
87
|
+
if target_is_com:
|
|
88
|
+
continue
|
|
89
|
+
try:
|
|
90
|
+
ascii_dom = domain.encode("idna").decode("ascii")
|
|
91
|
+
except Exception:
|
|
92
|
+
ascii_dom = domain
|
|
93
|
+
|
|
94
|
+
parts = ascii_dom.rsplit(".", 1)
|
|
95
|
+
if len(parts) == 2:
|
|
96
|
+
current_tld = parts[1].lower()
|
|
97
|
+
if current_tld.replace("-", "").isalnum():
|
|
98
|
+
if not target_is_all and not target_is_others and current_tld != tld_lower:
|
|
99
|
+
continue
|
|
100
|
+
results[current_tld].append(domain)
|
|
101
|
+
|
|
102
|
+
# Sort and deduplicate
|
|
103
|
+
for k in list(results.keys()):
|
|
104
|
+
results[k] = sorted(set(results[k]))
|
|
105
|
+
|
|
106
|
+
# Save to disk if output directory provided
|
|
107
|
+
if output_dir:
|
|
108
|
+
for current_tld, doms in results.items():
|
|
109
|
+
if current_tld == "com":
|
|
110
|
+
dest_dir = output_dir / "com" / year / month
|
|
111
|
+
else:
|
|
112
|
+
dest_dir = output_dir / "others" / current_tld / year / month
|
|
113
|
+
dest_dir.mkdir(parents=True, exist_ok=True)
|
|
114
|
+
dest_file = dest_dir / f"domains_{date_formatted}.txt"
|
|
115
|
+
with open(dest_file, "w", encoding="utf-8") as f:
|
|
116
|
+
f.write("\n".join(doms) + ("\n" if doms else ""))
|
|
117
|
+
|
|
118
|
+
return dict(results)
|
nethound/search.py
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import re
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Generator
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def find_matching_files(data_dir: Path, tld: str | None = None, date_str: str | None = None) -> list[Path]:
|
|
8
|
+
"""Find domain text files matching optional TLD and date filters."""
|
|
9
|
+
matched = []
|
|
10
|
+
if not data_dir.exists():
|
|
11
|
+
return matched
|
|
12
|
+
|
|
13
|
+
for root, _, files in os.walk(data_dir):
|
|
14
|
+
for f in files:
|
|
15
|
+
if not f.startswith("domains_") or not f.endswith(".txt"):
|
|
16
|
+
continue
|
|
17
|
+
path = Path(root) / f
|
|
18
|
+
|
|
19
|
+
# Date filter check
|
|
20
|
+
if date_str and f"domains_{date_str}.txt" != f:
|
|
21
|
+
continue
|
|
22
|
+
|
|
23
|
+
# TLD filter check
|
|
24
|
+
if tld:
|
|
25
|
+
rel = path.relative_to(data_dir)
|
|
26
|
+
parts = rel.parts
|
|
27
|
+
# Structure: in/year/month/file or com/year/month/file or others/tld/year/month/file
|
|
28
|
+
if parts[0] == "in" and tld.lower() != "in":
|
|
29
|
+
continue
|
|
30
|
+
elif parts[0] == "com" and tld.lower() != "com":
|
|
31
|
+
continue
|
|
32
|
+
elif parts[0] == "others" and len(parts) > 1 and parts[1].lower() != tld.lower():
|
|
33
|
+
continue
|
|
34
|
+
|
|
35
|
+
matched.append(path)
|
|
36
|
+
|
|
37
|
+
return sorted(matched)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def search_domains(
|
|
41
|
+
data_dir: Path,
|
|
42
|
+
pattern: str,
|
|
43
|
+
regex: bool = False,
|
|
44
|
+
tld: str | None = None,
|
|
45
|
+
date_str: str | None = None,
|
|
46
|
+
) -> Generator[tuple[str, Path], None, None]:
|
|
47
|
+
"""
|
|
48
|
+
Search domain records for a string or regex pattern.
|
|
49
|
+
Yields (domain, file_path).
|
|
50
|
+
"""
|
|
51
|
+
files = find_matching_files(data_dir, tld=tld, date_str=date_str)
|
|
52
|
+
|
|
53
|
+
if regex:
|
|
54
|
+
compiled = re.compile(pattern, re.IGNORECASE)
|
|
55
|
+
matcher = lambda d: bool(compiled.search(d))
|
|
56
|
+
else:
|
|
57
|
+
needle = pattern.lower()
|
|
58
|
+
matcher = lambda d: needle in d.lower()
|
|
59
|
+
|
|
60
|
+
for file_path in files:
|
|
61
|
+
try:
|
|
62
|
+
with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
|
|
63
|
+
for line in f:
|
|
64
|
+
dom = line.strip()
|
|
65
|
+
if dom and matcher(dom):
|
|
66
|
+
yield (dom, file_path)
|
|
67
|
+
except Exception:
|
|
68
|
+
continue
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: nethound
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Newly Registered Domains (NRDs) Tracker & Threat Intelligence Extractor
|
|
5
|
+
Author: xploitoverload
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/xploitoverload/NetHound
|
|
8
|
+
Project-URL: Repository, https://github.com/xploitoverload/NetHound.git
|
|
9
|
+
Project-URL: Issues, https://github.com/xploitoverload/NetHound/issues
|
|
10
|
+
Keywords: osint,threat-intelligence,nrd,security,domains,phishing-detection,pentesting
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Topic :: Security
|
|
14
|
+
Classifier: Intended Audience :: Information Technology
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
License-File: LICENSE
|
|
18
|
+
Requires-Dist: requests>=2.28.0
|
|
19
|
+
Requires-Dist: pypdf>=3.0.0
|
|
20
|
+
Provides-Extra: legacy-pdf
|
|
21
|
+
Requires-Dist: pdfplumber>=0.10.0; extra == "legacy-pdf"
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
<p align="center">
|
|
25
|
+
<img src="https://raw.githubusercontent.com/xploitoverload/NetHound/main/assets/nethound-icon.svg" alt="NetHound wolf logo" width="180" height="180">
|
|
26
|
+
</p>
|
|
27
|
+
|
|
28
|
+
# NetHound 🐕
|
|
29
|
+
### Newly Registered Domains (NRDs) Tracker & Threat Intelligence Engine
|
|
30
|
+
|
|
31
|
+
[](https://github.com/xploitoverload/NetHound/releases)
|
|
32
|
+
[](https://github.com/xploitoverload/NetHound/actions/workflows/daily-domain-fetch.yml)
|
|
33
|
+
[](https://opensource.org/licenses/MIT)
|
|
34
|
+
[](https://www.python.org/downloads/)
|
|
35
|
+
[](https://github.com/xploitoverload/NetHound)
|
|
36
|
+
|
|
37
|
+
**NetHound** is a fast, modular CLI tool and threat intelligence engine designed to monitor, extract, and search Newly Registered Domains (NRDs) across global registries (.IN, .COM, and hundreds of top-level domains).
|
|
38
|
+
|
|
39
|
+
Security teams, OSINT researchers, and penetration testers use newly registered domains to detect:
|
|
40
|
+
* **Phishing campaigns** and credential harvesting setups hours after registration.
|
|
41
|
+
* **Brand impersonation** and typosquatting attacks targeting organizations.
|
|
42
|
+
* **Malware C2 infrastructure** and fast-flux domains before active deployment.
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## ✨ Features
|
|
47
|
+
|
|
48
|
+
* **Multi-Registry Tracking:** Native support for `.IN` (Registry.in), `.COM`, and hundreds of generic TLDs (`.cn`, `.shop`, `.top`, `.net`, `.xyz`, etc.).
|
|
49
|
+
* **Pure Python & Fast Extraction:** Uses high-performance `pypdf` stream extraction with zero external system binaries.
|
|
50
|
+
* **Built-in Search Engine:** Search through historical and daily domain lists by exact string or regular expressions (`nethound search`).
|
|
51
|
+
* **Unix Pipe Friendly:** Stream domains straight to stdout (`--stdout` / `-s`) to pipe into `httpx`, `dnsx`, `subfinder`, or `nuclei`.
|
|
52
|
+
* **Debian & Kali Linux Ready:** Native `debian/` packaging included out-of-the-box for inclusion in security distributions.
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## 🚀 Installation
|
|
57
|
+
|
|
58
|
+
### Using pip (PyPI)
|
|
59
|
+
```bash
|
|
60
|
+
pip install nethound
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### From Source
|
|
64
|
+
```bash
|
|
65
|
+
git clone https://github.com/xploitoverload/NetHound.git
|
|
66
|
+
cd NetHound
|
|
67
|
+
pip install .
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Or install directly via Git:
|
|
71
|
+
```bash
|
|
72
|
+
pip install git+https://github.com/xploitoverload/NetHound.git
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Debian / Kali Linux Package Build
|
|
76
|
+
```bash
|
|
77
|
+
sudo apt install -y dh-python python3-all python3-requests python3-pypdf
|
|
78
|
+
cd NetHound
|
|
79
|
+
dpkg-buildpackage -us -uc -b
|
|
80
|
+
sudo dpkg -i ../nethound_1.0.0*.deb
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## 🛠️ Usage
|
|
86
|
+
|
|
87
|
+
### 1. Fetching Domains (`nethound fetch`)
|
|
88
|
+
|
|
89
|
+
Fetch all domains from yesterday across all supported registries:
|
|
90
|
+
```bash
|
|
91
|
+
nethound fetch --tld all
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Fetch only `.IN` newly registered domains for a specific date:
|
|
95
|
+
```bash
|
|
96
|
+
nethound fetch --tld in --date 2026-11-01
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Fetch only `.COM` domains:
|
|
100
|
+
```bash
|
|
101
|
+
nethound fetch --tld com --date 2026-11-01
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Fetch specific TLD (e.g. `.cn` or `.shop`):
|
|
105
|
+
```bash
|
|
106
|
+
nethound fetch --tld cn --date 2026-11-01
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
#### Pipeline Integration (Piping to Security Tools)
|
|
110
|
+
Stream domain names directly to standard output without status logs:
|
|
111
|
+
```bash
|
|
112
|
+
# Check live HTTP services on today's .in registrations
|
|
113
|
+
nethound fetch --tld in -s | httpx -silent -title -status-code
|
|
114
|
+
|
|
115
|
+
# Resolve DNS records for newly registered .com domains
|
|
116
|
+
nethound fetch --tld com -s | dnsx -silent -a -resp
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Output as structured JSON:
|
|
120
|
+
```bash
|
|
121
|
+
nethound fetch --tld in --json
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
---
|
|
125
|
+
|
|
126
|
+
### 2. Searching NRD Records (`nethound search`)
|
|
127
|
+
|
|
128
|
+
Search for keyword occurrences (e.g., brand monitoring for "paypal" or "bank"):
|
|
129
|
+
```bash
|
|
130
|
+
nethound search "paypal"
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
Search with Regular Expressions (`-r` / `--regex`):
|
|
134
|
+
```bash
|
|
135
|
+
nethound search -r "^login.*\.com$"
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Filter search by specific TLD:
|
|
139
|
+
```bash
|
|
140
|
+
nethound search "binance" --tld com
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Print source files and line locations:
|
|
144
|
+
```bash
|
|
145
|
+
nethound search "kali" -V
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
### 3. Database Statistics (`nethound stats`)
|
|
151
|
+
|
|
152
|
+
View a statistical summary of all currently tracked domains and file distributions:
|
|
153
|
+
```bash
|
|
154
|
+
nethound stats
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
---
|
|
158
|
+
|
|
159
|
+
## 📂 Repository Data Structure
|
|
160
|
+
|
|
161
|
+
For researchers interested in automated historical archives, domain lists are structured as follows:
|
|
162
|
+
|
|
163
|
+
```text
|
|
164
|
+
NetHound/
|
|
165
|
+
├── in/ # .IN Registry creations (yearly/monthly)
|
|
166
|
+
│ └── 2026/
|
|
167
|
+
│ └── 11/
|
|
168
|
+
│ └── domains_2026-11-01.txt
|
|
169
|
+
├── com/ # .COM Newly Registered Domains
|
|
170
|
+
│ └── 2026/
|
|
171
|
+
│ └── 11/
|
|
172
|
+
│ └── domains_2026-11-01.txt
|
|
173
|
+
├── others/ # Sub-partitioned per TLD
|
|
174
|
+
│ ├── cn/
|
|
175
|
+
│ ├── shop/
|
|
176
|
+
│ ├── top/
|
|
177
|
+
│ └── ...
|
|
178
|
+
├── nethound/ # Core Python package
|
|
179
|
+
│ ├── cli.py # Main CLI entrypoint
|
|
180
|
+
│ ├── config.py # Configurations & date parsers
|
|
181
|
+
│ ├── search.py # Search and regex engine
|
|
182
|
+
│ └── fetchers/ # Registry & WhoisDS extraction engines
|
|
183
|
+
├── debian/ # Official Debian/Kali packaging files
|
|
184
|
+
└── .github/workflows/ # Automated daily sync pipeline
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
---
|
|
188
|
+
|
|
189
|
+
## 🤖 Automated Daily Synchronization
|
|
190
|
+
|
|
191
|
+
NetHound runs an automated GitHub Action every morning at **09:00 AM IST (03:30 UTC)** to fetch, extract, and index all newly registered domain drops from global registries.
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## 🛡️ License
|
|
196
|
+
|
|
197
|
+
This project is licensed under the [MIT License](LICENSE).
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
## 👤 Author
|
|
202
|
+
|
|
203
|
+
Developed by **[xploitoverload](https://github.com/xploitoverload)**.
|
|
204
|
+
Contributions, issues, and feature requests are welcome!
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
<p align="center">
|
|
209
|
+
<img src="NetHound%20Cybersecurity%20Intelligence%20Banner.png" alt="NetHound Cybersecurity Intelligence Banner" width="100%">
|
|
210
|
+
</p>
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
nethound/__init__.py,sha256=1fAfR3TMVbo0x3J2F9JwNtekACmpE4ESQTzaKPDrY3I,165
|
|
2
|
+
nethound/cli.py,sha256=GjBDUSb-eYRL-R6LwNUJ5vbm5Dlj0SXmixrnd7F0RpU,7603
|
|
3
|
+
nethound/config.py,sha256=2q__6gsaLd1dmhZ3lFsyHDRxnDp_CAb-Pm6Q8a_2evQ,875
|
|
4
|
+
nethound/search.py,sha256=Q63Zhj5TwSxPWEt5tOm3c1hcLEaYFmBOk5zk9EXYll0,2168
|
|
5
|
+
nethound/fetchers/__init__.py,sha256=X_RO4pix7OzDmqaam5ZaARvxi7YmKlPf9uvuNQudll4,142
|
|
6
|
+
nethound/fetchers/registry_in.py,sha256=m8vBgMTh-KY5YjOK17Mq7oVAUM59_t1vwfPunQffmT4,2760
|
|
7
|
+
nethound/fetchers/whoisds.py,sha256=hc8dteGFaO7Wd0T32-70y_endV9afK1pEFCFJhrXCTs,4229
|
|
8
|
+
nethound-1.0.0.dist-info/licenses/LICENSE,sha256=4bx5FyT-vtUhr5IE87q5MVlspNCAFmmRZDHpP7RNQLU,1071
|
|
9
|
+
nethound-1.0.0.dist-info/METADATA,sha256=0r2FKwIqfLlu8ZKjntBDg5HgoAm-CesA019FZANn9Ic,6904
|
|
10
|
+
nethound-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
11
|
+
nethound-1.0.0.dist-info/entry_points.txt,sha256=hLOcDkbJR_6-r2NnW-b08BPGGO63WUYRxe9Y8Hwsnjc,47
|
|
12
|
+
nethound-1.0.0.dist-info/top_level.txt,sha256=irA2SGu-9dWDQC1Dj9IZQPnxeoLpIvljzc0dodDNugY,9
|
|
13
|
+
nethound-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 xploitoverload
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
nethound
|