osint-mcp-server 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- osint_mcp_server-0.1.0/.github/workflows/publish.yml +67 -0
- osint_mcp_server-0.1.0/.github/workflows/tests.yml +21 -0
- osint_mcp_server-0.1.0/.gitignore +9 -0
- osint_mcp_server-0.1.0/LICENSE +21 -0
- osint_mcp_server-0.1.0/PKG-INFO +115 -0
- osint_mcp_server-0.1.0/README.md +97 -0
- osint_mcp_server-0.1.0/pyproject.toml +40 -0
- osint_mcp_server-0.1.0/server.json +22 -0
- osint_mcp_server-0.1.0/src/osint_mcp/__init__.py +1 -0
- osint_mcp_server-0.1.0/src/osint_mcp/__main__.py +3 -0
- osint_mcp_server-0.1.0/src/osint_mcp/server.py +112 -0
- osint_mcp_server-0.1.0/src/osint_mcp/sources.py +282 -0
- osint_mcp_server-0.1.0/tests/test_sources.py +220 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
build:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v7
|
|
12
|
+
- uses: actions/setup-python@v7
|
|
13
|
+
with:
|
|
14
|
+
python-version: "3.12"
|
|
15
|
+
|
|
16
|
+
- name: check tag matches version
|
|
17
|
+
run: |
|
|
18
|
+
tag="${GITHUB_REF_NAME#v}"
|
|
19
|
+
py=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")
|
|
20
|
+
js=$(python -c "import json; print(json.load(open('server.json'))['version'])")
|
|
21
|
+
echo "tag=$tag pyproject=$py server.json=$js"
|
|
22
|
+
[ "$tag" = "$py" ] && [ "$tag" = "$js" ]
|
|
23
|
+
|
|
24
|
+
- run: pip install -e . pytest build
|
|
25
|
+
- run: pytest
|
|
26
|
+
- run: python -m build
|
|
27
|
+
- uses: actions/upload-artifact@v7
|
|
28
|
+
with:
|
|
29
|
+
name: dist
|
|
30
|
+
path: dist/
|
|
31
|
+
|
|
32
|
+
pypi:
|
|
33
|
+
needs: build
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
environment: pypi
|
|
36
|
+
permissions:
|
|
37
|
+
id-token: write
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/download-artifact@v8
|
|
40
|
+
with:
|
|
41
|
+
name: dist
|
|
42
|
+
path: dist/
|
|
43
|
+
- uses: pypa/gh-action-pypi-publish@release/v1
|
|
44
|
+
|
|
45
|
+
mcp-registry:
|
|
46
|
+
needs: pypi
|
|
47
|
+
runs-on: ubuntu-latest
|
|
48
|
+
permissions:
|
|
49
|
+
id-token: write
|
|
50
|
+
contents: read
|
|
51
|
+
steps:
|
|
52
|
+
- uses: actions/checkout@v7
|
|
53
|
+
|
|
54
|
+
- name: install mcp-publisher
|
|
55
|
+
run: |
|
|
56
|
+
curl -L "https://github.com/modelcontextprotocol/registry/releases/latest/download/mcp-publisher_$(uname -s | tr '[:upper:]' '[:lower:]')_$(uname -m | sed 's/x86_64/amd64/;s/aarch64/arm64/').tar.gz" | tar xz mcp-publisher
|
|
57
|
+
|
|
58
|
+
- run: ./mcp-publisher login github-oidc
|
|
59
|
+
|
|
60
|
+
# pypi can take a moment before the new version shows up in its api
|
|
61
|
+
- name: publish
|
|
62
|
+
run: |
|
|
63
|
+
for i in 1 2 3 4 5; do
|
|
64
|
+
./mcp-publisher publish && exit 0
|
|
65
|
+
sleep 20
|
|
66
|
+
done
|
|
67
|
+
exit 1
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
name: tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
strategy:
|
|
12
|
+
fail-fast: false
|
|
13
|
+
matrix:
|
|
14
|
+
python: ["3.10", "3.12", "3.14"]
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v7
|
|
17
|
+
- uses: actions/setup-python@v7
|
|
18
|
+
with:
|
|
19
|
+
python-version: ${{ matrix.python }}
|
|
20
|
+
- run: pip install -e . pytest
|
|
21
|
+
- run: pytest -v
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 robyroro
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: osint-mcp-server
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: MCP server with passive OSINT lookups (RDAP, DNS, crt.sh, Wayback, Shodan InternetDB)
|
|
5
|
+
Project-URL: Repository, https://github.com/robyroro/osint-mcp
|
|
6
|
+
Author: robyroro
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Keywords: claude,dns,mcp,osint,recon,whois
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Topic :: Internet :: Name Service (DNS)
|
|
12
|
+
Classifier: Topic :: Security
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Requires-Dist: dnspython>=2.6
|
|
15
|
+
Requires-Dist: httpx>=0.27
|
|
16
|
+
Requires-Dist: mcp>=2.0
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# osint-mcp
|
|
20
|
+
|
|
21
|
+
[](https://github.com/robyroro/osint-mcp/actions/workflows/tests.yml)
|
|
22
|
+
[](https://pypi.org/project/osint-mcp-server/)
|
|
23
|
+
|
|
24
|
+
MCP server that gives Claude (or any MCP client) a handful of passive recon lookups for domains and IPs. No API keys needed, everything comes from public sources.
|
|
25
|
+
|
|
26
|
+
I got tired of jumping between whois, crt.sh, the Wayback Machine and Shodan tabs when looking into a domain, so this lets the model do it and put the results together.
|
|
27
|
+
|
|
28
|
+
## Tools
|
|
29
|
+
|
|
30
|
+
| tool | what it does | source |
|
|
31
|
+
|---|---|---|
|
|
32
|
+
| `domain_whois` | registrar, created/expires, nameservers, abuse contact | RDAP (rdap.org) |
|
|
33
|
+
| `ip_whois` | network owner, CIDR, country, abuse contact, PTR | RDAP + reverse DNS |
|
|
34
|
+
| `dns_lookup` | A, AAAA, CNAME, MX, NS, TXT, SOA, CAA (or pick your own) | your resolver or a custom one |
|
|
35
|
+
| `subdomains` | subdomains found in certificate transparency logs | crt.sh |
|
|
36
|
+
| `wayback` | archived snapshots of a URL, supports `example.com/*` | Wayback CDX API |
|
|
37
|
+
| `http_headers` | status, redirect chain, headers, missing security headers | direct request |
|
|
38
|
+
| `shodan_internetdb` | open ports, hostnames, CPEs, known CVEs | Shodan InternetDB (free) |
|
|
39
|
+
|
|
40
|
+
Everything except `http_headers` is passive, the target never sees your traffic. `http_headers` is just a normal GET request, same as opening the page in a browser.
|
|
41
|
+
|
|
42
|
+
## Install
|
|
43
|
+
|
|
44
|
+
Needs Python 3.10+.
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
pip install osint-mcp-server
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
or if you use uv you don't need to install anything, just point the client at `uvx` (see below).
|
|
51
|
+
|
|
52
|
+
### Claude Desktop
|
|
53
|
+
|
|
54
|
+
Add this to `claude_desktop_config.json`:
|
|
55
|
+
|
|
56
|
+
```json
|
|
57
|
+
{
|
|
58
|
+
"mcpServers": {
|
|
59
|
+
"osint": {
|
|
60
|
+
"command": "uvx",
|
|
61
|
+
"args": ["osint-mcp-server"]
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
If you installed with pip, `"command": "osint-mcp-server"` with no args works too.
|
|
68
|
+
|
|
69
|
+
### Claude Code
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
claude mcp add osint -- uvx osint-mcp-server
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Cursor, Windsurf etc. take the same JSON as Claude Desktop.
|
|
76
|
+
|
|
77
|
+
## Example prompts
|
|
78
|
+
|
|
79
|
+
- "what can you find about example.com? registrar, dns, subdomains, anything interesting"
|
|
80
|
+
- "check the security headers on these 5 sites and tell me which are worst"
|
|
81
|
+
- "which of the subdomains of example.com resolve to something with open ports?"
|
|
82
|
+
- "what did example.com/about look like in 2015?"
|
|
83
|
+
|
|
84
|
+
## Notes
|
|
85
|
+
|
|
86
|
+
- crt.sh and the Wayback CDX API are slow and return 502/503 pretty often. The server retries a couple of times but sometimes you just have to try again later.
|
|
87
|
+
- InternetDB isn't real-time and only has IPs Shodan has actually scanned.
|
|
88
|
+
- `.ro`, `.de` and some other ccTLDs don't have public RDAP, so `domain_whois` can't do much for them.
|
|
89
|
+
|
|
90
|
+
## Development
|
|
91
|
+
|
|
92
|
+
```
|
|
93
|
+
git clone https://github.com/robyroro/osint-mcp
|
|
94
|
+
cd osint-mcp
|
|
95
|
+
pip install -e . pytest
|
|
96
|
+
pytest
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Tests don't hit the network, HTTP calls are mocked.
|
|
100
|
+
|
|
101
|
+
To poke at it with the MCP inspector:
|
|
102
|
+
|
|
103
|
+
```
|
|
104
|
+
npx @modelcontextprotocol/inspector osint-mcp
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
## Be reasonable
|
|
108
|
+
|
|
109
|
+
This only pulls public data, but still: use it on your own stuff, bug bounty targets that are in scope, or for research. Don't use it to go after people.
|
|
110
|
+
|
|
111
|
+
## License
|
|
112
|
+
|
|
113
|
+
MIT
|
|
114
|
+
|
|
115
|
+
<!-- mcp-name: io.github.robyroro/osint-mcp -->
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# osint-mcp
|
|
2
|
+
|
|
3
|
+
[](https://github.com/robyroro/osint-mcp/actions/workflows/tests.yml)
|
|
4
|
+
[](https://pypi.org/project/osint-mcp-server/)
|
|
5
|
+
|
|
6
|
+
MCP server that gives Claude (or any MCP client) a handful of passive recon lookups for domains and IPs. No API keys needed, everything comes from public sources.
|
|
7
|
+
|
|
8
|
+
I got tired of jumping between whois, crt.sh, the Wayback Machine and Shodan tabs when looking into a domain, so this lets the model do it and put the results together.
|
|
9
|
+
|
|
10
|
+
## Tools
|
|
11
|
+
|
|
12
|
+
| tool | what it does | source |
|
|
13
|
+
|---|---|---|
|
|
14
|
+
| `domain_whois` | registrar, created/expires, nameservers, abuse contact | RDAP (rdap.org) |
|
|
15
|
+
| `ip_whois` | network owner, CIDR, country, abuse contact, PTR | RDAP + reverse DNS |
|
|
16
|
+
| `dns_lookup` | A, AAAA, CNAME, MX, NS, TXT, SOA, CAA (or pick your own) | your resolver or a custom one |
|
|
17
|
+
| `subdomains` | subdomains found in certificate transparency logs | crt.sh |
|
|
18
|
+
| `wayback` | archived snapshots of a URL, supports `example.com/*` | Wayback CDX API |
|
|
19
|
+
| `http_headers` | status, redirect chain, headers, missing security headers | direct request |
|
|
20
|
+
| `shodan_internetdb` | open ports, hostnames, CPEs, known CVEs | Shodan InternetDB (free) |
|
|
21
|
+
|
|
22
|
+
Everything except `http_headers` is passive, the target never sees your traffic. `http_headers` is just a normal GET request, same as opening the page in a browser.
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
Needs Python 3.10+.
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
pip install osint-mcp-server
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
or if you use uv you don't need to install anything, just point the client at `uvx` (see below).
|
|
33
|
+
|
|
34
|
+
### Claude Desktop
|
|
35
|
+
|
|
36
|
+
Add this to `claude_desktop_config.json`:
|
|
37
|
+
|
|
38
|
+
```json
|
|
39
|
+
{
|
|
40
|
+
"mcpServers": {
|
|
41
|
+
"osint": {
|
|
42
|
+
"command": "uvx",
|
|
43
|
+
"args": ["osint-mcp-server"]
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
If you installed with pip, `"command": "osint-mcp-server"` with no args works too.
|
|
50
|
+
|
|
51
|
+
### Claude Code
|
|
52
|
+
|
|
53
|
+
```
|
|
54
|
+
claude mcp add osint -- uvx osint-mcp-server
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Cursor, Windsurf etc. take the same JSON as Claude Desktop.
|
|
58
|
+
|
|
59
|
+
## Example prompts
|
|
60
|
+
|
|
61
|
+
- "what can you find about example.com? registrar, dns, subdomains, anything interesting"
|
|
62
|
+
- "check the security headers on these 5 sites and tell me which are worst"
|
|
63
|
+
- "which of the subdomains of example.com resolve to something with open ports?"
|
|
64
|
+
- "what did example.com/about look like in 2015?"
|
|
65
|
+
|
|
66
|
+
## Notes
|
|
67
|
+
|
|
68
|
+
- crt.sh and the Wayback CDX API are slow and return 502/503 pretty often. The server retries a couple of times but sometimes you just have to try again later.
|
|
69
|
+
- InternetDB isn't real-time and only has IPs Shodan has actually scanned.
|
|
70
|
+
- `.ro`, `.de` and some other ccTLDs don't have public RDAP, so `domain_whois` can't do much for them.
|
|
71
|
+
|
|
72
|
+
## Development
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
git clone https://github.com/robyroro/osint-mcp
|
|
76
|
+
cd osint-mcp
|
|
77
|
+
pip install -e . pytest
|
|
78
|
+
pytest
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Tests don't hit the network, HTTP calls are mocked.
|
|
82
|
+
|
|
83
|
+
To poke at it with the MCP inspector:
|
|
84
|
+
|
|
85
|
+
```
|
|
86
|
+
npx @modelcontextprotocol/inspector osint-mcp
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Be reasonable
|
|
90
|
+
|
|
91
|
+
This only pulls public data, but still: use it on your own stuff, bug bounty targets that are in scope, or for research. Don't use it to go after people.
|
|
92
|
+
|
|
93
|
+
## License
|
|
94
|
+
|
|
95
|
+
MIT
|
|
96
|
+
|
|
97
|
+
<!-- mcp-name: io.github.robyroro/osint-mcp -->
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "osint-mcp-server"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "MCP server with passive OSINT lookups (RDAP, DNS, crt.sh, Wayback, Shodan InternetDB)"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
authors = [{ name = "robyroro" }]
|
|
8
|
+
requires-python = ">=3.10"
|
|
9
|
+
keywords = ["mcp", "osint", "recon", "whois", "dns", "claude"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"Topic :: Security",
|
|
13
|
+
"Topic :: Internet :: Name Service (DNS)",
|
|
14
|
+
]
|
|
15
|
+
dependencies = [
|
|
16
|
+
"mcp>=2.0",
|
|
17
|
+
"httpx>=0.27",
|
|
18
|
+
"dnspython>=2.6",
|
|
19
|
+
]
|
|
20
|
+
|
|
21
|
+
[project.urls]
|
|
22
|
+
Repository = "https://github.com/robyroro/osint-mcp"
|
|
23
|
+
|
|
24
|
+
[project.scripts]
|
|
25
|
+
osint-mcp = "osint_mcp.server:main"
|
|
26
|
+
# so `uvx osint-mcp-server` works without --from
|
|
27
|
+
osint-mcp-server = "osint_mcp.server:main"
|
|
28
|
+
|
|
29
|
+
[tool.hatch.build.targets.wheel]
|
|
30
|
+
packages = ["src/osint_mcp"]
|
|
31
|
+
|
|
32
|
+
[dependency-groups]
|
|
33
|
+
dev = ["pytest>=8"]
|
|
34
|
+
|
|
35
|
+
[build-system]
|
|
36
|
+
requires = ["hatchling"]
|
|
37
|
+
build-backend = "hatchling.build"
|
|
38
|
+
|
|
39
|
+
[tool.pytest.ini_options]
|
|
40
|
+
testpaths = ["tests"]
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json",
|
|
3
|
+
"name": "io.github.robyroro/osint-mcp",
|
|
4
|
+
"title": "OSINT",
|
|
5
|
+
"description": "Passive OSINT recon for domains and IPs: whois/RDAP, DNS, crt.sh, Wayback, Shodan InternetDB",
|
|
6
|
+
"repository": {
|
|
7
|
+
"url": "https://github.com/robyroro/osint-mcp",
|
|
8
|
+
"source": "github"
|
|
9
|
+
},
|
|
10
|
+
"version": "0.1.0",
|
|
11
|
+
"packages": [
|
|
12
|
+
{
|
|
13
|
+
"registryType": "pypi",
|
|
14
|
+
"identifier": "osint-mcp-server",
|
|
15
|
+
"version": "0.1.0",
|
|
16
|
+
"runtimeHint": "uvx",
|
|
17
|
+
"transport": {
|
|
18
|
+
"type": "stdio"
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
import httpx
|
|
4
|
+
from mcp.server.mcpserver import MCPServer
|
|
5
|
+
from mcp.server.mcpserver.exceptions import ToolError
|
|
6
|
+
|
|
7
|
+
from . import sources
|
|
8
|
+
|
|
9
|
+
# httpx logs every request at INFO, way too chatty
|
|
10
|
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
|
11
|
+
|
|
12
|
+
mcp = MCPServer(
|
|
13
|
+
"osint",
|
|
14
|
+
instructions=(
|
|
15
|
+
"Passive recon tools for domains and IPs: RDAP/whois, DNS, certificate "
|
|
16
|
+
"transparency, Wayback Machine, HTTP headers and Shodan InternetDB. "
|
|
17
|
+
"Nothing here scans or brute forces anything."
|
|
18
|
+
),
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _domain(value):
|
|
23
|
+
try:
|
|
24
|
+
return sources.clean_domain(value)
|
|
25
|
+
except ValueError as e:
|
|
26
|
+
raise ToolError(str(e))
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _ip(value):
|
|
30
|
+
try:
|
|
31
|
+
return sources.clean_ip(value)
|
|
32
|
+
except ValueError:
|
|
33
|
+
raise ToolError(f"not a valid ip address: {value!r}")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
async def _run(coro_fn, *args, **kwargs):
|
|
37
|
+
async with sources.make_client() as client:
|
|
38
|
+
try:
|
|
39
|
+
return await coro_fn(client, *args, **kwargs)
|
|
40
|
+
except httpx.HTTPStatusError as e:
|
|
41
|
+
raise ToolError(f"{e.request.url.host} returned {e.response.status_code}")
|
|
42
|
+
except httpx.TimeoutException as e:
|
|
43
|
+
raise ToolError(f"timed out talking to {e.request.url.host}, try again")
|
|
44
|
+
except httpx.RequestError as e:
|
|
45
|
+
raise ToolError(f"request failed: {e}")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@mcp.tool()
|
|
49
|
+
async def domain_whois(domain: str) -> dict:
|
|
50
|
+
"""Registration info for a domain via RDAP: registrar, creation/expiry dates,
|
|
51
|
+
nameservers, status and abuse contact."""
|
|
52
|
+
return await _run(sources.rdap_domain, _domain(domain))
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@mcp.tool()
|
|
56
|
+
async def ip_whois(ip: str) -> dict:
|
|
57
|
+
"""Who owns an IP: network name, CIDR, country, org, abuse contact and
|
|
58
|
+
reverse DNS (PTR)."""
|
|
59
|
+
ip = _ip(ip)
|
|
60
|
+
info = await _run(sources.rdap_ip, ip)
|
|
61
|
+
info["ptr"] = await sources.reverse_dns(ip)
|
|
62
|
+
return info
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@mcp.tool()
|
|
66
|
+
async def dns_lookup(domain: str, record_types: list[str] | None = None, nameserver: str | None = None) -> dict:
|
|
67
|
+
"""DNS records for a domain. Defaults to A, AAAA, CNAME, MX, NS, TXT, SOA, CAA.
|
|
68
|
+
Pass nameserver (e.g. "1.1.1.1") to query a specific resolver."""
|
|
69
|
+
return await sources.dns_records(_domain(domain), record_types, nameserver)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@mcp.tool()
|
|
73
|
+
async def subdomains(domain: str) -> dict:
|
|
74
|
+
"""Subdomains seen in public TLS certificates (crt.sh). Can be slow for
|
|
75
|
+
large domains."""
|
|
76
|
+
return await _run(sources.crtsh_subdomains, _domain(domain))
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@mcp.tool()
|
|
80
|
+
async def wayback(
|
|
81
|
+
url: str,
|
|
82
|
+
limit: int = 20,
|
|
83
|
+
newest_first: bool = True,
|
|
84
|
+
year_from: int | None = None,
|
|
85
|
+
year_to: int | None = None,
|
|
86
|
+
) -> dict:
|
|
87
|
+
"""Archived snapshots of a URL from the Wayback Machine. Use a wildcard like
|
|
88
|
+
"example.com/*" to list everything archived under a site."""
|
|
89
|
+
limit = max(1, min(limit, 500))
|
|
90
|
+
return await _run(sources.wayback_snapshots, url, limit, newest_first, year_from, year_to)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@mcp.tool()
|
|
94
|
+
async def http_headers(url: str) -> dict:
|
|
95
|
+
"""Fetch a URL and return status, redirect chain, response headers and which
|
|
96
|
+
common security headers are missing."""
|
|
97
|
+
return await _run(sources.http_headers, url)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@mcp.tool()
|
|
101
|
+
async def shodan_internetdb(ip: str) -> dict:
|
|
102
|
+
"""Open ports, hostnames, CPEs, tags and known CVEs for an IP from Shodan's
|
|
103
|
+
free InternetDB (no API key, data can be a few days old)."""
|
|
104
|
+
return await _run(sources.internetdb, _ip(ip))
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def main():
|
|
108
|
+
mcp.run()
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
if __name__ == "__main__":
|
|
112
|
+
main()
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import ipaddress
|
|
3
|
+
from urllib.parse import urlparse
|
|
4
|
+
|
|
5
|
+
import dns.asyncresolver
|
|
6
|
+
import dns.exception
|
|
7
|
+
import dns.resolver
|
|
8
|
+
import dns.reversename
|
|
9
|
+
import httpx
|
|
10
|
+
|
|
11
|
+
USER_AGENT = "osint-mcp/0.1 (+https://github.com/robyroro/osint-mcp)"
|
|
12
|
+
|
|
13
|
+
SECURITY_HEADERS = [
|
|
14
|
+
"strict-transport-security",
|
|
15
|
+
"content-security-policy",
|
|
16
|
+
"x-content-type-options",
|
|
17
|
+
"x-frame-options",
|
|
18
|
+
"referrer-policy",
|
|
19
|
+
"permissions-policy",
|
|
20
|
+
]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def make_client(timeout=15.0):
|
|
24
|
+
return httpx.AsyncClient(
|
|
25
|
+
headers={"User-Agent": USER_AGENT},
|
|
26
|
+
timeout=timeout,
|
|
27
|
+
follow_redirects=True,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
async def get_flaky(client, url, retries=2, **kwargs):
|
|
32
|
+
# crt.sh and archive.org both hand out 502/503s under load,
|
|
33
|
+
# a retry or two usually gets through
|
|
34
|
+
for attempt in range(retries + 1):
|
|
35
|
+
r = await client.get(url, **kwargs)
|
|
36
|
+
if r.status_code not in (502, 503, 504) or attempt == retries:
|
|
37
|
+
return r
|
|
38
|
+
await asyncio.sleep(2 * (attempt + 1))
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def clean_domain(value: str) -> str:
|
|
42
|
+
"""Accepts 'example.com', 'https://Example.com/x', 'www.example.com.' etc."""
|
|
43
|
+
value = value.strip()
|
|
44
|
+
if "://" in value:
|
|
45
|
+
value = urlparse(value).hostname or ""
|
|
46
|
+
value = value.split("/")[0].split(":")[0]
|
|
47
|
+
value = value.strip(".").lower()
|
|
48
|
+
if not value or "." not in value:
|
|
49
|
+
raise ValueError(f"doesn't look like a domain: {value!r}")
|
|
50
|
+
return value
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def clean_ip(value: str) -> str:
|
|
54
|
+
return str(ipaddress.ip_address(value.strip()))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# --- RDAP (the json replacement for whois) ---
|
|
58
|
+
|
|
59
|
+
def _vcard_name(entity):
|
|
60
|
+
vcard = entity.get("vcardArray")
|
|
61
|
+
if not vcard or len(vcard) < 2:
|
|
62
|
+
return None
|
|
63
|
+
for field in vcard[1]:
|
|
64
|
+
if field[0] == "fn":
|
|
65
|
+
return field[3] or None
|
|
66
|
+
return None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _vcard_email(entity):
|
|
70
|
+
vcard = entity.get("vcardArray")
|
|
71
|
+
if not vcard or len(vcard) < 2:
|
|
72
|
+
return None
|
|
73
|
+
for field in vcard[1]:
|
|
74
|
+
if field[0] == "email":
|
|
75
|
+
return field[3]
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _walk_entities(entities):
|
|
80
|
+
# abuse contacts are usually nested under the registrar/registrant
|
|
81
|
+
for e in entities or []:
|
|
82
|
+
yield e
|
|
83
|
+
yield from _walk_entities(e.get("entities"))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def parse_rdap_domain(data: dict) -> dict:
|
|
87
|
+
events = {e["eventAction"]: e.get("eventDate") for e in data.get("events", [])}
|
|
88
|
+
out = {
|
|
89
|
+
"domain": (data.get("ldhName") or "").lower(),
|
|
90
|
+
"status": data.get("status", []),
|
|
91
|
+
"registered": events.get("registration"),
|
|
92
|
+
"expires": events.get("expiration"),
|
|
93
|
+
"last_changed": events.get("last changed"),
|
|
94
|
+
"nameservers": sorted(
|
|
95
|
+
ns["ldhName"].lower() for ns in data.get("nameservers", []) if ns.get("ldhName")
|
|
96
|
+
),
|
|
97
|
+
"registrar": None,
|
|
98
|
+
"abuse_email": None,
|
|
99
|
+
}
|
|
100
|
+
for e in _walk_entities(data.get("entities")):
|
|
101
|
+
roles = e.get("roles", [])
|
|
102
|
+
if "registrar" in roles and not out["registrar"]:
|
|
103
|
+
out["registrar"] = _vcard_name(e)
|
|
104
|
+
if "abuse" in roles and not out["abuse_email"]:
|
|
105
|
+
out["abuse_email"] = _vcard_email(e)
|
|
106
|
+
return out
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def parse_rdap_ip(data: dict) -> dict:
|
|
110
|
+
cidrs = []
|
|
111
|
+
for c in data.get("cidr0_cidrs", []):
|
|
112
|
+
prefix = c.get("v4prefix") or c.get("v6prefix")
|
|
113
|
+
if prefix:
|
|
114
|
+
cidrs.append(f"{prefix}/{c.get('length')}")
|
|
115
|
+
|
|
116
|
+
out = {
|
|
117
|
+
"network": data.get("name"),
|
|
118
|
+
"handle": data.get("handle"),
|
|
119
|
+
"range": f"{data.get('startAddress')} - {data.get('endAddress')}",
|
|
120
|
+
"cidrs": cidrs,
|
|
121
|
+
"country": data.get("country"),
|
|
122
|
+
"org": None,
|
|
123
|
+
"abuse_email": None,
|
|
124
|
+
}
|
|
125
|
+
for e in _walk_entities(data.get("entities")):
|
|
126
|
+
roles = e.get("roles", [])
|
|
127
|
+
if "registrant" in roles and not out["org"]:
|
|
128
|
+
out["org"] = _vcard_name(e)
|
|
129
|
+
if "abuse" in roles and not out["abuse_email"]:
|
|
130
|
+
out["abuse_email"] = _vcard_email(e)
|
|
131
|
+
return out
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
async def rdap_domain(client, domain):
|
|
135
|
+
r = await client.get(f"https://rdap.org/domain/{domain}")
|
|
136
|
+
if r.status_code == 404:
|
|
137
|
+
return {
|
|
138
|
+
"domain": domain,
|
|
139
|
+
"error": "no RDAP data. either not registered or the TLD has no public "
|
|
140
|
+
"RDAP server (.ro, .de and a few other ccTLDs), use their web whois",
|
|
141
|
+
}
|
|
142
|
+
r.raise_for_status()
|
|
143
|
+
return parse_rdap_domain(r.json())
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
async def rdap_ip(client, ip):
|
|
147
|
+
r = await client.get(f"https://rdap.org/ip/{ip}")
|
|
148
|
+
r.raise_for_status()
|
|
149
|
+
return parse_rdap_ip(r.json())
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# --- DNS ---
|
|
153
|
+
|
|
154
|
+
DEFAULT_RECORD_TYPES = ["A", "AAAA", "CNAME", "MX", "NS", "TXT", "SOA", "CAA"]
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def _rdata_text(rdtype, rdata):
|
|
158
|
+
if rdtype == "TXT":
|
|
159
|
+
return b"".join(rdata.strings).decode(errors="replace")
|
|
160
|
+
return rdata.to_text()
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
async def dns_records(domain, types=None, nameserver=None):
|
|
164
|
+
resolver = dns.asyncresolver.Resolver()
|
|
165
|
+
resolver.lifetime = 5.0
|
|
166
|
+
if nameserver:
|
|
167
|
+
resolver.nameservers = [nameserver]
|
|
168
|
+
|
|
169
|
+
records = {}
|
|
170
|
+
for rdtype in types or DEFAULT_RECORD_TYPES:
|
|
171
|
+
rdtype = rdtype.upper()
|
|
172
|
+
try:
|
|
173
|
+
answer = await resolver.resolve(domain, rdtype)
|
|
174
|
+
except dns.resolver.NXDOMAIN:
|
|
175
|
+
return {"domain": domain, "error": "NXDOMAIN (domain doesn't exist)"}
|
|
176
|
+
except (dns.resolver.NoAnswer, dns.resolver.NoNameservers):
|
|
177
|
+
continue
|
|
178
|
+
except dns.exception.Timeout:
|
|
179
|
+
records[rdtype] = ["<timeout>"]
|
|
180
|
+
continue
|
|
181
|
+
records[rdtype] = sorted(_rdata_text(rdtype, r) for r in answer)
|
|
182
|
+
return {"domain": domain, "records": records}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
async def reverse_dns(ip):
|
|
186
|
+
resolver = dns.asyncresolver.Resolver()
|
|
187
|
+
resolver.lifetime = 5.0
|
|
188
|
+
try:
|
|
189
|
+
answer = await resolver.resolve(dns.reversename.from_address(ip), "PTR")
|
|
190
|
+
except dns.exception.DNSException:
|
|
191
|
+
return []
|
|
192
|
+
return sorted(r.to_text().rstrip(".") for r in answer)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# --- certificate transparency (crt.sh) ---
|
|
196
|
+
|
|
197
|
+
def parse_crtsh(rows: list, domain: str) -> list[str]:
|
|
198
|
+
names = set()
|
|
199
|
+
for row in rows:
|
|
200
|
+
for name in row.get("name_value", "").splitlines():
|
|
201
|
+
name = name.strip().lower().removeprefix("*.")
|
|
202
|
+
if name == domain or name.endswith("." + domain):
|
|
203
|
+
names.add(name)
|
|
204
|
+
return sorted(names)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
async def crtsh_subdomains(client, domain):
|
|
208
|
+
# crt.sh can take a while on big domains
|
|
209
|
+
r = await get_flaky(
|
|
210
|
+
client,
|
|
211
|
+
"https://crt.sh/",
|
|
212
|
+
params={"q": f"%.{domain}", "output": "json"},
|
|
213
|
+
timeout=60.0,
|
|
214
|
+
)
|
|
215
|
+
r.raise_for_status()
|
|
216
|
+
rows = r.json() if r.content else []
|
|
217
|
+
return {"domain": domain, "certs_seen": len(rows), "subdomains": parse_crtsh(rows, domain)}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
# --- wayback machine ---
|
|
221
|
+
|
|
222
|
+
def parse_cdx(rows: list) -> list[dict]:
|
|
223
|
+
if not rows:
|
|
224
|
+
return []
|
|
225
|
+
header, *rest = rows
|
|
226
|
+
out = []
|
|
227
|
+
for row in rest:
|
|
228
|
+
item = dict(zip(header, row))
|
|
229
|
+
item["archive_url"] = f"https://web.archive.org/web/{item['timestamp']}/{item['original']}"
|
|
230
|
+
out.append(item)
|
|
231
|
+
return out
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
async def wayback_snapshots(client, url, limit=20, newest_first=True, year_from=None, year_to=None):
|
|
235
|
+
params = {
|
|
236
|
+
"url": url,
|
|
237
|
+
"output": "json",
|
|
238
|
+
"fl": "timestamp,original,statuscode,mimetype",
|
|
239
|
+
"collapse": "digest",
|
|
240
|
+
# negative limit = last N results
|
|
241
|
+
"limit": -limit if newest_first else limit,
|
|
242
|
+
}
|
|
243
|
+
if year_from:
|
|
244
|
+
params["from"] = str(year_from)
|
|
245
|
+
if year_to:
|
|
246
|
+
params["to"] = str(year_to)
|
|
247
|
+
|
|
248
|
+
# the cdx api regularly takes 20-30s, give it room
|
|
249
|
+
r = await get_flaky(client, "https://web.archive.org/cdx/search/cdx", params=params, timeout=60.0)
|
|
250
|
+
r.raise_for_status()
|
|
251
|
+
snaps = parse_cdx(r.json() if r.content else [])
|
|
252
|
+
if newest_first:
|
|
253
|
+
snaps.reverse()
|
|
254
|
+
return {"url": url, "count": len(snaps), "snapshots": snaps}
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
# --- http headers ---
|
|
258
|
+
|
|
259
|
+
async def http_headers(client, url):
|
|
260
|
+
if "://" not in url:
|
|
261
|
+
url = "https://" + url
|
|
262
|
+
r = await client.get(url)
|
|
263
|
+
headers = {k.lower(): v for k, v in r.headers.items()}
|
|
264
|
+
return {
|
|
265
|
+
"url": url,
|
|
266
|
+
"final_url": str(r.url),
|
|
267
|
+
"status": r.status_code,
|
|
268
|
+
"redirects": [str(h.url) for h in r.history],
|
|
269
|
+
"server": headers.get("server"),
|
|
270
|
+
"headers": headers,
|
|
271
|
+
"missing_security_headers": [h for h in SECURITY_HEADERS if h not in headers],
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
# --- shodan internetdb (free, no key) ---
|
|
276
|
+
|
|
277
|
+
async def internetdb(client, ip):
|
|
278
|
+
r = await client.get(f"https://internetdb.shodan.io/{ip}")
|
|
279
|
+
if r.status_code == 404:
|
|
280
|
+
return {"ip": ip, "ports": [], "note": "shodan has nothing on this ip"}
|
|
281
|
+
r.raise_for_status()
|
|
282
|
+
return r.json()
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
|
|
3
|
+
import httpx
|
|
4
|
+
import pytest
|
|
5
|
+
|
|
6
|
+
from osint_mcp import sources
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@pytest.mark.parametrize(
|
|
10
|
+
"raw, expected",
|
|
11
|
+
[
|
|
12
|
+
("example.com", "example.com"),
|
|
13
|
+
(" Example.COM. ", "example.com"),
|
|
14
|
+
("https://www.example.com/login?x=1", "www.example.com"),
|
|
15
|
+
("example.com:8443/path", "example.com"),
|
|
16
|
+
],
|
|
17
|
+
)
|
|
18
|
+
def test_clean_domain(raw, expected):
|
|
19
|
+
assert sources.clean_domain(raw) == expected
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@pytest.mark.parametrize("raw", ["", "localhost", "https://"])
|
|
23
|
+
def test_clean_domain_rejects_garbage(raw):
|
|
24
|
+
with pytest.raises(ValueError):
|
|
25
|
+
sources.clean_domain(raw)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_clean_ip():
|
|
29
|
+
assert sources.clean_ip(" 8.8.8.8 ") == "8.8.8.8"
|
|
30
|
+
assert sources.clean_ip("2001:4860:4860:0:0:0:0:8888") == "2001:4860:4860::8888"
|
|
31
|
+
with pytest.raises(ValueError):
|
|
32
|
+
sources.clean_ip("999.1.1.1")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _vcard(name=None, email=None):
|
|
36
|
+
fields = [["version", {}, "text", "4.0"]]
|
|
37
|
+
if name:
|
|
38
|
+
fields.append(["fn", {}, "text", name])
|
|
39
|
+
if email:
|
|
40
|
+
fields.append(["email", {}, "text", email])
|
|
41
|
+
return ["vcard", fields]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
RDAP_DOMAIN = {
|
|
45
|
+
"ldhName": "EXAMPLE.COM",
|
|
46
|
+
"status": ["client transfer prohibited"],
|
|
47
|
+
"events": [
|
|
48
|
+
{"eventAction": "registration", "eventDate": "1995-08-14T04:00:00Z"},
|
|
49
|
+
{"eventAction": "expiration", "eventDate": "2026-08-13T04:00:00Z"},
|
|
50
|
+
{"eventAction": "last changed", "eventDate": "2025-08-14T07:01:39Z"},
|
|
51
|
+
],
|
|
52
|
+
"nameservers": [{"ldhName": "B.IANA-SERVERS.NET"}, {"ldhName": "A.IANA-SERVERS.NET"}],
|
|
53
|
+
"entities": [
|
|
54
|
+
{
|
|
55
|
+
"roles": ["registrar"],
|
|
56
|
+
"vcardArray": _vcard("RESERVED-Internet Assigned Numbers Authority"),
|
|
57
|
+
"entities": [{"roles": ["abuse"], "vcardArray": _vcard(email="abuse@iana.org")}],
|
|
58
|
+
}
|
|
59
|
+
],
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def test_parse_rdap_domain():
|
|
64
|
+
info = sources.parse_rdap_domain(RDAP_DOMAIN)
|
|
65
|
+
assert info["domain"] == "example.com"
|
|
66
|
+
assert info["registrar"] == "RESERVED-Internet Assigned Numbers Authority"
|
|
67
|
+
assert info["abuse_email"] == "abuse@iana.org"
|
|
68
|
+
assert info["registered"] == "1995-08-14T04:00:00Z"
|
|
69
|
+
assert info["expires"] == "2026-08-13T04:00:00Z"
|
|
70
|
+
assert info["nameservers"] == ["a.iana-servers.net", "b.iana-servers.net"]
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_parse_rdap_domain_handles_missing_bits():
|
|
74
|
+
info = sources.parse_rdap_domain({"ldhName": "foo.ro"})
|
|
75
|
+
assert info["registrar"] is None
|
|
76
|
+
assert info["nameservers"] == []
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_parse_rdap_ip():
|
|
80
|
+
data = {
|
|
81
|
+
"name": "GOGL",
|
|
82
|
+
"handle": "NET-8-8-8-0-2",
|
|
83
|
+
"startAddress": "8.8.8.0",
|
|
84
|
+
"endAddress": "8.8.8.255",
|
|
85
|
+
"cidr0_cidrs": [{"v4prefix": "8.8.8.0", "length": 24}],
|
|
86
|
+
"entities": [
|
|
87
|
+
{"roles": ["registrant"], "vcardArray": _vcard("Google LLC")},
|
|
88
|
+
{"roles": ["abuse"], "vcardArray": _vcard("Abuse", "network-abuse@google.com")},
|
|
89
|
+
],
|
|
90
|
+
}
|
|
91
|
+
info = sources.parse_rdap_ip(data)
|
|
92
|
+
assert info["network"] == "GOGL"
|
|
93
|
+
assert info["cidrs"] == ["8.8.8.0/24"]
|
|
94
|
+
assert info["org"] == "Google LLC"
|
|
95
|
+
assert info["abuse_email"] == "network-abuse@google.com"
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def test_parse_crtsh_dedupes_and_filters():
|
|
99
|
+
rows = [
|
|
100
|
+
{"name_value": "example.com\nwww.example.com"},
|
|
101
|
+
{"name_value": "*.example.com"},
|
|
102
|
+
{"name_value": "API.example.com\nmail.example.com"},
|
|
103
|
+
{"name_value": "www.example.com"},
|
|
104
|
+
{"name_value": "notexample.com\nexample.com.evil.net"},
|
|
105
|
+
]
|
|
106
|
+
assert sources.parse_crtsh(rows, "example.com") == [
|
|
107
|
+
"api.example.com",
|
|
108
|
+
"example.com",
|
|
109
|
+
"mail.example.com",
|
|
110
|
+
"www.example.com",
|
|
111
|
+
]
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def test_parse_cdx():
|
|
115
|
+
rows = [
|
|
116
|
+
["timestamp", "original", "statuscode", "mimetype"],
|
|
117
|
+
["20200101000000", "http://example.com/", "200", "text/html"],
|
|
118
|
+
]
|
|
119
|
+
snaps = sources.parse_cdx(rows)
|
|
120
|
+
assert snaps[0]["statuscode"] == "200"
|
|
121
|
+
assert snaps[0]["archive_url"] == "https://web.archive.org/web/20200101000000/http://example.com/"
|
|
122
|
+
assert sources.parse_cdx([]) == []
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _client(handler):
|
|
126
|
+
return httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def test_crtsh_subdomains_request():
|
|
130
|
+
seen = {}
|
|
131
|
+
|
|
132
|
+
def handler(request):
|
|
133
|
+
seen["q"] = request.url.params["q"]
|
|
134
|
+
return httpx.Response(200, json=[{"name_value": "a.example.com"}])
|
|
135
|
+
|
|
136
|
+
async def go():
|
|
137
|
+
async with _client(handler) as c:
|
|
138
|
+
return await sources.crtsh_subdomains(c, "example.com")
|
|
139
|
+
|
|
140
|
+
result = asyncio.run(go())
|
|
141
|
+
assert seen["q"] == "%.example.com"
|
|
142
|
+
assert result["subdomains"] == ["a.example.com"]
|
|
143
|
+
assert result["certs_seen"] == 1
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def test_wayback_newest_first():
|
|
147
|
+
def handler(request):
|
|
148
|
+
assert request.url.params["limit"] == "-2"
|
|
149
|
+
return httpx.Response(200, json=[
|
|
150
|
+
["timestamp", "original", "statuscode", "mimetype"],
|
|
151
|
+
["20190101000000", "http://example.com/", "200", "text/html"],
|
|
152
|
+
["20240101000000", "http://example.com/", "200", "text/html"],
|
|
153
|
+
])
|
|
154
|
+
|
|
155
|
+
async def go():
|
|
156
|
+
async with _client(handler) as c:
|
|
157
|
+
return await sources.wayback_snapshots(c, "example.com", limit=2)
|
|
158
|
+
|
|
159
|
+
result = asyncio.run(go())
|
|
160
|
+
assert [s["timestamp"] for s in result["snapshots"]] == ["20240101000000", "20190101000000"]
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def test_internetdb_404_is_not_an_error():
|
|
164
|
+
async def go():
|
|
165
|
+
async with _client(lambda r: httpx.Response(404, json={"detail": "No information available"})) as c:
|
|
166
|
+
return await sources.internetdb(c, "10.0.0.1")
|
|
167
|
+
|
|
168
|
+
assert asyncio.run(go())["ports"] == []
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def test_http_headers_reports_missing():
|
|
172
|
+
def handler(request):
|
|
173
|
+
return httpx.Response(200, headers={"Server": "nginx", "X-Frame-Options": "DENY"})
|
|
174
|
+
|
|
175
|
+
async def go():
|
|
176
|
+
async with _client(handler) as c:
|
|
177
|
+
return await sources.http_headers(c, "example.com")
|
|
178
|
+
|
|
179
|
+
result = asyncio.run(go())
|
|
180
|
+
assert result["url"] == "https://example.com"
|
|
181
|
+
assert result["server"] == "nginx"
|
|
182
|
+
assert "x-frame-options" not in result["missing_security_headers"]
|
|
183
|
+
assert "strict-transport-security" in result["missing_security_headers"]
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def test_flaky_endpoints_get_retried(monkeypatch):
|
|
187
|
+
async def no_sleep(_):
|
|
188
|
+
pass
|
|
189
|
+
|
|
190
|
+
monkeypatch.setattr(sources.asyncio, "sleep", no_sleep)
|
|
191
|
+
responses = iter([httpx.Response(503), httpx.Response(502), httpx.Response(200, json=[])])
|
|
192
|
+
|
|
193
|
+
async def go():
|
|
194
|
+
async with _client(lambda r: next(responses)) as c:
|
|
195
|
+
return await sources.crtsh_subdomains(c, "example.com")
|
|
196
|
+
|
|
197
|
+
assert asyncio.run(go())["subdomains"] == []
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def test_gives_up_after_retries(monkeypatch):
|
|
201
|
+
async def no_sleep(_):
|
|
202
|
+
pass
|
|
203
|
+
|
|
204
|
+
monkeypatch.setattr(sources.asyncio, "sleep", no_sleep)
|
|
205
|
+
|
|
206
|
+
async def go():
|
|
207
|
+
async with _client(lambda r: httpx.Response(503)) as c:
|
|
208
|
+
return await sources.wayback_snapshots(c, "example.com")
|
|
209
|
+
|
|
210
|
+
with pytest.raises(httpx.HTTPStatusError):
|
|
211
|
+
asyncio.run(go())
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def test_rdap_domain_404_explains_itself():
|
|
215
|
+
async def go():
|
|
216
|
+
async with _client(lambda r: httpx.Response(404)) as c:
|
|
217
|
+
return await sources.rdap_domain(c, "emag.ro")
|
|
218
|
+
|
|
219
|
+
result = asyncio.run(go())
|
|
220
|
+
assert "no RDAP data" in result["error"]
|