tech-teardown 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tech_teardown-0.1.1/.github/workflows/ci.yml +28 -0
- tech_teardown-0.1.1/.github/workflows/release.yml +47 -0
- tech_teardown-0.1.1/.gitignore +10 -0
- tech_teardown-0.1.1/.mcp.json.example +12 -0
- tech_teardown-0.1.1/AGENTS.md +116 -0
- tech_teardown-0.1.1/CLAUDE.md +6 -0
- tech_teardown-0.1.1/LICENSE +21 -0
- tech_teardown-0.1.1/PKG-INFO +242 -0
- tech_teardown-0.1.1/PUBLISHING.md +67 -0
- tech_teardown-0.1.1/README.md +220 -0
- tech_teardown-0.1.1/docs/design.md +118 -0
- tech_teardown-0.1.1/docs/intelligence-design.md +31 -0
- tech_teardown-0.1.1/docs/intelligence-plan.md +15 -0
- tech_teardown-0.1.1/docs/modular-scans-design.md +26 -0
- tech_teardown-0.1.1/docs/modular-scans-plan.md +13 -0
- tech_teardown-0.1.1/docs/superpowers/plans/2026-11-16-scan-hardening.md +76 -0
- tech_teardown-0.1.1/pyproject.toml +36 -0
- tech_teardown-0.1.1/requirements.txt +3 -0
- tech_teardown-0.1.1/scripts/update_builtin_fingerprints.mjs +23 -0
- tech_teardown-0.1.1/teardown/__init__.py +5 -0
- tech_teardown-0.1.1/teardown/__main__.py +4 -0
- tech_teardown-0.1.1/teardown/analyze.py +421 -0
- tech_teardown-0.1.1/teardown/cache.py +60 -0
- tech_teardown-0.1.1/teardown/cli.py +241 -0
- tech_teardown-0.1.1/teardown/collectors/__init__.py +1 -0
- tech_teardown-0.1.1/teardown/collectors/api_discovery.py +165 -0
- tech_teardown-0.1.1/teardown/collectors/headers.py +169 -0
- tech_teardown-0.1.1/teardown/collectors/html_js.py +475 -0
- tech_teardown-0.1.1/teardown/data/LICENSE-opentechalyzer +21 -0
- tech_teardown-0.1.1/teardown/data/LICENSE.md +6 -0
- tech_teardown-0.1.1/teardown/data/__init__.py +3 -0
- tech_teardown-0.1.1/teardown/data/curated.json +15 -0
- tech_teardown-0.1.1/teardown/data/fingerprints.json +8631 -0
- tech_teardown-0.1.1/teardown/diff.py +56 -0
- tech_teardown-0.1.1/teardown/enumerate.py +239 -0
- tech_teardown-0.1.1/teardown/fingerprints.py +246 -0
- tech_teardown-0.1.1/teardown/http_client.py +201 -0
- tech_teardown-0.1.1/teardown/mcp_server.py +81 -0
- tech_teardown-0.1.1/teardown/models.py +196 -0
- tech_teardown-0.1.1/teardown/render.py +299 -0
- tech_teardown-0.1.1/teardown/safety.py +85 -0
- tech_teardown-0.1.1/teardown/scan_modules.py +142 -0
- tech_teardown-0.1.1/tests/test_analyze.py +368 -0
- tech_teardown-0.1.1/tests/test_api_discovery.py +212 -0
- tech_teardown-0.1.1/tests/test_enumerate.py +211 -0
- tech_teardown-0.1.1/tests/test_hardening.py +275 -0
- tech_teardown-0.1.1/tests/test_headers.py +124 -0
- tech_teardown-0.1.1/tests/test_html_js.py +252 -0
- tech_teardown-0.1.1/tests/test_intelligence.py +356 -0
- tech_teardown-0.1.1/tests/test_render.py +170 -0
- tech_teardown-0.1.1/tests/test_scan_modules.py +116 -0
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
permissions:
|
|
10
|
+
contents: read
|
|
11
|
+
|
|
12
|
+
jobs:
|
|
13
|
+
test:
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
strategy:
|
|
16
|
+
fail-fast: false
|
|
17
|
+
matrix:
|
|
18
|
+
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v7
|
|
21
|
+
- uses: actions/setup-python@v7
|
|
22
|
+
with:
|
|
23
|
+
python-version: ${{ matrix.python-version }}
|
|
24
|
+
- run: python -m pip install -e '.[active-dns,mcp]' pytest pytest-asyncio build
|
|
25
|
+
- run: python -m pytest -q
|
|
26
|
+
- run: python -m build
|
|
27
|
+
- run: python -m pip install --force-reinstall dist/*.whl
|
|
28
|
+
- run: teardown --help
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*'
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
build:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
steps:
|
|
15
|
+
- uses: actions/checkout@v7
|
|
16
|
+
with:
|
|
17
|
+
fetch-depth: 0
|
|
18
|
+
- uses: actions/setup-python@v7
|
|
19
|
+
with:
|
|
20
|
+
python-version: '3.11'
|
|
21
|
+
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
|
|
22
|
+
- name: Verify tag and main ancestry
|
|
23
|
+
run: |
|
|
24
|
+
git fetch origin main
|
|
25
|
+
git merge-base --is-ancestor "$GITHUB_SHA" origin/main
|
|
26
|
+
python -c 'import os, tomllib; from pathlib import Path; version = tomllib.loads(Path("pyproject.toml").read_text())["project"]["version"]; assert os.environ["GITHUB_REF_NAME"] == "v" + version, "tag does not match project version"'
|
|
27
|
+
- run: python -m pip install -e '.[active-dns,mcp]' pytest pytest-asyncio
|
|
28
|
+
- run: python -m pytest -q
|
|
29
|
+
- run: uv build --no-sources
|
|
30
|
+
- uses: actions/upload-artifact@v7
|
|
31
|
+
with:
|
|
32
|
+
name: dist
|
|
33
|
+
path: dist/
|
|
34
|
+
|
|
35
|
+
publish:
|
|
36
|
+
needs: build
|
|
37
|
+
runs-on: ubuntu-latest
|
|
38
|
+
permissions:
|
|
39
|
+
id-token: write
|
|
40
|
+
contents: read
|
|
41
|
+
steps:
|
|
42
|
+
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
|
|
43
|
+
- uses: actions/download-artifact@v8
|
|
44
|
+
with:
|
|
45
|
+
name: dist
|
|
46
|
+
path: dist/
|
|
47
|
+
- run: uv publish --trusted-publishing always dist/*
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
# AGENTS.md
|
|
2
|
+
|
|
3
|
+
Dev guide for `teardown` — a passive web-app architecture recon CLI with
|
|
4
|
+
subdomain fan-out. Read this before changing code.
|
|
5
|
+
|
|
6
|
+
## What this is
|
|
7
|
+
|
|
8
|
+
Reverse-engineers a web app's architecture from **publicly accessible frontend
|
|
9
|
+
signals only** — no auth, no exploitation. Given a domain it:
|
|
10
|
+
|
|
11
|
+
1. Enumerates subdomains (crt.sh, CertSpotter, Wayback passive; optional
|
|
12
|
+
subfinder; `--active-dns` opt-in DNS brute).
|
|
13
|
+
2. Filters to live hosts.
|
|
14
|
+
3. Tears down each host: headers, CSP, framework/CMS, JS bundles + source maps,
|
|
15
|
+
REST/GraphQL/OpenAPI/WordPress API surface, third-party fingerprints, redacted
|
|
16
|
+
secret leaks, OPSEC findings.
|
|
17
|
+
4. Emits an **agent-readable JSON contract** (`ScanResult`/`HostTeardown`, schema
|
|
18
|
+
1.2) plus a vault-compatible markdown rendering and cross-run diff.
|
|
19
|
+
|
|
20
|
+
The guiding split: **deterministic collection (this code) vs. interpretation (the
|
|
21
|
+
agent that reads the JSON).** Keep narrative/judgment out of the tool; it produces
|
|
22
|
+
normalized signals and the mechanical findings, nothing fuzzier.
|
|
23
|
+
|
|
24
|
+
## Layout
|
|
25
|
+
|
|
26
|
+
```
|
|
27
|
+
teardown/
|
|
28
|
+
models.py # THE contract — dataclasses + SCHEMA_VERSION. Change carefully.
|
|
29
|
+
http_client.py # async httpx client + Fetcher protocol + FetchResponse
|
|
30
|
+
fingerprints.py # data-driven matcher + optional upstream importer
|
|
31
|
+
scan_modules.py # ordered, injectable host collectors
|
|
32
|
+
data/ # bundled MIT fingerprints and attribution
|
|
33
|
+
cache.py # sanitized daily response cache
|
|
34
|
+
collectors/
|
|
35
|
+
headers.py # parse_headers, parse_csp (pure)
|
|
36
|
+
html_js.py # frameworks/CMS, bundles, third-parties, secret leaks; enrich_bundles (fetcher)
|
|
37
|
+
api_discovery.py # discover_api (fetcher)
|
|
38
|
+
enumerate.py # enumerate_subdomains, filter_live (fetcher)
|
|
39
|
+
diff.py # stable cross-run comparison
|
|
40
|
+
mcp_server.py # optional FastMCP stdio / Streamable HTTP
|
|
41
|
+
analyze.py # analyze_host, build_aggregate (pure)
|
|
42
|
+
render.py # render_markdown, render_host_markdown, write_outputs (pure)
|
|
43
|
+
cli.py / __main__ # argparse + asyncio orchestration
|
|
44
|
+
tests/ # pytest, fully offline (fake Fetcher)
|
|
45
|
+
docs/design.md # the design doc / why
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Core conventions (do not break)
|
|
49
|
+
|
|
50
|
+
- **Collectors never import httpx.** Anything that needs the network takes an
|
|
51
|
+
injected `Fetcher` (see `http_client.Fetcher`); pure parsers take already-fetched
|
|
52
|
+
text/headers and return partial model objects. This is what keeps tests offline.
|
|
53
|
+
- **`Fetcher.get/request` never raise.** Failures come back as `FetchResponse` with
|
|
54
|
+
`status=None` / `error` set. Collector code must tolerate that, not try/except the
|
|
55
|
+
network.
|
|
56
|
+
- **Secrets are never stored in full** — only a redacted prefix (`value_prefix`).
|
|
57
|
+
Tests assert the full secret is absent from output. Keep it that way.
|
|
58
|
+
- **Passive only.** No authenticated requests, no exploit payloads, no evasion.
|
|
59
|
+
`--active-dns` is the only active enumeration path and is opt-in. Passive
|
|
60
|
+
source adapters must stay bounded and scope-filtered.
|
|
61
|
+
- **Politeness lives in `HttpClient`** (bounded semaphore + token-bucket rate limit +
|
|
62
|
+
identifying UA). Don't bypass it with raw httpx calls.
|
|
63
|
+
- `from __future__ import annotations` at the top of every module; `requires-python
|
|
64
|
+
>= 3.10`.
|
|
65
|
+
|
|
66
|
+
## Running
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
# tests (must stay offline + fast, ~0.1s)
|
|
70
|
+
python -m venv .venv && .venv/bin/pip install -e '.[active-dns]' pytest pytest-asyncio
|
|
71
|
+
.venv/bin/python -m pytest -q
|
|
72
|
+
|
|
73
|
+
# run the CLI in dev
|
|
74
|
+
.venv/bin/teardown example.com --single --json-only
|
|
75
|
+
|
|
76
|
+
# run exactly as users do, from the git remote
|
|
77
|
+
uvx --from git+ssh://git@github.com/ReyemTech/teardown teardown example.com --single
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Publishing to PyPI: see `PUBLISHING.md` (manual `uv build` + `uv publish`; not yet
|
|
81
|
+
on PyPI).
|
|
82
|
+
|
|
83
|
+
## Extending
|
|
84
|
+
|
|
85
|
+
- **New third-party / secret fingerprint:** add the regex to `html_js.py`
|
|
86
|
+
(`fingerprint_third_parties` / `detect_secret_leaks`) with a redaction, and a
|
|
87
|
+
fixture test in `tests/test_html_js.py`. Mirror the pattern style already there.
|
|
88
|
+
- **New technology fingerprint:** add data to `teardown/data/curated.json`, or
|
|
89
|
+
update the MIT upstream conversion with `scripts/update_builtin_fingerprints.mjs`.
|
|
90
|
+
Match on existing passive signals and keep evidence free of raw secret values.
|
|
91
|
+
- **New OPSEC finding:** add the rule in `analyze.py::analyze_host`, pick the right
|
|
92
|
+
`Severity`, cite the actual value in `detail`, set `evidence_url` when derivable,
|
|
93
|
+
and add a test that it fires on the signal AND does NOT fire on a clean host.
|
|
94
|
+
- **New collector:** implement a `ScanModule` over `ScanContext`, use its injected
|
|
95
|
+
`Fetcher`, and add it to `scan_modules.py` or register it in-process. Return a
|
|
96
|
+
redacted mapping for the namespaced `HostTeardown.extensions` payload and
|
|
97
|
+
give it an offline fake-Fetcher test.
|
|
98
|
+
- **Schema change:** edit `models.py`, bump `SCHEMA_VERSION`, and update
|
|
99
|
+
`render.py` + any consumer. The JSON is a contract other agents read — treat a
|
|
100
|
+
field rename as breaking.
|
|
101
|
+
|
|
102
|
+
## Known gotchas (already handled — don't regress)
|
|
103
|
+
|
|
104
|
+
- **crt.sh is flaky** — intermittently returns a 404/HTML error page (a "successful"
|
|
105
|
+
HTTP response the client won't retry). `enumerate.py` retries at the app layer with
|
|
106
|
+
backoff and a bare-domain query fallback. `CRTSH_RETRY_BACKOFF` is a module constant
|
|
107
|
+
so tests can zero it.
|
|
108
|
+
- **Source-map findings are first-party only, severity MEDIUM.** A third-party CDN
|
|
109
|
+
(Sentry, jsdelivr) publishing a map for its own SDK is not the target's exposure.
|
|
110
|
+
Don't re-classify these as CRITICAL or drop the first-party check.
|
|
111
|
+
- Server-rendered stacks can now be detected from headers, cookies and markup.
|
|
112
|
+
Preserve cookie names for matching, but never cookie values.
|
|
113
|
+
|
|
114
|
+
## Commits
|
|
115
|
+
|
|
116
|
+
Conventional, imperative subject. **No AI/Claude co-authorship trailers.**
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Mario Meyer / Reyem Technologies Inc.
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: tech-teardown
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary: Passive web-app architecture reconnaissance with subdomain fan-out, emitting an agent-readable JSON contract.
|
|
5
|
+
Author-email: Mario Meyer <mario@reyem.tech>
|
|
6
|
+
License: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: osint,passive,recon,security,subdomains,teardown
|
|
9
|
+
Classifier: Environment :: Console
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Topic :: Security
|
|
13
|
+
Requires-Python: >=3.10
|
|
14
|
+
Requires-Dist: httpx<1.0,>=0.27
|
|
15
|
+
Requires-Dist: tldextract<6.0,>=5.1
|
|
16
|
+
Provides-Extra: active-dns
|
|
17
|
+
Requires-Dist: dnspython<3.0,>=2.6; extra == 'active-dns'
|
|
18
|
+
Provides-Extra: mcp
|
|
19
|
+
Requires-Dist: fastmcp<3,>=2.14; (python_version < '3.11') and extra == 'mcp'
|
|
20
|
+
Requires-Dist: fastmcp<5,>=2.14; (python_version >= '3.11') and extra == 'mcp'
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# Teardown Tool
|
|
24
|
+
|
|
25
|
+
Passive web-app architecture reconnaissance with subdomain fan-out. It collects
|
|
26
|
+
public frontend signals and writes a versioned JSON record for agents plus a
|
|
27
|
+
vault-compatible Markdown report for people. Collection is deterministic;
|
|
28
|
+
interpretation belongs to the agent reading the JSON.
|
|
29
|
+
|
|
30
|
+
**Passive recon only.** No authenticated access, no exploitation. Active DNS
|
|
31
|
+
brute-force is opt-in (`--active-dns`) and off by default.
|
|
32
|
+
|
|
33
|
+
## Run with uvx (no install)
|
|
34
|
+
|
|
35
|
+
```bash
|
|
36
|
+
# Once published to PyPI:
|
|
37
|
+
uvx --from tech-teardown teardown example.com --single
|
|
38
|
+
|
|
39
|
+
# Straight from the repo (works today):
|
|
40
|
+
uvx --from git+https://github.com/ReyemTech/teardown teardown example.com --single
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The PyPI distribution is `tech-teardown`; the command stays `teardown`. `uvx` builds an
|
|
44
|
+
ephemeral environment and runs it — nothing to install. For the active-DNS extra:
|
|
45
|
+
`uvx --with dnspython --from git+https://github.com/ReyemTech/teardown teardown example.com --active-dns`.
|
|
46
|
+
|
|
47
|
+
## Local dev install
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
python3 -m venv .venv
|
|
51
|
+
.venv/bin/python -m pip install -e '.[active-dns,mcp]'
|
|
52
|
+
.venv/bin/teardown example.com --single
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
`dnspython` is only needed for `--active-dns`; the passive path runs without it.
|
|
56
|
+
|
|
57
|
+
## Usage
|
|
58
|
+
|
|
59
|
+
```bash
|
|
60
|
+
# Single host, no subdomain enumeration
|
|
61
|
+
teardown example.com --single --out-dir .
|
|
62
|
+
|
|
63
|
+
# Full: enumerate via crt.sh, CertSpotter and Wayback; check up to 100 hosts
|
|
64
|
+
teardown example.com --out-dir .
|
|
65
|
+
|
|
66
|
+
# Limit liveness checks and host teardowns separately
|
|
67
|
+
teardown example.com --max-probes 40 --max-hosts 10
|
|
68
|
+
|
|
69
|
+
# Add installed subfinder's passive sources
|
|
70
|
+
teardown example.com --subfinder
|
|
71
|
+
|
|
72
|
+
# Opt in to active DNS brute-force
|
|
73
|
+
teardown example.com --active-dns --wordlist words.txt
|
|
74
|
+
|
|
75
|
+
# Machine output only (no files)
|
|
76
|
+
teardown example.com --single --json-only
|
|
77
|
+
|
|
78
|
+
# Compare two saved JSON scans
|
|
79
|
+
teardown diff last-month.json this-month.json
|
|
80
|
+
|
|
81
|
+
# Fetch and convert the optional community fingerprint database
|
|
82
|
+
teardown fingerprints update
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
### Flags
|
|
86
|
+
|
|
87
|
+
| Flag | Default | Meaning |
|
|
88
|
+
|------|---------|---------|
|
|
89
|
+
| `--single` | off | Scan only the given host; skip enumeration. |
|
|
90
|
+
| `--active-dns` | off | Add DNS brute-force (needs `dnspython`). |
|
|
91
|
+
| `--subfinder` | off | Merge installed subfinder's passive results. |
|
|
92
|
+
| `--wordlist PATH` | built-in | Wordlist for `--active-dns`. |
|
|
93
|
+
| `--max-hosts N` | 25 | Cap live hosts torn down (0 = no cap). |
|
|
94
|
+
| `--max-probes N` | 100 | Cap enumerated hosts checked for liveness (0 = no cap). |
|
|
95
|
+
| `--concurrency N` | 10 | Max concurrent requests. |
|
|
96
|
+
| `--rate R` | 8.0 | Global requests/sec cap. |
|
|
97
|
+
| `--out-dir DIR` | `.` | Where JSON + markdown land. |
|
|
98
|
+
| `--name NAME` | from domain | Output basename. |
|
|
99
|
+
| `--json-only` | off | Print JSON to stdout, write nothing. |
|
|
100
|
+
| `--cache-dir DIR` | `.teardown-cache/` | Daily response cache location. |
|
|
101
|
+
| `--no-cache` | off | Fetch fresh responses. |
|
|
102
|
+
|
|
103
|
+
## Output
|
|
104
|
+
|
|
105
|
+
Two files per scan: `<name>-teardown.json` (machine, versioned schema) and
|
|
106
|
+
`<name>-teardown.md` (vault-ready, matches the existing teardown frontmatter).
|
|
107
|
+
|
|
108
|
+
The **JSON is the contract an agent reads** — one `HostTeardown` record per host
|
|
109
|
+
inside a `ScanResult` rollup. Shapes are defined in `teardown/models.py`
|
|
110
|
+
(`SCHEMA_VERSION`). An agent should consume the JSON and narrate; the markdown is
|
|
111
|
+
for humans.
|
|
112
|
+
|
|
113
|
+
Schema 1.2 adds `detections` per host, `aggregate.detections_union`, and
|
|
114
|
+
namespaced `extensions` for additional scan modules. Each detection
|
|
115
|
+
record has a technology name,
|
|
116
|
+
categories, deterministic `confidence` (0–100), and signal labels in `evidence`.
|
|
117
|
+
Confidence reflects matched signal strength and corroboration; it is not a
|
|
118
|
+
statistically calibrated probability. The legacy `frameworks` and `cms` views
|
|
119
|
+
remain. `http` now includes page title, redirect status chain, observed TLS
|
|
120
|
+
version/cipher/certificate expiry and CDN header fingerprint when available.
|
|
121
|
+
`teardown diff` reports added and removed hosts plus field changes; scan times
|
|
122
|
+
and volatile response headers are excluded.
|
|
123
|
+
|
|
124
|
+
The wheel bundles 589 MIT-licensed fingerprints (588 from OpenTechAlyzer plus
|
|
125
|
+
the Livewire definition). `teardown fingerprints update` downloads and converts
|
|
126
|
+
the larger WebAppAnalyzer database automatically into the user's local cache.
|
|
127
|
+
That optional dataset is GPL-3.0 and is not distributed with this repository or
|
|
128
|
+
wheel. Set `TEARDOWN_FINGERPRINT_DB` to choose its local path. Imported
|
|
129
|
+
definitions update bundled names; the local curated definitions take final
|
|
130
|
+
precedence. Attribution is in
|
|
131
|
+
[`teardown/data/LICENSE.md`](teardown/data/LICENSE.md).
|
|
132
|
+
|
|
133
|
+
## MCP server
|
|
134
|
+
|
|
135
|
+
Install the optional extra, then use stdio or local Streamable HTTP:
|
|
136
|
+
|
|
137
|
+
```bash
|
|
138
|
+
.venv/bin/python -m pip install -e '.[mcp]'
|
|
139
|
+
teardown-mcp
|
|
140
|
+
teardown-mcp --transport http --host 127.0.0.1 --port 8000
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The HTTP endpoint is `http://127.0.0.1:8000/mcp`. The tools are
|
|
144
|
+
`teardown_host`, `teardown_domain`, `enumerate_subdomains`, `scan_domain`
|
|
145
|
+
(compatibility alias), and `diff_scans` (two JSON objects). They reuse the CLI
|
|
146
|
+
collection path. The default bind address is loopback. See
|
|
147
|
+
[`.mcp.json.example`](.mcp.json.example) for a stdio registration example.
|
|
148
|
+
|
|
149
|
+
## How collection behaves
|
|
150
|
+
|
|
151
|
+
- The registrable root domain comes from a bundled public suffix list, so a
|
|
152
|
+
target such as `app.example.co.uk` enumerates `example.co.uk`. This does not
|
|
153
|
+
fetch the suffix list during a scan. Private suffixes are respected, so
|
|
154
|
+
`app.foo.github.io` stays within the `foo.github.io` tenant.
|
|
155
|
+
- The scan fetches only hosts under that root, plus crt.sh, CertSpotter and
|
|
156
|
+
Wayback for passive enumeration. It stops at redirects outside that scope.
|
|
157
|
+
Third-party asset URLs
|
|
158
|
+
may appear in the result, but their bodies are not fetched.
|
|
159
|
+
- Liveness checks stop after `--max-probes` candidates. The 25-host teardown cap
|
|
160
|
+
applies after liveness checks. The liveness response is reused for collection.
|
|
161
|
+
Each HTTP response body is limited to 8 MB.
|
|
162
|
+
- GET responses are cached by URL and UTC day under `.teardown-cache/` so a
|
|
163
|
+
rerun resumes without repeating successful requests. Bodies mentioning
|
|
164
|
+
sensitive concepts, containing long opaque values, or matching recognized
|
|
165
|
+
secret patterns are not persisted. The cache keeps only cookie names, never
|
|
166
|
+
cookie values.
|
|
167
|
+
- API paths count as observed only when their responses support the claim: REST
|
|
168
|
+
requires JSON or an access-denied status; GraphQL requires a GraphQL-shaped
|
|
169
|
+
response; OpenAPI requires a version marker and `paths` object. A generic
|
|
170
|
+
HTML fallback does not count. A source map must parse as a v3 source map.
|
|
171
|
+
- Extracted keys are stored as prefixes, credential-bearing headers are
|
|
172
|
+
redacted, and sensitive URL query values are masked in output. Inspect the
|
|
173
|
+
JSON before sharing it: public pages can contain identifiers outside these
|
|
174
|
+
recognized patterns. Assets with credential-bearing query parameters are
|
|
175
|
+
listed with masked URLs and may not be fetchable for bundle enrichment.
|
|
176
|
+
|
|
177
|
+
## Architecture
|
|
178
|
+
|
|
179
|
+
```
|
|
180
|
+
teardown/
|
|
181
|
+
models.py # versioned dataclasses — the contract
|
|
182
|
+
http_client.py # polite async httpx client + Fetcher protocol
|
|
183
|
+
safety.py # output redaction + request scope checks
|
|
184
|
+
fingerprints.py # database matcher + optional WebAppAnalyzer importer
|
|
185
|
+
scan_modules.py # ordered, injectable host scan modules
|
|
186
|
+
cache.py # sanitized daily response cache
|
|
187
|
+
collectors/
|
|
188
|
+
headers.py # headers + CSP
|
|
189
|
+
html_js.py # CMS details, bundles, third-parties, secret leaks
|
|
190
|
+
api_discovery.py # REST / GraphQL / OpenAPI / WordPress probes
|
|
191
|
+
enumerate.py # passive source adapters + optional DNS/subfinder + liveness
|
|
192
|
+
diff.py # normalized cross-run comparison
|
|
193
|
+
mcp_server.py # optional stdio / Streamable HTTP server
|
|
194
|
+
analyze.py # signals -> OPSEC findings + cross-host aggregate
|
|
195
|
+
render.py # JSON writer + vault markdown
|
|
196
|
+
cli.py # argparse + asyncio orchestration
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
Collectors take an injected `Fetcher` so they unit-test with a fake, no network.
|
|
200
|
+
Design notes: [`docs/design.md`](docs/design.md).
|
|
201
|
+
|
|
202
|
+
### Adding a scan module
|
|
203
|
+
|
|
204
|
+
Built-in host scans run as named modules over the same fetched response and
|
|
205
|
+
injected `Fetcher`. To add a collector in Python, register it before calling
|
|
206
|
+
`run_scan` in the same process:
|
|
207
|
+
|
|
208
|
+
```python
|
|
209
|
+
from teardown.scan_modules import ScanModule, register_module
|
|
210
|
+
|
|
211
|
+
async def collect(context):
|
|
212
|
+
# Use context.fetcher for any bounded, in-scope request.
|
|
213
|
+
return {"public_marker": "observed"}
|
|
214
|
+
|
|
215
|
+
register_module(ScanModule("example", collect))
|
|
216
|
+
```
|
|
217
|
+
|
|
218
|
+
The returned mapping appears at `hosts[].extensions.example` in the JSON.
|
|
219
|
+
Modules run independently, so one collector's error does not stop later ones.
|
|
220
|
+
For a built-in addition, add the module to `scan_modules.py`; use
|
|
221
|
+
`run_scan(..., modules=[...])` to select modules for one scan. Keep module
|
|
222
|
+
payloads deterministic and redact secret values before returning them.
|
|
223
|
+
|
|
224
|
+
## Tests
|
|
225
|
+
|
|
226
|
+
```bash
|
|
227
|
+
.venv/bin/python -m pytest -q # offline
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
## Notes & gotchas
|
|
231
|
+
|
|
232
|
+
- **crt.sh is flaky** — it intermittently returns a 404/HTML error page. The
|
|
233
|
+
enumerator retries (with backoff) and falls back to the bare-domain query form.
|
|
234
|
+
- **Source-map findings are first-party only.** A third-party CDN (Sentry,
|
|
235
|
+
jsdelivr) publishing maps for its own SDK is not the target's exposure, so those
|
|
236
|
+
are skipped; first-party maps are flagged MEDIUM (info disclosure, not a
|
|
237
|
+
credential leak).
|
|
238
|
+
- **Secrets are redacted at collection and output boundaries.** Third-party keys
|
|
239
|
+
keep only a prefix, and sensitive headers and URL query values are masked.
|
|
240
|
+
- Astro is matched from `/_astro/` JS and stylesheet URLs as well as its page
|
|
241
|
+
markers. Hosting labels follow current response evidence; a prior Cloudflare
|
|
242
|
+
result does not carry forward when a host now identifies Fastly or GitHub Pages.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Publishing
|
|
2
|
+
|
|
3
|
+
The PyPI distribution is `tech-teardown` and the console command is `teardown`. It builds with
|
|
4
|
+
hatchling and publishes from GitHub Actions through PyPI trusted publishing.
|
|
5
|
+
|
|
6
|
+
## 1. Bump the version
|
|
7
|
+
|
|
8
|
+
Edit `version` in `pyproject.toml` (semver). A PyPI version can only be uploaded
|
|
9
|
+
once — never reuse a number.
|
|
10
|
+
|
|
11
|
+
## 2. Build
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
uv build --out-dir dist-release
|
|
15
|
+
# -> dist-release/tech_teardown-<version>-py3-none-any.whl
|
|
16
|
+
# dist-release/tech_teardown-<version>.tar.gz
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
Sanity-check the built artifact before publishing:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
uvx --from dist-release/tech_teardown-<version>-py3-none-any.whl teardown example.com --single --json-only
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## 3. Configure PyPI once
|
|
26
|
+
|
|
27
|
+
For the first release, add a [pending trusted publisher](https://docs.pypi.org/trusted-publishers/creating-a-project-through-oidc/)
|
|
28
|
+
in your PyPI account with these exact fields:
|
|
29
|
+
|
|
30
|
+
| Field | Value |
|
|
31
|
+
|-------|-------|
|
|
32
|
+
| PyPI project | `tech-teardown` |
|
|
33
|
+
| GitHub owner | `ReyemTech` |
|
|
34
|
+
| Repository | `teardown` |
|
|
35
|
+
| Workflow filename | `release.yml` |
|
|
36
|
+
| Environment | leave blank |
|
|
37
|
+
|
|
38
|
+
PyPI converts the pending publisher to a normal publisher on first upload. No
|
|
39
|
+
long-lived token is needed in the repository or workflow.
|
|
40
|
+
|
|
41
|
+
## 4. Publish to PyPI
|
|
42
|
+
|
|
43
|
+
Tag the verified `main` commit. The tag starts the release workflow, which
|
|
44
|
+
reruns tests, builds both distributions, and publishes with GitHub OIDC:
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
git tag -a v<version> -m v<version>
|
|
48
|
+
git push origin v<version>
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## 5. After publish
|
|
52
|
+
|
|
53
|
+
Once live, anyone can run it with no install:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
uvx --from tech-teardown teardown example.com --single
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Notes
|
|
60
|
+
|
|
61
|
+
- `requires-python = ">=3.10"`.
|
|
62
|
+
- Runtime deps: `httpx` and `tldextract` (using its bundled public suffix list).
|
|
63
|
+
`dnspython` is an optional extra (`tech-teardown[active-dns]`),
|
|
64
|
+
only needed for `--active-dns`. FastMCP is an optional extra (`tech-teardown[mcp]`).
|
|
65
|
+
- CI runs tests on Python 3.10–3.14, builds distributions, and checks the
|
|
66
|
+
installed wheel's CLI. `.github/workflows/release.yml` publishes only on
|
|
67
|
+
version tags after its build and test job succeeds.
|