osintkit 0.4.0__tar.gz → 0.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {osintkit-0.4.0 → osintkit-0.6.0}/CHANGELOG.md +22 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/PKG-INFO +23 -3
- {osintkit-0.4.0 → osintkit-0.6.0}/README.md +22 -2
- {osintkit-0.4.0 → osintkit-0.6.0}/docs/REFERENCE.md +23 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/pyproject.toml +1 -1
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/__init__.py +1 -1
- osintkit-0.6.0/src/osintkit/catalogue.py +94 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/cli.py +46 -2
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/__init__.py +1 -0
- osintkit-0.6.0/src/osintkit/collectors/infostealer.py +71 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/config.py +1 -1
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/llm.py +23 -4
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/normalise.py +22 -8
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/offline.py +25 -5
- osintkit-0.6.0/src/osintkit/passwords.py +66 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/pipeline.py +37 -3
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/synthesis.py +1 -1
- osintkit-0.6.0/tests/test_catalogue.py +70 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_cli.py +25 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_e2e.py +1 -1
- osintkit-0.6.0/tests/test_infostealer.py +64 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_llm.py +33 -1
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_normalise.py +9 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_offline.py +6 -0
- osintkit-0.6.0/tests/test_passwords.py +58 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_pipeline.py +52 -1
- {osintkit-0.4.0 → osintkit-0.6.0}/.editorconfig +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/CODEOWNERS +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/dependabot.yml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/pull_request_template.md +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/workflows/ci.yml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/workflows/codeql.yml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/workflows/publish.yml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/workflows/release.yml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.github/workflows/smoke.yml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.gitignore +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.mailmap +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/.pre-commit-config.yaml +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/CODE_OF_CONDUCT.md +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/CONTRIBUTING.md +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/LICENSE +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/SECURITY.md +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/docs/architecture.md +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/docs/demo.svg +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/docs/sample-output.md +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/docs/social-preview.png +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/examples/README.md +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/examples/targets.csv +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/accounts.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/base.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/breaches.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/ct.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/dns.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/github.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/pastes.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/ports.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/subdomains.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/collectors/whois_lookup.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/diff.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/doctor.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/errors.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/http.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/logging_store.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/models.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/py.typed +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/render.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/retention.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/targets.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/src/osintkit/userconfig.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/conftest.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/fixtures/model_reply.json +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_accounts.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_breaches.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_collectors.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_config.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_ct.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_diff.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_doctor.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_http.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_pastes.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_render.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_retention.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_synthesis.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_targets.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_userconfig.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/tests/test_whois.py +0 -0
- {osintkit-0.4.0 → osintkit-0.6.0}/uv.lock +0 -0
|
@@ -6,6 +6,28 @@ All notable changes are recorded here. The format follows
|
|
|
6
6
|
|
|
7
7
|
## Unreleased
|
|
8
8
|
|
|
9
|
+
## 0.6.0 — 2026-11-02
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `infostealer` collector for Hudson Rock stealer logs across email, username, and domain
|
|
14
|
+
- any stealer infection forces a high risk floor and a rotate-everything recommendation
|
|
15
|
+
|
|
16
|
+
## 0.5.0 — 2026-11-02
|
|
17
|
+
|
|
18
|
+
### Added
|
|
19
|
+
|
|
20
|
+
- breach rows are enriched with date, exposed fields, domain, and record count from the keyless
|
|
21
|
+
HIBP and XposedOrNot catalogues
|
|
22
|
+
- `osintkit password` checks Pwned Passwords by SHA-1 prefix; no key, no plaintext sent
|
|
23
|
+
- email lookups also collect the address domain's DNS, mail posture, and WHOIS, marked as derived
|
|
24
|
+
- a failed synthesis now falls back to a locally built report instead of an empty one
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
|
|
28
|
+
- large prompts came back empty on reasoning models: token truncation is detected and retried,
|
|
29
|
+
and the prompt carries at most 15 breach rows
|
|
30
|
+
|
|
9
31
|
## 0.4.0 — 2026-11-02
|
|
10
32
|
|
|
11
33
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: osintkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.6.0
|
|
4
4
|
Summary: Public-data OSINT aggregator for emails, usernames, domains, and companies
|
|
5
5
|
Project-URL: Homepage, https://github.com/sjdsjadjajdjsa/osintkit
|
|
6
6
|
Project-URL: Issues, https://github.com/sjdsjadjajdjsa/osintkit/issues
|
|
@@ -103,6 +103,7 @@ Optional keys: `HIBP_API_KEY` turns on Have I Been Pwned, `GITHUB_TOKEN` lifts G
|
|
|
103
103
|
| `lookup` | One target, one report (`--email`, `--username`, `--domain`, `--company`) |
|
|
104
104
|
| `batch` | A CSV of targets, run concurrently, reported in order |
|
|
105
105
|
| `diff` | What changed between two archived runs |
|
|
106
|
+
| `password` | Is a password in Pwned Passwords (free, no key, hash-prefix only) |
|
|
106
107
|
| `history` | Past runs, replayable by id |
|
|
107
108
|
| `doctor` | Key, endpoint, and directory checks (`--deep` verifies auth) |
|
|
108
109
|
| `collectors` | Every collector and what it touches |
|
|
@@ -132,8 +133,13 @@ target -> type detection -> collectors (parallel, cached, retried)
|
|
|
132
133
|
Collectors degrade to a logged gap, never a dead run; a report is produced even when sources or
|
|
133
134
|
the model fail.
|
|
134
135
|
|
|
135
|
-
- **Merged breaches.**
|
|
136
|
-
source and the earliest date
|
|
136
|
+
- **Merged, enriched breaches.** Records collapse into one row per incident, keeping every
|
|
137
|
+
source and the earliest date, then pick up dates and exposed-field lists from the public HIBP
|
|
138
|
+
and XposedOrNot catalogues. Those catalogues are free; only HIBP *account* checks need a key.
|
|
139
|
+
- **Email domains follow the address.** An email lookup also checks the domain's DNS, mail
|
|
140
|
+
posture, and WHOIS, marked as derived.
|
|
141
|
+
- **Info-stealer logs.** Hudson Rock reports machines infected by stealers that saved
|
|
142
|
+
credentials, with dates and hostnames; one infection forces a high risk floor.
|
|
137
143
|
- **Risk floor.** Breach count and credential types set a floor the model may exceed but never
|
|
138
144
|
undercut.
|
|
139
145
|
- **Cached everything.** 24h default, shorter for volatile sources; repeat prompts cost nothing.
|
|
@@ -148,6 +154,20 @@ connect probes, GitHub (profile, commits, user search, org repos), psbdmp with a
|
|
|
148
154
|
grep.app, Hacker News, Stack Exchange, and username presence. The full table is in
|
|
149
155
|
[docs/REFERENCE.md](docs/REFERENCE.md#collectors) or via `osintkit collectors`.
|
|
150
156
|
|
|
157
|
+
## Breach depth
|
|
158
|
+
|
|
159
|
+
Breach name lists are free; breach *metadata* is what makes them useful. Every row is joined
|
|
160
|
+
against the public HIBP and XposedOrNot catalogues for date, exposed fields, and record count, so
|
|
161
|
+
the report can say "LinkedIn, 2021-06, emails and password hashes" instead of a bare name. Add a
|
|
162
|
+
`HIBP_API_KEY` (about $4/month) to get per-account membership with the same metadata, plus pastes.
|
|
163
|
+
|
|
164
|
+
Stealer logs matter more than most breaches: the credentials were live on an infected machine,
|
|
165
|
+
not just leaked years ago. They are queried for every target type, keyless.
|
|
166
|
+
|
|
167
|
+
Passwords are checked separately with `osintkit password`: Pwned Passwords covers hundreds of
|
|
168
|
+
millions of leaked passwords, needs no key, and only the first five SHA-1 characters leave the
|
|
169
|
+
machine.
|
|
170
|
+
|
|
151
171
|
## Safety and legality
|
|
152
172
|
|
|
153
173
|
- Public data only. No logins, no scraping behind auth, no paid brokers.
|
|
@@ -71,6 +71,7 @@ Optional keys: `HIBP_API_KEY` turns on Have I Been Pwned, `GITHUB_TOKEN` lifts G
|
|
|
71
71
|
| `lookup` | One target, one report (`--email`, `--username`, `--domain`, `--company`) |
|
|
72
72
|
| `batch` | A CSV of targets, run concurrently, reported in order |
|
|
73
73
|
| `diff` | What changed between two archived runs |
|
|
74
|
+
| `password` | Is a password in Pwned Passwords (free, no key, hash-prefix only) |
|
|
74
75
|
| `history` | Past runs, replayable by id |
|
|
75
76
|
| `doctor` | Key, endpoint, and directory checks (`--deep` verifies auth) |
|
|
76
77
|
| `collectors` | Every collector and what it touches |
|
|
@@ -100,8 +101,13 @@ target -> type detection -> collectors (parallel, cached, retried)
|
|
|
100
101
|
Collectors degrade to a logged gap, never a dead run; a report is produced even when sources or
|
|
101
102
|
the model fail.
|
|
102
103
|
|
|
103
|
-
- **Merged breaches.**
|
|
104
|
-
source and the earliest date
|
|
104
|
+
- **Merged, enriched breaches.** Records collapse into one row per incident, keeping every
|
|
105
|
+
source and the earliest date, then pick up dates and exposed-field lists from the public HIBP
|
|
106
|
+
and XposedOrNot catalogues. Those catalogues are free; only HIBP *account* checks need a key.
|
|
107
|
+
- **Email domains follow the address.** An email lookup also checks the domain's DNS, mail
|
|
108
|
+
posture, and WHOIS, marked as derived.
|
|
109
|
+
- **Info-stealer logs.** Hudson Rock reports machines infected by stealers that saved
|
|
110
|
+
credentials, with dates and hostnames; one infection forces a high risk floor.
|
|
105
111
|
- **Risk floor.** Breach count and credential types set a floor the model may exceed but never
|
|
106
112
|
undercut.
|
|
107
113
|
- **Cached everything.** 24h default, shorter for volatile sources; repeat prompts cost nothing.
|
|
@@ -116,6 +122,20 @@ connect probes, GitHub (profile, commits, user search, org repos), psbdmp with a
|
|
|
116
122
|
grep.app, Hacker News, Stack Exchange, and username presence. The full table is in
|
|
117
123
|
[docs/REFERENCE.md](docs/REFERENCE.md#collectors) or via `osintkit collectors`.
|
|
118
124
|
|
|
125
|
+
## Breach depth
|
|
126
|
+
|
|
127
|
+
Breach name lists are free; breach *metadata* is what makes them useful. Every row is joined
|
|
128
|
+
against the public HIBP and XposedOrNot catalogues for date, exposed fields, and record count, so
|
|
129
|
+
the report can say "LinkedIn, 2021-06, emails and password hashes" instead of a bare name. Add a
|
|
130
|
+
`HIBP_API_KEY` (about $4/month) to get per-account membership with the same metadata, plus pastes.
|
|
131
|
+
|
|
132
|
+
Stealer logs matter more than most breaches: the credentials were live on an infected machine,
|
|
133
|
+
not just leaked years ago. They are queried for every target type, keyless.
|
|
134
|
+
|
|
135
|
+
Passwords are checked separately with `osintkit password`: Pwned Passwords covers hundreds of
|
|
136
|
+
millions of leaked passwords, needs no key, and only the first five SHA-1 characters leave the
|
|
137
|
+
machine.
|
|
138
|
+
|
|
119
139
|
## Safety and legality
|
|
120
140
|
|
|
121
141
|
- Public data only. No logins, no scraping behind auth, no paid brokers.
|
|
@@ -6,6 +6,7 @@ Everything the README leaves out: every flag, every collector, storage, and fail
|
|
|
6
6
|
|
|
7
7
|
| Collector | Targets | Source | Needs key |
|
|
8
8
|
| --- | --- | --- | --- |
|
|
9
|
+
| `infostealer` | email / username / domain | Hudson Rock stealer logs | no |
|
|
9
10
|
| `hibp_breaches`, `hibp_pastes` | email | Have I Been Pwned | yes |
|
|
10
11
|
| `xposed_or_not`, `leakcheck_public` | email | free public breach indexes | no |
|
|
11
12
|
| `gravatar` | email | Gravatar | no |
|
|
@@ -61,6 +62,28 @@ Explicit `--base-url` and `--model` beat both the provider and the prefix. Tempe
|
|
|
61
62
|
Per-command: `--json`, `--markdown`, `-o/--out`, `--raw`, `--offline`, `--dry-run`, `--only`, `--skip`,
|
|
62
63
|
`--ports`, `--fail-fast`, `-c/--concurrency`, `--limit`, `--show`, `--days`, `--clear`, `--deep`.
|
|
63
64
|
|
|
65
|
+
## Info-stealer logs
|
|
66
|
+
|
|
67
|
+
`infostealer` queries Hudson Rock's keyless endpoint for infections that saved credentials on a
|
|
68
|
+
compromised machine. It returns the infection count, dates, hostnames, IPs, antivirus state, and
|
|
69
|
+
masked logins and passwords. Any infection forces a `high` risk floor; masked values are shown as
|
|
70
|
+
the source provides them and are not usable as credentials.
|
|
71
|
+
|
|
72
|
+
The address goes to Hudson Rock; nothing else does.
|
|
73
|
+
|
|
74
|
+
## Breach catalogue enrichment
|
|
75
|
+
|
|
76
|
+
Breach names from free indexes are joined against `haveibeenpwned.com/api/v3/breaches` (keyless
|
|
77
|
+
catalogue) with an XposedOrNot catalogue fallback, cached a week, to add dates, exposed fields,
|
|
78
|
+
domains, and record counts. At most 15 enriched rows go into the prompt; the true count stays in
|
|
79
|
+
`signals`.
|
|
80
|
+
|
|
81
|
+
## Password checks
|
|
82
|
+
|
|
83
|
+
`osintkit password [VALUE]` hashes locally, sends only the five-character SHA-1 prefix to
|
|
84
|
+
`api.pwnedpasswords.com`, and reports the count and a commonness rating. Exit code `1` means
|
|
85
|
+
pwned. `--json` for machines. Nothing about the password beyond the prefix leaves the machine.
|
|
86
|
+
|
|
64
87
|
## Configuration file
|
|
65
88
|
|
|
66
89
|
`osintkit init` writes `~/.config/osintkit/config.toml` (or `$XDG_CONFIG_HOME`) with mode 600 in
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Public breach catalogues used to enrich bare breach names."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
from .errors import OsintKitError
|
|
9
|
+
from .http import HttpClient, cache_hit, cache_put
|
|
10
|
+
|
|
11
|
+
NAMESPACE = "catalogue"
|
|
12
|
+
TTL = 7 * 24 * 3600
|
|
13
|
+
KEY_RE = re.compile(r"[^a-z0-9]+")
|
|
14
|
+
|
|
15
|
+
HIBP = "https://haveibeenpwned.com/api/v3/breaches"
|
|
16
|
+
XPOSED = "https://api.xposedornot.com/v1/breaches"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _key(name: str) -> str:
|
|
20
|
+
return KEY_RE.sub("", name.lower())
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _from_hibp(rows: Any) -> dict[str, dict[str, Any]]:
|
|
24
|
+
if not isinstance(rows, list):
|
|
25
|
+
return {}
|
|
26
|
+
out: dict[str, dict[str, Any]] = {}
|
|
27
|
+
for row in rows:
|
|
28
|
+
if not isinstance(row, dict):
|
|
29
|
+
continue
|
|
30
|
+
name = str(row.get("Name") or row.get("Title") or "").strip()
|
|
31
|
+
if not name:
|
|
32
|
+
continue
|
|
33
|
+
out[_key(name)] = {
|
|
34
|
+
"name": name,
|
|
35
|
+
"date": row.get("BreachDate"),
|
|
36
|
+
"data_types": [str(kind) for kind in row.get("DataClasses") or []],
|
|
37
|
+
"domain": row.get("Domain"),
|
|
38
|
+
"pwn_count": row.get("PwnCount"),
|
|
39
|
+
}
|
|
40
|
+
return out
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _from_xposed(body: Any) -> dict[str, dict[str, Any]]:
|
|
44
|
+
rows = body.get("exposedBreaches") if isinstance(body, dict) else None
|
|
45
|
+
if not isinstance(rows, list):
|
|
46
|
+
return {}
|
|
47
|
+
out: dict[str, dict[str, Any]] = {}
|
|
48
|
+
for row in rows:
|
|
49
|
+
if not isinstance(row, dict):
|
|
50
|
+
continue
|
|
51
|
+
name = str(row.get("breachID") or "").strip()
|
|
52
|
+
if not name:
|
|
53
|
+
continue
|
|
54
|
+
out[_key(name)] = {
|
|
55
|
+
"name": name,
|
|
56
|
+
"date": (row.get("breachedDate") or "")[:10] or None,
|
|
57
|
+
"data_types": [str(kind) for kind in row.get("exposedData") or []],
|
|
58
|
+
"domain": row.get("domain"),
|
|
59
|
+
"pwn_count": row.get("exposedRecords"),
|
|
60
|
+
}
|
|
61
|
+
return out
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
async def _download(http: HttpClient) -> dict[str, dict[str, Any]]:
|
|
65
|
+
try:
|
|
66
|
+
response = await http.get(HIBP, namespace=NAMESPACE, accept="application/json")
|
|
67
|
+
if response.status_code == 200:
|
|
68
|
+
data = _from_hibp(response.json())
|
|
69
|
+
if data:
|
|
70
|
+
return data
|
|
71
|
+
except (OsintKitError, ValueError):
|
|
72
|
+
pass
|
|
73
|
+
|
|
74
|
+
try:
|
|
75
|
+
response = await http.get(XPOSED, namespace=NAMESPACE, accept="application/json")
|
|
76
|
+
except OsintKitError:
|
|
77
|
+
return {}
|
|
78
|
+
if response.status_code != 200:
|
|
79
|
+
return {}
|
|
80
|
+
try:
|
|
81
|
+
return _from_xposed(response.json())
|
|
82
|
+
except ValueError:
|
|
83
|
+
return {}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
async def fetch(http: HttpClient) -> dict[str, dict[str, Any]]:
|
|
87
|
+
"""Return breach metadata keyed by normalised name, cached for a week."""
|
|
88
|
+
cached = cache_hit(http.settings, NAMESPACE, "index", TTL)
|
|
89
|
+
if cached is not None:
|
|
90
|
+
return cached
|
|
91
|
+
data = await _download(http)
|
|
92
|
+
if data:
|
|
93
|
+
cache_put(http.settings, NAMESPACE, "index", data)
|
|
94
|
+
return data
|
|
@@ -18,7 +18,7 @@ from rich.console import Console
|
|
|
18
18
|
from rich.panel import Panel
|
|
19
19
|
from rich.table import Table
|
|
20
20
|
|
|
21
|
-
from . import __version__, diff, doctor, pipeline, render, retention, userconfig
|
|
21
|
+
from . import __version__, diff, doctor, passwords, pipeline, render, retention, userconfig
|
|
22
22
|
from .collectors import REGISTRY
|
|
23
23
|
from .config import (
|
|
24
24
|
DEFAULT_RESOLVERS,
|
|
@@ -74,7 +74,7 @@ def root(
|
|
|
74
74
|
model: str | None = typer.Option(None, "--model", help="Model name (overrides the provider default)."),
|
|
75
75
|
api_key: str | None = typer.Option(None, "--api-key", help="API key (or set API_KEY)."),
|
|
76
76
|
temperature: float = typer.Option(0.3, "--temperature", min=0.0, max=2.0),
|
|
77
|
-
max_tokens: int = typer.Option(
|
|
77
|
+
max_tokens: int = typer.Option(4000, "--max-tokens", min=256, max=16000),
|
|
78
78
|
timeout: float = typer.Option(30.0, "--timeout", help="HTTP timeout in seconds."),
|
|
79
79
|
max_connections: int = typer.Option(16, "--max-connections", min=1, max=64),
|
|
80
80
|
max_attempts: int = typer.Option(3, "--max-attempts", min=1, max=6),
|
|
@@ -627,6 +627,50 @@ def init(
|
|
|
627
627
|
out.print("[dim]try: osintkit lookup --domain example.com[/dim]")
|
|
628
628
|
|
|
629
629
|
|
|
630
|
+
@app.command()
|
|
631
|
+
def password(
|
|
632
|
+
value: str | None = typer.Argument(None, help="Password to check; prompt if omitted."),
|
|
633
|
+
json_out: bool = typer.Option(False, "--json", help="JSON output."),
|
|
634
|
+
) -> None:
|
|
635
|
+
"""Check a password against Pwned Passwords, without sending it."""
|
|
636
|
+
out = _console()
|
|
637
|
+
secret = value
|
|
638
|
+
if not secret:
|
|
639
|
+
if not sys.stdin.isatty():
|
|
640
|
+
secret = sys.stdin.readline().rstrip("\n")
|
|
641
|
+
if not secret:
|
|
642
|
+
secret = typer.prompt("Password", hide_input=True)
|
|
643
|
+
if not secret:
|
|
644
|
+
out.print("[red]no password given[/red]")
|
|
645
|
+
raise typer.Exit(2)
|
|
646
|
+
|
|
647
|
+
settings: Settings = STATE["settings"] # type: ignore[assignment]
|
|
648
|
+
|
|
649
|
+
async def run() -> dict[str, object]:
|
|
650
|
+
from .http import HttpClient
|
|
651
|
+
|
|
652
|
+
async with HttpClient(settings) as http:
|
|
653
|
+
return await passwords.check(http, secret)
|
|
654
|
+
|
|
655
|
+
try:
|
|
656
|
+
result = asyncio.run(run())
|
|
657
|
+
except OsintKitError as exc:
|
|
658
|
+
out.print(f"[red]{exc}[/red]")
|
|
659
|
+
raise typer.Exit(1) from exc
|
|
660
|
+
|
|
661
|
+
if json_out:
|
|
662
|
+
print(json.dumps(result, indent=2))
|
|
663
|
+
return
|
|
664
|
+
if result["pwned"]:
|
|
665
|
+
out.print(
|
|
666
|
+
f"[red]pwned[/red] {result['count']:,} times "
|
|
667
|
+
f"({result['rating']}); still only the first 5 SHA-1 characters left this machine"
|
|
668
|
+
)
|
|
669
|
+
raise typer.Exit(1)
|
|
670
|
+
out.print(f"[green]not seen[/green] in Pwned Passwords (checked by hash prefix {result['prefix']})")
|
|
671
|
+
out.print("[yellow]absence is not proof of strength; length and uniqueness still matter[/yellow]")
|
|
672
|
+
|
|
673
|
+
|
|
630
674
|
@app.command("config")
|
|
631
675
|
def config_cmd(
|
|
632
676
|
reveal: bool = typer.Option(False, "--reveal", help="Show the key in full."),
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""Info-stealer log exposure via Hudson Rock's keyless endpoint."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from urllib.parse import quote
|
|
6
|
+
|
|
7
|
+
from .base import Context, collector
|
|
8
|
+
|
|
9
|
+
BASE = "https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by"
|
|
10
|
+
MAX_STEALERS = 5
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _clean(value: object) -> str | None:
|
|
14
|
+
text = str(value or "").strip()
|
|
15
|
+
if not text or text.lower() == "not found":
|
|
16
|
+
return None
|
|
17
|
+
return text[:120]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _summary(body: object) -> dict:
|
|
21
|
+
if not isinstance(body, dict):
|
|
22
|
+
return {"infected": False, "infections": 0, "stealers": []}
|
|
23
|
+
raw = body.get("stealers") or []
|
|
24
|
+
stealers = []
|
|
25
|
+
for row in raw[:MAX_STEALERS]:
|
|
26
|
+
if not isinstance(row, dict):
|
|
27
|
+
continue
|
|
28
|
+
stealers.append(
|
|
29
|
+
{
|
|
30
|
+
"date": (row.get("date_compromised") or "")[:10] or None,
|
|
31
|
+
"computer": _clean(row.get("computer_name")),
|
|
32
|
+
"os": _clean(row.get("operating_system")),
|
|
33
|
+
"ip": _clean(row.get("ip")),
|
|
34
|
+
"antiviruses": [av for av in row.get("antiviruses") or [] if av != "None"][:5],
|
|
35
|
+
"logins": [str(item) for item in (row.get("top_logins") or [])[:5]],
|
|
36
|
+
"passwords": [str(item) for item in (row.get("top_passwords") or [])[:5]],
|
|
37
|
+
}
|
|
38
|
+
)
|
|
39
|
+
return {
|
|
40
|
+
"infected": bool(stealers),
|
|
41
|
+
"infections": len(raw),
|
|
42
|
+
"corporate_services": body.get("total_corporate_services"),
|
|
43
|
+
"user_services": body.get("total_user_services"),
|
|
44
|
+
"stealers": stealers,
|
|
45
|
+
"source": "hudsonrock",
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
async def _lookup(ctx: Context, kind: str) -> dict:
|
|
50
|
+
response = await ctx.http.get(
|
|
51
|
+
f"{BASE}-{kind}",
|
|
52
|
+
namespace="hudsonrock",
|
|
53
|
+
params={kind: ctx.target},
|
|
54
|
+
empty_on=(404, 429),
|
|
55
|
+
timeout=ctx.http_timeout or 45,
|
|
56
|
+
)
|
|
57
|
+
if response.status_code != 200:
|
|
58
|
+
return {"infected": False, "infections": 0, "stealers": [], "note": f"HTTP {response.status_code}"}
|
|
59
|
+
try:
|
|
60
|
+
return _summary(response.json())
|
|
61
|
+
except ValueError:
|
|
62
|
+
return {"infected": False, "infections": 0, "stealers": [], "note": "non-JSON"}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@collector("infostealer", ("email", "username", "domain"), "Info-stealer log infections", order=5, ttl=43200)
|
|
66
|
+
async def infostealer(ctx: Context) -> dict:
|
|
67
|
+
kind = {"email": "email", "username": "username", "domain": "domain"}[ctx.target_type]
|
|
68
|
+
result = await _lookup(ctx, kind)
|
|
69
|
+
result["target_kind"] = kind
|
|
70
|
+
result["checked"] = quote(ctx.target)
|
|
71
|
+
return result
|
|
@@ -110,7 +110,7 @@ class Settings:
|
|
|
110
110
|
model: str = DEFAULT_MODEL
|
|
111
111
|
api_key: str | None = None
|
|
112
112
|
temperature: float = DEFAULT_TEMPERATURE
|
|
113
|
-
max_tokens: int =
|
|
113
|
+
max_tokens: int = 4000
|
|
114
114
|
timeout: float = DEFAULT_TIMEOUT
|
|
115
115
|
max_connections: int = 16
|
|
116
116
|
max_attempts: int = DEFAULT_MAX_ATTEMPTS
|
|
@@ -34,7 +34,7 @@ class LLMClient:
|
|
|
34
34
|
messages: Sequence[Mapping[str, str]],
|
|
35
35
|
*,
|
|
36
36
|
temperature: float | None = None,
|
|
37
|
-
max_tokens: int =
|
|
37
|
+
max_tokens: int = 4000,
|
|
38
38
|
response_format: Mapping[str, Any] | None = None,
|
|
39
39
|
) -> str:
|
|
40
40
|
api_key = self.settings.resolved_api_key()
|
|
@@ -51,6 +51,15 @@ class LLMClient:
|
|
|
51
51
|
if response_format:
|
|
52
52
|
payload["response_format"] = dict(response_format)
|
|
53
53
|
|
|
54
|
+
content, finish = await self._post(payload, api_key)
|
|
55
|
+
if finish == "length" and not content and max_tokens < 8000:
|
|
56
|
+
payload["max_tokens"] = min(8000, max_tokens * 2)
|
|
57
|
+
content, finish = await self._post(payload, api_key)
|
|
58
|
+
if finish == "length" and not content:
|
|
59
|
+
raise SynthesisError("model ran out of tokens before answering")
|
|
60
|
+
return content
|
|
61
|
+
|
|
62
|
+
async def _post(self, payload: dict[str, Any], api_key: str | None) -> tuple[str, str | None]:
|
|
54
63
|
try:
|
|
55
64
|
response = await self._http.post_json(
|
|
56
65
|
self.endpoint(),
|
|
@@ -69,7 +78,17 @@ class LLMClient:
|
|
|
69
78
|
body = response.json()
|
|
70
79
|
except ValueError as exc:
|
|
71
80
|
raise SynthesisError("Model endpoint returned non-JSON") from exc
|
|
72
|
-
return extract_content(body)
|
|
81
|
+
return extract_content(body), extract_finish(body)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def extract_finish(body: Any) -> str | None:
|
|
85
|
+
if not isinstance(body, dict):
|
|
86
|
+
return None
|
|
87
|
+
choices = body.get("choices")
|
|
88
|
+
if not isinstance(choices, list) or not choices:
|
|
89
|
+
return None
|
|
90
|
+
finish = choices[0].get("finish_reason")
|
|
91
|
+
return finish if isinstance(finish, str) else None
|
|
73
92
|
|
|
74
93
|
|
|
75
94
|
def extract_content(body: Any) -> str:
|
|
@@ -86,6 +105,6 @@ def extract_content(body: Any) -> str:
|
|
|
86
105
|
if isinstance(content, list):
|
|
87
106
|
parts = [c.get("text", "") for c in content if isinstance(c, dict)]
|
|
88
107
|
content = "".join(parts)
|
|
89
|
-
if not isinstance(content, str)
|
|
90
|
-
|
|
108
|
+
if not isinstance(content, str):
|
|
109
|
+
return ""
|
|
91
110
|
return content
|
|
@@ -11,9 +11,10 @@ BREACH_SOURCES = frozenset({"hibp_breaches", "xposed_or_not", "leakcheck_public"
|
|
|
11
11
|
KEY_RE = re.compile(r"[^a-z0-9]+")
|
|
12
12
|
CREDENTIAL_RE = re.compile(r"pass|hash|credential|secret|token|api[ _-]?key", re.I)
|
|
13
13
|
SEVERITY = ("low", "medium", "high")
|
|
14
|
+
MAX_PROMPT_BREACHES = 15
|
|
14
15
|
|
|
15
16
|
|
|
16
|
-
def signals_for(merged: list[dict[str, Any]]) -> dict[str, Any]:
|
|
17
|
+
def signals_for(merged: list[dict[str, Any]], infections: int = 0) -> dict[str, Any]:
|
|
17
18
|
"""Compute risk signals the model can check itself against."""
|
|
18
19
|
credential = [
|
|
19
20
|
entry
|
|
@@ -21,10 +22,12 @@ def signals_for(merged: list[dict[str, Any]]) -> dict[str, Any]:
|
|
|
21
22
|
if any(CREDENTIAL_RE.search(str(kind)) for kind in entry.get("data_types", []))
|
|
22
23
|
]
|
|
23
24
|
count = len(merged)
|
|
25
|
+
floor = "high" if infections else _floor(count, len(credential))
|
|
24
26
|
return {
|
|
25
27
|
"breach_count": count,
|
|
26
28
|
"credential_bearing": len(credential),
|
|
27
|
-
"
|
|
29
|
+
"stealer_infections": infections,
|
|
30
|
+
"risk_floor": floor,
|
|
28
31
|
}
|
|
29
32
|
|
|
30
33
|
|
|
@@ -66,7 +69,10 @@ def _records(result: CollectorResult) -> list[dict[str, Any]]:
|
|
|
66
69
|
return []
|
|
67
70
|
|
|
68
71
|
|
|
69
|
-
def merge_breaches(
|
|
72
|
+
def merge_breaches(
|
|
73
|
+
results: list[CollectorResult],
|
|
74
|
+
catalogue: dict[str, dict[str, Any]] | None = None,
|
|
75
|
+
) -> tuple[list[CollectorResult], int]:
|
|
70
76
|
"""Merge duplicate breach names across sources.
|
|
71
77
|
|
|
72
78
|
Returns the results to prompt with (breach sources replaced by one index)
|
|
@@ -88,18 +94,23 @@ def merge_breaches(results: list[CollectorResult]) -> tuple[list[CollectorResult
|
|
|
88
94
|
if not name:
|
|
89
95
|
continue
|
|
90
96
|
key = _key(name)
|
|
97
|
+
known = (catalogue or {}).get(key) or {}
|
|
98
|
+
types = set(_strings(entry.get("data_types"))) or set(known.get("data_types") or [])
|
|
99
|
+
date = entry.get("date") or known.get("date")
|
|
91
100
|
existing = merged.get(key)
|
|
92
101
|
if existing is None:
|
|
93
102
|
merged[key] = {
|
|
94
103
|
"name": name,
|
|
95
|
-
"date":
|
|
96
|
-
"data_types":
|
|
104
|
+
"date": date,
|
|
105
|
+
"data_types": types,
|
|
97
106
|
"sources": {source},
|
|
107
|
+
"domain": known.get("domain"),
|
|
108
|
+
"pwn_count": known.get("pwn_count"),
|
|
98
109
|
}
|
|
99
110
|
continue
|
|
100
111
|
duplicates += 1
|
|
101
|
-
if not existing["date"] and
|
|
102
|
-
existing["date"] =
|
|
112
|
+
if not existing["date"] and date:
|
|
113
|
+
existing["date"] = date
|
|
103
114
|
existing["data_types"].update(_strings(entry.get("data_types")))
|
|
104
115
|
existing["sources"].add(source)
|
|
105
116
|
|
|
@@ -110,13 +121,16 @@ def merge_breaches(results: list[CollectorResult]) -> tuple[list[CollectorResult
|
|
|
110
121
|
{
|
|
111
122
|
"name": item["name"],
|
|
112
123
|
"date": item["date"],
|
|
113
|
-
"data_types": sorted(item["data_types"]),
|
|
124
|
+
"data_types": sorted(item["data_types"])[:8],
|
|
114
125
|
"sources": sorted(item["sources"]),
|
|
115
126
|
"source": ", ".join(sorted(item["sources"])),
|
|
127
|
+
"domain": item.get("domain"),
|
|
128
|
+
"pwn_count": item.get("pwn_count"),
|
|
116
129
|
}
|
|
117
130
|
for item in merged.values()
|
|
118
131
|
]
|
|
119
132
|
rendered.sort(key=lambda item: (-len(item["data_types"]), item["name"].lower()))
|
|
133
|
+
rendered = rendered[:MAX_PROMPT_BREACHES]
|
|
120
134
|
|
|
121
135
|
index = CollectorResult(
|
|
122
136
|
collector="breach_index",
|
|
@@ -91,6 +91,15 @@ def _infrastructure(results: list[CollectorResult], report_notes: list[str]) ->
|
|
|
91
91
|
if isinstance(wildcard, dict) and wildcard.get("wildcard"):
|
|
92
92
|
notes.append("Wildcard DNS is enabled, so unknown labels resolve")
|
|
93
93
|
|
|
94
|
+
stealer = _data(results, "infostealer")
|
|
95
|
+
if isinstance(stealer, dict) and stealer.get("infections"):
|
|
96
|
+
first = (stealer.get("stealers") or [{}])[0]
|
|
97
|
+
notes.append(
|
|
98
|
+
f"{stealer['infections']} info-stealer infection(s)"
|
|
99
|
+
+ (f", latest {first.get('date')}" if first.get("date") else "")
|
|
100
|
+
+ (f" on {first.get('computer')}" if first.get("computer") else "")
|
|
101
|
+
)
|
|
102
|
+
|
|
94
103
|
whois = _data(results, "whois_registration")
|
|
95
104
|
if isinstance(whois, dict) and whois.get("registrar"):
|
|
96
105
|
parts = [f"registrar {whois['registrar']}"]
|
|
@@ -117,9 +126,16 @@ def _infrastructure(results: list[CollectorResult], report_notes: list[str]) ->
|
|
|
117
126
|
|
|
118
127
|
|
|
119
128
|
def _recommendations(
|
|
120
|
-
creds: int,
|
|
129
|
+
creds: int,
|
|
130
|
+
mail: dict | None,
|
|
131
|
+
wildcard: bool,
|
|
132
|
+
ports: list[int],
|
|
133
|
+
breach_count: int,
|
|
134
|
+
infections: int = 0,
|
|
121
135
|
) -> list[str]:
|
|
122
136
|
recs: list[str] = []
|
|
137
|
+
if infections:
|
|
138
|
+
recs.append("Change every password saved on the infected machine, then sign out all sessions")
|
|
123
139
|
if creds:
|
|
124
140
|
recs.append("Rotate passwords for the affected services and enable MFA where offered")
|
|
125
141
|
elif breach_count:
|
|
@@ -142,7 +158,9 @@ def _recommendations(
|
|
|
142
158
|
return recs
|
|
143
159
|
|
|
144
160
|
|
|
145
|
-
def build(
|
|
161
|
+
def build(
|
|
162
|
+
results: list[CollectorResult], report_notes: list[str], target: str, *, narrated: bool = False
|
|
163
|
+
) -> Analysis:
|
|
146
164
|
breaches_raw = next((r.data for r in results if r.collector == "breach_index" and r.has_data), [])
|
|
147
165
|
signals = signals_for(breaches_raw)
|
|
148
166
|
accounts = _found_accounts(results)
|
|
@@ -158,8 +176,8 @@ def build(results: list[CollectorResult], report_notes: list[str], target: str)
|
|
|
158
176
|
failed = [r.collector for r in results if not r.ok and not r.skipped]
|
|
159
177
|
|
|
160
178
|
summary_parts = [
|
|
161
|
-
f"Collected public data for {target} from {len(collected)} of "
|
|
162
|
-
|
|
179
|
+
f"Collected public data for {target} from {len(collected)} of {len(results)} sources"
|
|
180
|
+
+ (", with no model call." if not narrated else "; the model returned no summary.")
|
|
163
181
|
]
|
|
164
182
|
if signals["breach_count"]:
|
|
165
183
|
summary_parts.append(
|
|
@@ -174,7 +192,8 @@ def build(results: list[CollectorResult], report_notes: list[str], target: str)
|
|
|
174
192
|
summary_parts.append(f"Failed sources: {', '.join(failed)}.")
|
|
175
193
|
if skipped:
|
|
176
194
|
summary_parts.append(f"Skipped sources: {', '.join(skipped)}.")
|
|
177
|
-
|
|
195
|
+
if not narrated:
|
|
196
|
+
summary_parts.append("Risk is computed from counts and policy checks, not narrated.")
|
|
178
197
|
|
|
179
198
|
risk = apply_floor("low", signals["risk_floor"])
|
|
180
199
|
if risky or (wildcard and isinstance(mail, dict) and not mail.get("dmarc_present")):
|
|
@@ -201,5 +220,6 @@ def build(results: list[CollectorResult], report_notes: list[str], target: str)
|
|
|
201
220
|
wildcard,
|
|
202
221
|
risky,
|
|
203
222
|
signals["breach_count"],
|
|
223
|
+
int((_data(results, "infostealer") or {}).get("infections") or 0),
|
|
204
224
|
),
|
|
205
225
|
)
|