osintkit 0.6.0__tar.gz → 0.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {osintkit-0.6.0 → osintkit-0.7.0}/CHANGELOG.md +8 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/PKG-INFO +7 -3
- {osintkit-0.6.0 → osintkit-0.7.0}/README.md +6 -2
- {osintkit-0.6.0 → osintkit-0.7.0}/docs/REFERENCE.md +10 -2
- {osintkit-0.6.0 → osintkit-0.7.0}/pyproject.toml +1 -1
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/__init__.py +1 -1
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/cli.py +13 -1
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/__init__.py +1 -0
- osintkit-0.7.0/src/osintkit/collectors/xposed_analytics.py +73 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/config.py +2 -1
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/doctor.py +2 -2
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/normalise.py +1 -1
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/pipeline.py +4 -1
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/userconfig.py +2 -2
- osintkit-0.7.0/src/osintkit/verify.py +77 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_cli.py +45 -0
- osintkit-0.7.0/tests/test_verify.py +54 -0
- osintkit-0.7.0/tests/test_xposed_analytics.py +61 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.editorconfig +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/CODEOWNERS +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/ISSUE_TEMPLATE/bug_report.yml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/ISSUE_TEMPLATE/config.yml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/ISSUE_TEMPLATE/feature_request.yml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/dependabot.yml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/pull_request_template.md +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/workflows/ci.yml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/workflows/codeql.yml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/workflows/publish.yml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/workflows/release.yml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.github/workflows/smoke.yml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.gitignore +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.mailmap +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/.pre-commit-config.yaml +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/CODE_OF_CONDUCT.md +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/CONTRIBUTING.md +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/LICENSE +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/SECURITY.md +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/docs/architecture.md +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/docs/demo.svg +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/docs/sample-output.md +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/docs/social-preview.png +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/examples/README.md +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/examples/targets.csv +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/catalogue.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/accounts.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/base.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/breaches.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/ct.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/dns.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/github.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/infostealer.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/pastes.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/ports.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/subdomains.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/collectors/whois_lookup.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/diff.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/errors.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/http.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/llm.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/logging_store.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/models.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/offline.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/passwords.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/py.typed +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/render.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/retention.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/synthesis.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/src/osintkit/targets.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/conftest.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/fixtures/model_reply.json +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_accounts.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_breaches.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_catalogue.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_collectors.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_config.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_ct.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_diff.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_doctor.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_e2e.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_http.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_infostealer.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_llm.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_normalise.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_offline.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_passwords.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_pastes.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_pipeline.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_render.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_retention.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_synthesis.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_targets.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_userconfig.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/tests/test_whois.py +0 -0
- {osintkit-0.6.0 → osintkit-0.7.0}/uv.lock +0 -0
|
@@ -6,6 +6,14 @@ All notable changes are recorded here. The format follows
|
|
|
6
6
|
|
|
7
7
|
## Unreleased
|
|
8
8
|
|
|
9
|
+
## 0.7.0 — 2026-11-02
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- `xposed_analytics`: dated, typed, verified breach membership and pastes, keyless
|
|
14
|
+
- post-synthesis self-check listing any claimed breach, date, host, or account URL absent from
|
|
15
|
+
the supplied data
|
|
16
|
+
|
|
9
17
|
## 0.6.0 — 2026-11-02
|
|
10
18
|
|
|
11
19
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: osintkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.7.0
|
|
4
4
|
Summary: Public-data OSINT aggregator for emails, usernames, domains, and companies
|
|
5
5
|
Project-URL: Homepage, https://github.com/sjdsjadjajdjsa/osintkit
|
|
6
6
|
Project-URL: Issues, https://github.com/sjdsjadjajdjsa/osintkit/issues
|
|
@@ -93,8 +93,10 @@ Any OpenAI-compatible endpoint works. `init --provider ollama` needs no key for
|
|
|
93
93
|
`--provider`, `--base-url`, and `--model` cover everything else. The preset table, key variables,
|
|
94
94
|
and precedence order are in [docs/REFERENCE.md](docs/REFERENCE.md#providers).
|
|
95
95
|
|
|
96
|
-
Optional keys:
|
|
97
|
-
|
|
96
|
+
Optional keys: a Have I Been Pwned subscription (`--hibp-key` on init, or `HIBP_API_KEY`) adds
|
|
97
|
+
per-account breach and paste membership; `GITHUB_TOKEN` lifts GitHub from 60 to 5000
|
|
98
|
+
requests/hour. Both stay local — the key is yours, and every user of the tool brings their own.
|
|
99
|
+
Everything else needs no credentials.
|
|
98
100
|
|
|
99
101
|
## Commands
|
|
100
102
|
|
|
@@ -144,6 +146,8 @@ the model fail.
|
|
|
144
146
|
undercut.
|
|
145
147
|
- **Cached everything.** 24h default, shorter for volatile sources; repeat prompts cost nothing.
|
|
146
148
|
- **Validated output.** Strict JSON schema, one repair round-trip, then a clean failure.
|
|
149
|
+
- **Self-checked claims.** Every breach, date, host, and account URL is compared back to the
|
|
150
|
+
collected data; anything unsupported is called out.
|
|
147
151
|
|
|
148
152
|
## Collectors
|
|
149
153
|
|
|
@@ -61,8 +61,10 @@ Any OpenAI-compatible endpoint works. `init --provider ollama` needs no key for
|
|
|
61
61
|
`--provider`, `--base-url`, and `--model` cover everything else. The preset table, key variables,
|
|
62
62
|
and precedence order are in [docs/REFERENCE.md](docs/REFERENCE.md#providers).
|
|
63
63
|
|
|
64
|
-
Optional keys:
|
|
65
|
-
|
|
64
|
+
Optional keys: a Have I Been Pwned subscription (`--hibp-key` on init, or `HIBP_API_KEY`) adds
|
|
65
|
+
per-account breach and paste membership; `GITHUB_TOKEN` lifts GitHub from 60 to 5000
|
|
66
|
+
requests/hour. Both stay local — the key is yours, and every user of the tool brings their own.
|
|
67
|
+
Everything else needs no credentials.
|
|
66
68
|
|
|
67
69
|
## Commands
|
|
68
70
|
|
|
@@ -112,6 +114,8 @@ the model fail.
|
|
|
112
114
|
undercut.
|
|
113
115
|
- **Cached everything.** 24h default, shorter for volatile sources; repeat prompts cost nothing.
|
|
114
116
|
- **Validated output.** Strict JSON schema, one repair round-trip, then a clean failure.
|
|
117
|
+
- **Self-checked claims.** Every breach, date, host, and account URL is compared back to the
|
|
118
|
+
collected data; anything unsupported is called out.
|
|
115
119
|
|
|
116
120
|
## Collectors
|
|
117
121
|
|
|
@@ -8,7 +8,7 @@ Everything the README leaves out: every flag, every collector, storage, and fail
|
|
|
8
8
|
| --- | --- | --- | --- |
|
|
9
9
|
| `infostealer` | email / username / domain | Hudson Rock stealer logs | no |
|
|
10
10
|
| `hibp_breaches`, `hibp_pastes` | email | Have I Been Pwned | yes |
|
|
11
|
-
| `xposed_or_not`, `leakcheck_public` | email | free public breach indexes | no |
|
|
11
|
+
| `xposed_or_not`, `xposed_analytics`, `leakcheck_public` | email | free public breach indexes | no |
|
|
12
12
|
| `gravatar` | email | Gravatar | no |
|
|
13
13
|
| `crt_sh_subdomains`, `crt_sh_certificates` | domain | crt.sh | no |
|
|
14
14
|
| `crt_sh_organization` | company | crt.sh organisation search, with apex-domain identity fallback | no |
|
|
@@ -62,6 +62,13 @@ Explicit `--base-url` and `--model` beat both the provider and the prefix. Tempe
|
|
|
62
62
|
Per-command: `--json`, `--markdown`, `-o/--out`, `--raw`, `--offline`, `--dry-run`, `--only`, `--skip`,
|
|
63
63
|
`--ports`, `--fail-fast`, `-c/--concurrency`, `--limit`, `--show`, `--days`, `--clear`, `--deep`.
|
|
64
64
|
|
|
65
|
+
## Self-check
|
|
66
|
+
|
|
67
|
+
After synthesis every breach name, date, subdomain, and account URL is compared with the supplied
|
|
68
|
+
data. Anything the model asserted that no source returned is listed under a "not found in source
|
|
69
|
+
data" note, so drift is visible instead of plausible. Checks only run for source families that
|
|
70
|
+
were actually collected.
|
|
71
|
+
|
|
65
72
|
## Info-stealer logs
|
|
66
73
|
|
|
67
74
|
`infostealer` queries Hudson Rock's keyless endpoint for infections that saved credentials on a
|
|
@@ -88,7 +95,7 @@ pwned. `--json` for machines. Nothing about the password beyond the prefix leave
|
|
|
88
95
|
|
|
89
96
|
`osintkit init` writes `~/.config/osintkit/config.toml` (or `$XDG_CONFIG_HOME`) with mode 600 in
|
|
90
97
|
a 700 directory, after checking the key with a one-token request. Keys recognised:
|
|
91
|
-
`provider`, `model`, `base_url`, `api_key`, `resolvers`.
|
|
98
|
+
`provider`, `model`, `base_url`, `api_key`, `hibp_api_key`, `resolvers`.
|
|
92
99
|
|
|
93
100
|
`osintkit config` prints them masked, `--reveal` shows the key, `--unset KEY` removes one, and
|
|
94
101
|
`--path` prints the location. Deleting the file is the same as `--unset` everything.
|
|
@@ -96,6 +103,7 @@ a 700 directory, after checking the key with a one-token request. Keys recognise
|
|
|
96
103
|
Precedence for each value: command-line flag, then the provider-specific environment variable,
|
|
97
104
|
then a key prefix, then the config file, then the built-in default. Stored values apply only when
|
|
98
105
|
the stored provider matches the one in use, so a one-off key export never gets the stored key.
|
|
106
|
+
`HIBP_API_KEY` in the environment beats a stored `hibp_api_key`.
|
|
99
107
|
|
|
100
108
|
## Offline mode
|
|
101
109
|
|
|
@@ -112,6 +112,7 @@ def root(
|
|
|
112
112
|
base_url=base_url or (stored.get("base_url") if config_applies else None) or preset[0],
|
|
113
113
|
model=model or (stored.get("model") if config_applies else None) or preset[1],
|
|
114
114
|
api_key=api_key,
|
|
115
|
+
hibp_api_key_value=stored.get("hibp_api_key"),
|
|
115
116
|
temperature=temperature,
|
|
116
117
|
max_tokens=max_tokens,
|
|
117
118
|
timeout=timeout,
|
|
@@ -561,6 +562,9 @@ def init(
|
|
|
561
562
|
provider: str | None = typer.Option(None, "--provider", help="Provider to store."),
|
|
562
563
|
api_key: str | None = typer.Option(None, "--api-key", help="Key to store."),
|
|
563
564
|
model: str | None = typer.Option(None, "--model", help="Default model to store."),
|
|
565
|
+
hibp_key: str | None = typer.Option(
|
|
566
|
+
None, "--hibp-key", help="Optional Have I Been Pwned key for breach and paste checks."
|
|
567
|
+
),
|
|
564
568
|
base_url: str | None = typer.Option(None, "--base-url", help="Custom endpoint to store."),
|
|
565
569
|
force: bool = typer.Option(False, "--force", help="Replace an existing config."),
|
|
566
570
|
verify: bool = typer.Option(True, "--verify/--no-verify", help="Check the key afterwards."),
|
|
@@ -603,11 +607,19 @@ def init(
|
|
|
603
607
|
raise typer.Exit(2)
|
|
604
608
|
data["api_key"] = key
|
|
605
609
|
|
|
610
|
+
stored_hibp = stored.get("hibp_api_key")
|
|
611
|
+
if hibp_key:
|
|
612
|
+
data["hibp_api_key"] = hibp_key
|
|
613
|
+
elif stored_hibp and not force:
|
|
614
|
+
data["hibp_api_key"] = stored_hibp
|
|
615
|
+
|
|
606
616
|
written = userconfig.save(data)
|
|
607
617
|
out.print(f"[green]stored[/green] {written} (mode 600)")
|
|
608
618
|
out.print(f"provider: {chosen}")
|
|
609
619
|
if data.get("api_key"):
|
|
610
620
|
out.print(f"key: {userconfig.mask(str(data['api_key']))}")
|
|
621
|
+
if data.get("hibp_api_key"):
|
|
622
|
+
out.print(f"HIBP key: {userconfig.mask(str(data['hibp_api_key']))}")
|
|
611
623
|
|
|
612
624
|
if verify:
|
|
613
625
|
probe = Settings(
|
|
@@ -698,7 +710,7 @@ def config_cmd(
|
|
|
698
710
|
table.add_column("value")
|
|
699
711
|
for name, value in stored.items():
|
|
700
712
|
shown: object = value
|
|
701
|
-
if name
|
|
713
|
+
if name in {"api_key", "hibp_api_key"} and isinstance(value, str) and not reveal:
|
|
702
714
|
shown = userconfig.mask(value)
|
|
703
715
|
table.add_row(name, render.escape(str(shown)))
|
|
704
716
|
out.print(table)
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""XposedOrNot breach analytics: dated, typed membership plus pastes."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .base import Context, collector
|
|
8
|
+
|
|
9
|
+
URL = "https://api.xposedornot.com/v1/breach-analytics"
|
|
10
|
+
MAX_LISTED = 80
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _types(value: Any) -> list[str]:
|
|
14
|
+
return [part.strip() for part in str(value or "").split(";") if part.strip()]
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _breach(row: dict[str, Any]) -> dict[str, Any] | None:
|
|
18
|
+
name = str(row.get("breach") or "").strip()
|
|
19
|
+
if not name:
|
|
20
|
+
return None
|
|
21
|
+
date = str(row.get("xposed_date") or "").strip() or None
|
|
22
|
+
return {
|
|
23
|
+
"name": name,
|
|
24
|
+
"date": date,
|
|
25
|
+
"data_types": _types(row.get("xposed_data")),
|
|
26
|
+
"domain": row.get("domain"),
|
|
27
|
+
"pwn_count": row.get("xposed_records"),
|
|
28
|
+
"verified": str(row.get("verified") or "").lower() == "yes",
|
|
29
|
+
"password_risk": row.get("password_risk"),
|
|
30
|
+
"source": "xposedornot-analytics",
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _paste(row: dict[str, Any]) -> dict[str, Any] | None:
|
|
35
|
+
paste_id = str(row.get("paste_id") or row.get("id") or "").strip()
|
|
36
|
+
if not paste_id and not row.get("title"):
|
|
37
|
+
return None
|
|
38
|
+
return {
|
|
39
|
+
"id": paste_id or None,
|
|
40
|
+
"title": row.get("title"),
|
|
41
|
+
"date": row.get("date") or row.get("xposed_date"),
|
|
42
|
+
"source": row.get("source") or "xposedornot-analytics",
|
|
43
|
+
"origin": "xposedornot",
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@collector("xposed_analytics", ("email",), "Dated breach and paste membership", order=14, ttl=21600)
|
|
48
|
+
async def xposed_analytics(ctx: Context) -> dict:
|
|
49
|
+
response = await ctx.http.get(
|
|
50
|
+
URL,
|
|
51
|
+
namespace="xposed-analytics",
|
|
52
|
+
params={"email": ctx.target},
|
|
53
|
+
empty_on=(404, 422, 429),
|
|
54
|
+
timeout=ctx.http_timeout or 60,
|
|
55
|
+
)
|
|
56
|
+
if response.status_code != 200:
|
|
57
|
+
return {"breaches": [], "pastes": [], "note": f"HTTP {response.status_code}"}
|
|
58
|
+
try:
|
|
59
|
+
body = response.json()
|
|
60
|
+
except ValueError:
|
|
61
|
+
return {"breaches": [], "pastes": [], "note": "non-JSON"}
|
|
62
|
+
|
|
63
|
+
raw = ((body.get("ExposedBreaches") or {}).get("breaches_details")) or []
|
|
64
|
+
breaches = [entry for entry in (_breach(row) for row in raw if isinstance(row, dict)) if entry]
|
|
65
|
+
raw_pastes = ((body.get("ExposedPastes") or {}).get("pastes_details")) or []
|
|
66
|
+
pastes = [entry for entry in (_paste(row) for row in raw_pastes if isinstance(row, dict)) if entry]
|
|
67
|
+
breaches.sort(key=lambda row: str(row.get("date") or ""), reverse=True)
|
|
68
|
+
return {
|
|
69
|
+
"found": len(breaches),
|
|
70
|
+
"breaches": breaches[:MAX_LISTED],
|
|
71
|
+
"pastes": pastes[:25],
|
|
72
|
+
"source": "xposedornot-analytics",
|
|
73
|
+
}
|
|
@@ -109,6 +109,7 @@ class Settings:
|
|
|
109
109
|
base_url: str = DEFAULT_BASE_URL
|
|
110
110
|
model: str = DEFAULT_MODEL
|
|
111
111
|
api_key: str | None = None
|
|
112
|
+
hibp_api_key_value: str | None = None
|
|
112
113
|
temperature: float = DEFAULT_TEMPERATURE
|
|
113
114
|
max_tokens: int = 4000
|
|
114
115
|
timeout: float = DEFAULT_TIMEOUT
|
|
@@ -127,7 +128,7 @@ class Settings:
|
|
|
127
128
|
|
|
128
129
|
@property
|
|
129
130
|
def hibp_api_key(self) -> str | None:
|
|
130
|
-
return os.environ.get("HIBP_API_KEY") or None
|
|
131
|
+
return os.environ.get("HIBP_API_KEY") or self.hibp_api_key_value or None
|
|
131
132
|
|
|
132
133
|
@property
|
|
133
134
|
def github_token(self) -> str | None:
|
|
@@ -124,10 +124,10 @@ def collect_checks(settings: Settings) -> list[Check]:
|
|
|
124
124
|
),
|
|
125
125
|
]
|
|
126
126
|
for name, variable, why in (
|
|
127
|
-
("Have I Been Pwned", "HIBP_API_KEY", "breach collectors will be skipped"),
|
|
127
|
+
("Have I Been Pwned", "HIBP_API_KEY", "breach and paste collectors will be skipped"),
|
|
128
128
|
("GitHub", "GITHUB_TOKEN", "GitHub is limited to 60 requests/hour"),
|
|
129
129
|
):
|
|
130
|
-
present = bool(os.environ.get(variable))
|
|
130
|
+
present = bool(os.environ.get(variable)) or (name.startswith("Have I") and bool(settings.hibp_api_key))
|
|
131
131
|
checks.append(Check(name, "ok" if present else "warn", "set" if present else why))
|
|
132
132
|
if key:
|
|
133
133
|
checks.append(Check("api key shape", "ok" if len(key) >= 20 else "warn", f"{len(key)} chars"))
|
|
@@ -7,7 +7,7 @@ from typing import Any
|
|
|
7
7
|
|
|
8
8
|
from .models import CollectorResult
|
|
9
9
|
|
|
10
|
-
BREACH_SOURCES = frozenset({"hibp_breaches", "xposed_or_not", "leakcheck_public"})
|
|
10
|
+
BREACH_SOURCES = frozenset({"hibp_breaches", "xposed_or_not", "xposed_analytics", "leakcheck_public"})
|
|
11
11
|
KEY_RE = re.compile(r"[^a-z0-9]+")
|
|
12
12
|
CREDENTIAL_RE = re.compile(r"pass|hash|credential|secret|token|api[ _-]?key", re.I)
|
|
13
13
|
SEVERITY = ("low", "medium", "high")
|
|
@@ -14,7 +14,7 @@ from pathlib import Path
|
|
|
14
14
|
|
|
15
15
|
from rich.console import Console
|
|
16
16
|
|
|
17
|
-
from . import catalogue, collectors, normalise, offline, render, retention, synthesis
|
|
17
|
+
from . import catalogue, collectors, normalise, offline, render, retention, synthesis, verify
|
|
18
18
|
from .config import Settings
|
|
19
19
|
from .errors import MissingAPIKey, OsintKitError, SynthesisError, TargetParseError
|
|
20
20
|
from .http import HttpClient
|
|
@@ -248,6 +248,9 @@ async def execute(
|
|
|
248
248
|
report.analysis.risk_level = normalise.apply_floor(
|
|
249
249
|
report.analysis.risk_level, signals.get("risk_floor", "low")
|
|
250
250
|
)
|
|
251
|
+
unsupported = verify.claims(report.analysis, prompt_results)
|
|
252
|
+
if unsupported:
|
|
253
|
+
report.notes.append("not found in source data: " + "; ".join(unsupported))
|
|
251
254
|
if not report.analysis.summary.strip():
|
|
252
255
|
fallback = offline.build(prompt_results, report.notes, target, narrated=True)
|
|
253
256
|
report.analysis.summary = fallback.summary
|
|
@@ -7,8 +7,8 @@ from pathlib import Path
|
|
|
7
7
|
from typing import Any
|
|
8
8
|
|
|
9
9
|
FILENAME = "config.toml"
|
|
10
|
-
KEYS = ("provider", "model", "base_url", "api_key", "resolvers")
|
|
11
|
-
SECRET_KEYS = frozenset({"api_key"})
|
|
10
|
+
KEYS = ("provider", "model", "base_url", "api_key", "hibp_api_key", "resolvers")
|
|
11
|
+
SECRET_KEYS = frozenset({"api_key", "hibp_api_key"})
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
def config_home() -> Path:
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Post-synthesis check that claims exist in the collected data."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .models import Analysis, CollectorResult
|
|
6
|
+
|
|
7
|
+
MAX_ISSUES = 5
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def _breach_rows(results: list[CollectorResult]) -> list[dict]:
|
|
11
|
+
rows: list[dict] = []
|
|
12
|
+
for result in results:
|
|
13
|
+
if result.collector == "breach_index" and isinstance(result.data, list):
|
|
14
|
+
rows.extend(row for row in result.data if isinstance(row, dict))
|
|
15
|
+
return rows
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _names(results: list[CollectorResult]) -> set[str]:
|
|
19
|
+
return {str(row.get("name") or "").strip().lower() for row in _breach_rows(results)}
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _dates(results: list[CollectorResult]) -> set[str]:
|
|
23
|
+
return {str(row.get("date") or "")[:10] for row in _breach_rows(results) if row.get("date")}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _subdomains(results: list[CollectorResult]) -> set[str]:
|
|
27
|
+
names: set[str] = set()
|
|
28
|
+
for result in results:
|
|
29
|
+
data = result.data
|
|
30
|
+
if result.collector in {"crt_sh_subdomains"} and isinstance(data, list):
|
|
31
|
+
names.update(str(item).lower() for item in data)
|
|
32
|
+
if result.collector == "passive_subdomains" and isinstance(data, dict):
|
|
33
|
+
for key in ("live_hosts", "common_prefix_hits"):
|
|
34
|
+
names.update(str(row.get("host")).lower() for row in data.get(key) or [] if row.get("host"))
|
|
35
|
+
return names
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _urls(results: list[CollectorResult]) -> set[str]:
|
|
39
|
+
urls: set[str] = set()
|
|
40
|
+
for result in results:
|
|
41
|
+
data = result.data
|
|
42
|
+
if result.collector == "username_presence" and isinstance(data, dict):
|
|
43
|
+
urls.update(str(row.get("url")) for row in data.get("found") or [] if row.get("url"))
|
|
44
|
+
if result.collector == "github_profile" and isinstance(data, dict):
|
|
45
|
+
url = (data.get("profile") or {}).get("url")
|
|
46
|
+
if url:
|
|
47
|
+
urls.add(str(url))
|
|
48
|
+
return urls
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def claims(analysis: Analysis, results: list[CollectorResult]) -> list[str]:
|
|
52
|
+
"""Return short descriptions of claims that the source data does not support."""
|
|
53
|
+
issues: list[str] = []
|
|
54
|
+
names = _names(results)
|
|
55
|
+
dates = _dates(results)
|
|
56
|
+
hosts = _subdomains(results)
|
|
57
|
+
urls = _urls(results)
|
|
58
|
+
checked = {result.collector for result in results}
|
|
59
|
+
has_breaches = "breach_index" in checked
|
|
60
|
+
has_hosts = bool(checked & {"crt_sh_subdomains", "passive_subdomains"})
|
|
61
|
+
has_accounts = bool(checked & {"username_presence", "github_profile"})
|
|
62
|
+
|
|
63
|
+
for breach in analysis.breaches:
|
|
64
|
+
if has_breaches and breach.name.strip().lower() not in names:
|
|
65
|
+
issues.append(f"breach {breach.name!r} not in the supplied data")
|
|
66
|
+
if has_breaches and breach.date and breach.date[:10] not in dates:
|
|
67
|
+
issues.append(f"date {breach.date} for {breach.name!r} not in the supplied data")
|
|
68
|
+
|
|
69
|
+
for host in analysis.subdomains:
|
|
70
|
+
if has_hosts and host.strip().lower() not in hosts:
|
|
71
|
+
issues.append(f"subdomain {host!r} not in the supplied data")
|
|
72
|
+
|
|
73
|
+
for account in analysis.associated_accounts:
|
|
74
|
+
if has_accounts and account.url and account.url.rstrip("/") not in {url.rstrip("/") for url in urls}:
|
|
75
|
+
issues.append(f"account URL {account.url!r} not in the supplied data")
|
|
76
|
+
|
|
77
|
+
return issues[:MAX_ISSUES]
|
|
@@ -643,3 +643,48 @@ def test_password_command_hit_and_miss(monkeypatch):
|
|
|
643
643
|
miss = runner.invoke(app, ["password", "hunter3", "--json"])
|
|
644
644
|
assert miss.exit_code == 0
|
|
645
645
|
assert jsonish(miss.output)["pwned"] is False
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def test_init_stores_optional_hibp_key(xdg):
|
|
649
|
+
from osintkit import userconfig
|
|
650
|
+
|
|
651
|
+
result = runner.invoke(
|
|
652
|
+
app,
|
|
653
|
+
["init", "--provider", "venice", "--api-key", "k" * 30, "--hibp-key", "h" * 32, "--no-verify"],
|
|
654
|
+
)
|
|
655
|
+
assert result.exit_code == 0
|
|
656
|
+
assert userconfig.load()["hibp_api_key"] == "h" * 32
|
|
657
|
+
shown = runner.invoke(app, ["config"])
|
|
658
|
+
assert "hhhhhh…" in shown.output
|
|
659
|
+
assert "h" * 32 not in shown.output
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def test_stored_hibp_key_reaches_settings(xdg, monkeypatch):
|
|
663
|
+
from osintkit import userconfig
|
|
664
|
+
|
|
665
|
+
userconfig.save({"provider": "venice", "api_key": "k" * 30, "hibp_api_key": "stored-hibp"})
|
|
666
|
+
captured = {}
|
|
667
|
+
|
|
668
|
+
async def spy(target, settings, *args, **kwargs):
|
|
669
|
+
captured["hibp"] = settings.hibp_api_key
|
|
670
|
+
raise RuntimeError("stop")
|
|
671
|
+
|
|
672
|
+
monkeypatch.setattr(pipeline, "execute", spy)
|
|
673
|
+
runner.invoke(app, ["lookup", "--domain", "a.com"])
|
|
674
|
+
assert captured["hibp"] == "stored-hibp"
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def test_env_hibp_key_beats_stored(xdg, monkeypatch):
|
|
678
|
+
from osintkit import userconfig
|
|
679
|
+
|
|
680
|
+
userconfig.save({"provider": "venice", "api_key": "k" * 30, "hibp_api_key": "stored-hibp"})
|
|
681
|
+
monkeypatch.setenv("HIBP_API_KEY", "env-hibp")
|
|
682
|
+
captured = {}
|
|
683
|
+
|
|
684
|
+
async def spy(target, settings, *args, **kwargs):
|
|
685
|
+
captured["hibp"] = settings.hibp_api_key
|
|
686
|
+
raise RuntimeError("stop")
|
|
687
|
+
|
|
688
|
+
monkeypatch.setattr(pipeline, "execute", spy)
|
|
689
|
+
runner.invoke(app, ["lookup", "--domain", "a.com"])
|
|
690
|
+
assert captured["hibp"] == "env-hibp"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from osintkit import verify
|
|
2
|
+
from osintkit.models import Analysis, CollectorResult
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def result(name, data):
|
|
6
|
+
return CollectorResult(collector=name, target="t", target_type="email", data=data)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def analysis(**kwargs):
|
|
10
|
+
base = {"summary": "s", "risk_level": "low"}
|
|
11
|
+
base.update(kwargs)
|
|
12
|
+
return Analysis.model_validate(base)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_supported_claims_pass():
|
|
16
|
+
results = [result("breach_index", [{"name": "Acme", "date": "2020-01-01"}])]
|
|
17
|
+
check = analysis(breaches=[{"name": "Acme", "date": "2020-01-01"}])
|
|
18
|
+
assert verify.claims(check, results) == []
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def test_unknown_breach_is_flagged():
|
|
22
|
+
results = [result("breach_index", [{"name": "Acme"}])]
|
|
23
|
+
check = analysis(breaches=[{"name": "Invented"}])
|
|
24
|
+
assert any("Invented" in issue for issue in verify.claims(check, results))
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def test_wrong_date_is_flagged():
|
|
28
|
+
results = [result("breach_index", [{"name": "Acme", "date": "2020-01-01"}])]
|
|
29
|
+
check = analysis(breaches=[{"name": "Acme", "date": "2024-09-09"}])
|
|
30
|
+
assert any("2024-09-09" in issue for issue in verify.claims(check, results))
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def test_unknown_subdomain_and_url_are_flagged():
|
|
34
|
+
results = [
|
|
35
|
+
result("crt_sh_subdomains", ["api.example.com"]),
|
|
36
|
+
result("username_presence", {"found": [{"url": "https://github.com/other"}]}),
|
|
37
|
+
]
|
|
38
|
+
check = analysis(
|
|
39
|
+
subdomains=["api.example.com", "made-up.example.com"],
|
|
40
|
+
associated_accounts=[{"platform": "GitHub", "url": "https://github.com/x"}],
|
|
41
|
+
)
|
|
42
|
+
issues = verify.claims(check, results)
|
|
43
|
+
assert any("made-up" in issue for issue in issues)
|
|
44
|
+
assert any("github.com/x" in issue for issue in issues)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_no_source_data_means_no_noise():
|
|
48
|
+
assert verify.claims(analysis(breaches=[{"name": "Anything"}]), []) == []
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_issue_list_is_capped():
|
|
52
|
+
results = [result("breach_index", [])]
|
|
53
|
+
check = analysis(breaches=[{"name": f"B{index}"} for index in range(20)])
|
|
54
|
+
assert len(verify.claims(check, results)) == verify.MAX_ISSUES
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import httpx
|
|
2
|
+
import respx
|
|
3
|
+
|
|
4
|
+
from osintkit.collectors import xposed_analytics as xa
|
|
5
|
+
|
|
6
|
+
URL = "https://api.xposedornot.com/v1/breach-analytics"
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@respx.mock
|
|
10
|
+
async def test_breaches_are_normalised_and_sorted(make_context):
|
|
11
|
+
ctx = make_context("a@b.co", "email")
|
|
12
|
+
respx.get(URL).mock(
|
|
13
|
+
return_value=httpx.Response(
|
|
14
|
+
200,
|
|
15
|
+
json={
|
|
16
|
+
"ExposedBreaches": {
|
|
17
|
+
"breaches_details": [
|
|
18
|
+
{"breach": "Old", "xposed_date": "2019", "xposed_data": "Emails;Passwords"},
|
|
19
|
+
{"breach": "New", "xposed_date": "2026", "xposed_records": 10, "verified": "Yes"},
|
|
20
|
+
]
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
)
|
|
24
|
+
)
|
|
25
|
+
result = await xa.xposed_analytics(ctx)
|
|
26
|
+
assert [row["name"] for row in result["breaches"]] == ["New", "Old"]
|
|
27
|
+
assert result["breaches"][1]["data_types"] == ["Emails", "Passwords"]
|
|
28
|
+
assert result["breaches"][0]["verified"] is True
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@respx.mock
|
|
32
|
+
async def test_pastes_are_normalised(make_context):
|
|
33
|
+
ctx = make_context("a@b.co", "email")
|
|
34
|
+
respx.get(URL).mock(
|
|
35
|
+
return_value=httpx.Response(
|
|
36
|
+
200,
|
|
37
|
+
json={"ExposedPastes": {"pastes_details": [{"paste_id": "abc", "title": "t"}]}},
|
|
38
|
+
)
|
|
39
|
+
)
|
|
40
|
+
result = await xa.xposed_analytics(ctx)
|
|
41
|
+
assert result["pastes"][0]["id"] == "abc"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@respx.mock
|
|
45
|
+
async def test_error_status_is_a_note_not_a_crash(make_context):
|
|
46
|
+
ctx = make_context("a@b.co", "email")
|
|
47
|
+
respx.get(URL).mock(return_value=httpx.Response(429))
|
|
48
|
+
result = await xa.xposed_analytics(ctx)
|
|
49
|
+
assert result["breaches"] == []
|
|
50
|
+
assert "429" in result["note"]
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@respx.mock
|
|
54
|
+
async def test_junk_rows_are_dropped(make_context):
|
|
55
|
+
ctx = make_context("a@b.co", "email")
|
|
56
|
+
respx.get(URL).mock(
|
|
57
|
+
return_value=httpx.Response(
|
|
58
|
+
200, json={"ExposedBreaches": {"breaches_details": [None, {}, {"breach": ""}]}}
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
assert (await xa.xposed_analytics(ctx))["breaches"] == []
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|