stille-hunter 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- stille_hunter/__init__.py +9 -0
- stille_hunter/check.py +332 -0
- stille_hunter/cli.py +46 -0
- stille_hunter/generate.py +232 -0
- stille_hunter/sources/__init__.py +8 -0
- stille_hunter/sources/github.py +136 -0
- stille_hunter/sources/hibp.py +103 -0
- stille_hunter/sources/null.py +20 -0
- stille_hunter/types/__init__.py +8 -0
- stille_hunter/types/api_token.py +97 -0
- stille_hunter/types/aws_key.py +82 -0
- stille_hunter/types/azure_credential.py +83 -0
- stille_hunter/types/kubeconfig.py +116 -0
- stille_hunter/types/mcp_config.py +88 -0
- stille_hunter/types/member.py +370 -0
- stille_hunter-0.2.0.dist-info/METADATA +172 -0
- stille_hunter-0.2.0.dist-info/RECORD +20 -0
- stille_hunter-0.2.0.dist-info/WHEEL +4 -0
- stille_hunter-0.2.0.dist-info/entry_points.txt +2 -0
- stille_hunter-0.2.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""stille-hunter: leak-canary generation and matching.
|
|
3
|
+
|
|
4
|
+
Public, Apache-2.0 part of stille-hunter. Generate realistic fake user records
|
|
5
|
+
to seed as leak canaries (generate), and check a canary ledger against leak
|
|
6
|
+
sources (check). Proprietary sources (Tor dark-web crawl, commercial threat
|
|
7
|
+
feeds) live in a separate package and plug into the same CheckSource contract.
|
|
8
|
+
"""
|
|
9
|
+
__version__ = "0.2.0"
|
stille_hunter/check.py
ADDED
|
@@ -0,0 +1,332 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""stille-hunter: leak-matcher skeleton (`stille check`).
|
|
4
|
+
|
|
5
|
+
Reads a canary ledger (produced by canary_gen.py) and checks each canary's
|
|
6
|
+
tokens against one or more *sources*. A source is any place a leaked record
|
|
7
|
+
might surface: Have I Been Pwned, public code search, a paste site, a threat
|
|
8
|
+
feed, and so on.
|
|
9
|
+
|
|
10
|
+
This file is the PLUGGABLE FRAME only. It does the parts every source shares —
|
|
11
|
+
load the ledger, pick which tokens to look for, fan out to the registered
|
|
12
|
+
sources, collect hits, report them — and leaves the actual lookups to small
|
|
13
|
+
plugins that implement one method. A real source (HIBP, GitHub, ...) is added
|
|
14
|
+
by subclassing CheckSource and registering it; the frame does not change.
|
|
15
|
+
|
|
16
|
+
Design notes (stille-hunter principles):
|
|
17
|
+
* Read-only, search-only. A source may only *look up* a token in a public or
|
|
18
|
+
authorized place. It never logs in anywhere it shouldn't, never downloads
|
|
19
|
+
or stores leaked data, never touches anyone else's system beyond a lookup.
|
|
20
|
+
* detect flag decides eligibility. Canaries flagged detect=inbox are caught
|
|
21
|
+
by catch-all mail, not here; by default `check` only looks at detect=search
|
|
22
|
+
canaries. --include-inbox overrides (e.g. to also search for self-domain
|
|
23
|
+
addresses that may have leaked).
|
|
24
|
+
* A source gets TOKENS to look for, not raw records. The only things worth
|
|
25
|
+
searching are the unique markers: record_id, the email, the user_id, and
|
|
26
|
+
the password_hash. Everything else in a record is filler.
|
|
27
|
+
"""
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import argparse
|
|
31
|
+
import json
|
|
32
|
+
import sys
|
|
33
|
+
from dataclasses import dataclass, field
|
|
34
|
+
from datetime import datetime, timezone
|
|
35
|
+
from typing import Iterable
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
# Data carried through the pipeline
|
|
40
|
+
# ---------------------------------------------------------------------------
|
|
41
|
+
@dataclass(frozen=True)
|
|
42
|
+
class Canary:
|
|
43
|
+
"""One ledger entry, reduced to what a source needs.
|
|
44
|
+
|
|
45
|
+
The member-type fields (email/user_id/password_hash) are kept as named
|
|
46
|
+
attributes for convenience and may be empty for non-member canaries. Any
|
|
47
|
+
other ledger markers a canary type recorded (aws_access_key_id, api_token,
|
|
48
|
+
...) ride along in `extra`, so a source can search whatever a given canary
|
|
49
|
+
actually carries without the frame knowing every type."""
|
|
50
|
+
record_id: str
|
|
51
|
+
seed_tag: str
|
|
52
|
+
detect: str # "inbox" | "search"
|
|
53
|
+
locale: str
|
|
54
|
+
email: str
|
|
55
|
+
user_id: str
|
|
56
|
+
password_hash: str
|
|
57
|
+
extra: dict = field(default_factory=dict)
|
|
58
|
+
|
|
59
|
+
def tokens(self) -> dict[str, str]:
|
|
60
|
+
"""All searchable markers for this canary, keyed by kind (named fields
|
|
61
|
+
plus any type-specific markers in `extra`, empties dropped)."""
|
|
62
|
+
out = {
|
|
63
|
+
"email": self.email,
|
|
64
|
+
"user_id": self.user_id,
|
|
65
|
+
"password_hash": self.password_hash,
|
|
66
|
+
"record_id": self.record_id,
|
|
67
|
+
}
|
|
68
|
+
out.update(self.extra)
|
|
69
|
+
return {k: v for k, v in out.items() if v}
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass
|
|
73
|
+
class Hit:
|
|
74
|
+
"""A source found a canary token somewhere it shouldn't be."""
|
|
75
|
+
canary: Canary
|
|
76
|
+
source: str # source name, e.g. "hibp"
|
|
77
|
+
token_kind: str # which marker matched, e.g. "email"
|
|
78
|
+
token: str # the matched value
|
|
79
|
+
location: str # where it was found (URL, breach name, ...)
|
|
80
|
+
detail: str = "" # free-form extra context
|
|
81
|
+
found_at: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
|
|
82
|
+
|
|
83
|
+
def to_dict(self) -> dict:
|
|
84
|
+
return {
|
|
85
|
+
"record_id": self.canary.record_id,
|
|
86
|
+
"seed_tag": self.canary.seed_tag,
|
|
87
|
+
"locale": self.canary.locale,
|
|
88
|
+
"source": self.source,
|
|
89
|
+
"token_kind": self.token_kind,
|
|
90
|
+
"token": self.token,
|
|
91
|
+
"location": self.location,
|
|
92
|
+
"detail": self.detail,
|
|
93
|
+
"found_at": self.found_at,
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
# ---------------------------------------------------------------------------
|
|
98
|
+
# The plugin contract
|
|
99
|
+
# ---------------------------------------------------------------------------
|
|
100
|
+
class CheckSource:
|
|
101
|
+
"""Base class for a leak source. Subclass and implement check().
|
|
102
|
+
|
|
103
|
+
A source must be side-effect-free beyond read-only lookups: no login to
|
|
104
|
+
places you aren't authorized for, no downloading or storing leaked data,
|
|
105
|
+
no touching third-party systems beyond the lookup itself.
|
|
106
|
+
"""
|
|
107
|
+
name: str = "base"
|
|
108
|
+
#: which token kinds this source can meaningfully search for
|
|
109
|
+
supports: tuple[str, ...] = ("email", "user_id", "password_hash", "record_id")
|
|
110
|
+
|
|
111
|
+
def __init__(self, **config):
|
|
112
|
+
self.config = config
|
|
113
|
+
# The frame sets this to a callable(done:int, total:int, note:str).
|
|
114
|
+
# A slow source should call self.progress(i, n, "...") as it goes so
|
|
115
|
+
# the run doesn't look frozen. Default is a no-op.
|
|
116
|
+
self.progress = lambda done, total, note="": None
|
|
117
|
+
|
|
118
|
+
def available(self) -> tuple[bool, str]:
|
|
119
|
+
"""Return (ok, reason). A source that needs an API key it doesn't have
|
|
120
|
+
returns (False, "...") and is skipped with that reason reported."""
|
|
121
|
+
return True, ""
|
|
122
|
+
|
|
123
|
+
def check(self, canaries: list[Canary]) -> Iterable[Hit]:
|
|
124
|
+
"""Look the canaries up and yield a Hit for each match found.
|
|
125
|
+
|
|
126
|
+
Receives the whole batch so a source can use bulk endpoints (e.g. HIBP
|
|
127
|
+
domain search, a single query of many tokens) instead of one call per
|
|
128
|
+
canary. Must not raise for an individual miss; raise only on a hard
|
|
129
|
+
failure (network/auth), which the frame catches and reports per-source.
|
|
130
|
+
"""
|
|
131
|
+
raise NotImplementedError
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
# ---------------------------------------------------------------------------
|
|
135
|
+
# Source registry (real sources register themselves here)
|
|
136
|
+
# ---------------------------------------------------------------------------
|
|
137
|
+
_REGISTRY: dict[str, type[CheckSource]] = {}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def register(cls: type[CheckSource]) -> type[CheckSource]:
|
|
141
|
+
"""Decorator: add a CheckSource subclass to the registry by its .name."""
|
|
142
|
+
if not getattr(cls, "name", None) or cls.name == "base":
|
|
143
|
+
raise ValueError("a CheckSource must set a unique .name")
|
|
144
|
+
if cls.name in _REGISTRY:
|
|
145
|
+
raise ValueError(f"duplicate source name: {cls.name}")
|
|
146
|
+
_REGISTRY[cls.name] = cls
|
|
147
|
+
return cls
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# Sources live in the stille_hunter.sources subpackage and register themselves
|
|
151
|
+
# when imported (see load_plugins). Proprietary sources ship as a separate
|
|
152
|
+
# package, stille_hunter_pro.sources, discovered the same way.
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# ---------------------------------------------------------------------------
|
|
156
|
+
# Ledger loading
|
|
157
|
+
# ---------------------------------------------------------------------------
|
|
158
|
+
def load_ledger(path: str, include_inbox: bool) -> list[Canary]:
|
|
159
|
+
canaries: list[Canary] = []
|
|
160
|
+
is_csv = path.endswith(".csv")
|
|
161
|
+
with open(path, encoding="utf-8", newline="") as f:
|
|
162
|
+
rows: Iterable[dict]
|
|
163
|
+
if is_csv:
|
|
164
|
+
import csv
|
|
165
|
+
rows = csv.DictReader(f)
|
|
166
|
+
else:
|
|
167
|
+
rows = (json.loads(line) for line in f if line.strip())
|
|
168
|
+
# ledger keys the frame handles itself; everything else is a
|
|
169
|
+
# type-specific marker and goes into Canary.extra.
|
|
170
|
+
known = {"record_id", "seed_tag", "detect", "locale", "email",
|
|
171
|
+
"user_id", "password_hash", "created_at"}
|
|
172
|
+
for r in rows:
|
|
173
|
+
detect = r.get("detect", "search")
|
|
174
|
+
if detect == "inbox" and not include_inbox:
|
|
175
|
+
continue
|
|
176
|
+
extra = {k: v for k, v in r.items() if k not in known and v}
|
|
177
|
+
canaries.append(Canary(
|
|
178
|
+
record_id=r["record_id"],
|
|
179
|
+
seed_tag=r.get("seed_tag", ""),
|
|
180
|
+
detect=detect,
|
|
181
|
+
locale=r.get("locale", ""),
|
|
182
|
+
email=r.get("email", ""),
|
|
183
|
+
user_id=r.get("user_id", ""),
|
|
184
|
+
password_hash=r.get("password_hash", ""),
|
|
185
|
+
extra=extra,
|
|
186
|
+
))
|
|
187
|
+
return canaries
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
# ---------------------------------------------------------------------------
|
|
191
|
+
# Runner
|
|
192
|
+
# ---------------------------------------------------------------------------
|
|
193
|
+
def run_check(canaries: list[Canary], source_names: list[str],
|
|
194
|
+
source_config: dict, show_progress: bool = True
|
|
195
|
+
) -> tuple[list[Hit], list[tuple[str, str]]]:
|
|
196
|
+
"""Return (hits, skipped) where skipped is [(source_name, reason)]."""
|
|
197
|
+
hits: list[Hit] = []
|
|
198
|
+
skipped: list[tuple[str, str]] = []
|
|
199
|
+
for name in source_names:
|
|
200
|
+
cls = _REGISTRY.get(name)
|
|
201
|
+
if cls is None:
|
|
202
|
+
skipped.append((name, "no such source registered"))
|
|
203
|
+
continue
|
|
204
|
+
src = cls(**source_config.get(name, {}))
|
|
205
|
+
ok, reason = src.available()
|
|
206
|
+
if not ok:
|
|
207
|
+
skipped.append((name, reason or "unavailable"))
|
|
208
|
+
continue
|
|
209
|
+
if show_progress:
|
|
210
|
+
def _report(done, total, note="", _n=name):
|
|
211
|
+
msg = f"\r [{_n}] {done}/{total} {note}".ljust(60)
|
|
212
|
+
sys.stderr.write(msg)
|
|
213
|
+
sys.stderr.flush()
|
|
214
|
+
if done >= total:
|
|
215
|
+
sys.stderr.write("\n")
|
|
216
|
+
src.progress = _report
|
|
217
|
+
# only hand a source the token kinds it supports
|
|
218
|
+
eligible = [c for c in canaries if src.supports]
|
|
219
|
+
try:
|
|
220
|
+
hits.extend(src.check(eligible))
|
|
221
|
+
except Exception as e: # a hard failure in one source must not kill the run
|
|
222
|
+
skipped.append((name, f"error: {e}"))
|
|
223
|
+
return hits, skipped
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def load_plugins(packages=("stille_hunter.sources", "stille_hunter_pro.sources")
|
|
227
|
+
) -> list[str]:
|
|
228
|
+
"""Import every module in each source package so its @register'd sources
|
|
229
|
+
attach themselves. Returns a list of load problems (empty = clean).
|
|
230
|
+
|
|
231
|
+
Importing a source module runs its @register at import time. The public
|
|
232
|
+
sources live in stille_hunter.sources; proprietary ones, when installed,
|
|
233
|
+
in stille_hunter_pro.sources — both discovered the same way. A package
|
|
234
|
+
that isn't installed (e.g. the pro one) is skipped silently.
|
|
235
|
+
"""
|
|
236
|
+
import importlib
|
|
237
|
+
import pkgutil
|
|
238
|
+
problems: list[str] = []
|
|
239
|
+
for pkg_name in packages:
|
|
240
|
+
try:
|
|
241
|
+
pkg = importlib.import_module(pkg_name)
|
|
242
|
+
except ImportError:
|
|
243
|
+
continue # package not installed (fine for the optional pro one)
|
|
244
|
+
for mod in pkgutil.iter_modules(pkg.__path__):
|
|
245
|
+
if mod.name.startswith("_"):
|
|
246
|
+
continue
|
|
247
|
+
try:
|
|
248
|
+
importlib.import_module(f"{pkg_name}.{mod.name}")
|
|
249
|
+
except Exception as e:
|
|
250
|
+
problems.append(f"{pkg_name}.{mod.name}: {e}")
|
|
251
|
+
return problems
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def main(argv=None):
|
|
255
|
+
# Discover sources from the stille_hunter.sources package (and the pro
|
|
256
|
+
# package if installed). Importing each registers it.
|
|
257
|
+
plugin_problems = load_plugins()
|
|
258
|
+
|
|
259
|
+
p = argparse.ArgumentParser(
|
|
260
|
+
description="Check canary ledgers against leak sources (stille-hunter).")
|
|
261
|
+
p.add_argument("ledger", nargs="?",
|
|
262
|
+
help="Path to a canary ledger (.jsonl or .csv). "
|
|
263
|
+
"Not needed with --list-sources.")
|
|
264
|
+
p.add_argument("--sources", default="null",
|
|
265
|
+
help="Comma-separated source names. Registered: "
|
|
266
|
+
+ ",".join(sorted(_REGISTRY)) + " (default: null)")
|
|
267
|
+
p.add_argument("--include-inbox", action="store_true",
|
|
268
|
+
help="Also check detect=inbox canaries (normally caught by "
|
|
269
|
+
"catch-all mail, not searched here).")
|
|
270
|
+
p.add_argument("--config", help="Optional JSON file of per-source config "
|
|
271
|
+
'(e.g. {"hibp": {"api_key": "..."}}).')
|
|
272
|
+
p.add_argument("--out", help="Write hits as JSONL to this path (default: stdout summary).")
|
|
273
|
+
p.add_argument("--list-sources", action="store_true",
|
|
274
|
+
help="List registered sources and exit.")
|
|
275
|
+
args = p.parse_args(argv)
|
|
276
|
+
|
|
277
|
+
for prob in plugin_problems:
|
|
278
|
+
sys.stderr.write(f"WARNING: could not load source plugin {prob}\n")
|
|
279
|
+
|
|
280
|
+
if args.list_sources:
|
|
281
|
+
width = max(map(len, _REGISTRY))
|
|
282
|
+
for name in sorted(_REGISTRY):
|
|
283
|
+
cls = _REGISTRY[name]
|
|
284
|
+
print(f"{name:{width}} supports={','.join(cls.supports)}")
|
|
285
|
+
return
|
|
286
|
+
|
|
287
|
+
if not args.ledger:
|
|
288
|
+
p.error("a ledger path is required (or use --list-sources)")
|
|
289
|
+
|
|
290
|
+
source_config = {}
|
|
291
|
+
if args.config:
|
|
292
|
+
try:
|
|
293
|
+
with open(args.config, encoding="utf-8") as f:
|
|
294
|
+
source_config = json.load(f)
|
|
295
|
+
except FileNotFoundError:
|
|
296
|
+
sys.stderr.write(f"ERROR: config file not found: {args.config}\n")
|
|
297
|
+
sys.exit(2)
|
|
298
|
+
except json.JSONDecodeError as e:
|
|
299
|
+
sys.stderr.write(f"ERROR: config file is not valid JSON ({args.config}): {e}\n")
|
|
300
|
+
sys.exit(2)
|
|
301
|
+
|
|
302
|
+
try:
|
|
303
|
+
canaries = load_ledger(args.ledger, args.include_inbox)
|
|
304
|
+
except FileNotFoundError:
|
|
305
|
+
sys.stderr.write(f"ERROR: ledger not found: {args.ledger}\n")
|
|
306
|
+
sys.exit(2)
|
|
307
|
+
source_names = [s.strip() for s in args.sources.split(",") if s.strip()]
|
|
308
|
+
|
|
309
|
+
hits, skipped = run_check(canaries, source_names, source_config)
|
|
310
|
+
|
|
311
|
+
print(f"checked {len(canaries)} canaries against {len(source_names)} "
|
|
312
|
+
f"source(s): {len(hits)} hit(s)")
|
|
313
|
+
for name, reason in skipped:
|
|
314
|
+
print(f" source '{name}' skipped: {reason}")
|
|
315
|
+
|
|
316
|
+
if args.out:
|
|
317
|
+
with open(args.out, "w", encoding="utf-8") as f:
|
|
318
|
+
for h in hits:
|
|
319
|
+
f.write(json.dumps(h.to_dict(), ensure_ascii=False) + "\n")
|
|
320
|
+
if hits:
|
|
321
|
+
print(f" wrote {len(hits)} hit(s) to {args.out}")
|
|
322
|
+
else:
|
|
323
|
+
for h in hits:
|
|
324
|
+
print(f" HIT {h.canary.record_id} [{h.source}] {h.token_kind}="
|
|
325
|
+
f"{h.token} @ {h.location}")
|
|
326
|
+
|
|
327
|
+
# non-zero exit when something leaked, so cron/CI can alert on it
|
|
328
|
+
sys.exit(1 if hits else 0)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
if __name__ == "__main__":
|
|
332
|
+
main()
|
stille_hunter/cli.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""stille-hunter unified CLI: `stille-hunter generate ...` / `check ...`.
|
|
3
|
+
|
|
4
|
+
Thin dispatcher. `generate` and `check` each own their own argument parsing in
|
|
5
|
+
their module's main(); this just routes the subcommand to the right one.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
|
|
13
|
+
_USAGE = """\
|
|
14
|
+
stille-hunter <command> [options]
|
|
15
|
+
|
|
16
|
+
commands:
|
|
17
|
+
generate Generate leak-canary user records to seed into your data.
|
|
18
|
+
check Check a canary ledger against leak sources.
|
|
19
|
+
|
|
20
|
+
Run 'stille-hunter <command> --help' for command options.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main(argv=None):
|
|
25
|
+
argv = list(sys.argv[1:] if argv is None else argv)
|
|
26
|
+
if not argv or argv[0] in ("-h", "--help"):
|
|
27
|
+
sys.stdout.write(_USAGE)
|
|
28
|
+
return
|
|
29
|
+
if argv[0] in ("-V", "--version"):
|
|
30
|
+
print(f"stille-hunter {__version__}")
|
|
31
|
+
return
|
|
32
|
+
|
|
33
|
+
cmd, rest = argv[0], argv[1:]
|
|
34
|
+
if cmd == "generate":
|
|
35
|
+
from .generate import main as gen_main
|
|
36
|
+
gen_main(rest)
|
|
37
|
+
elif cmd == "check":
|
|
38
|
+
from .check import main as check_main
|
|
39
|
+
check_main(rest)
|
|
40
|
+
else:
|
|
41
|
+
sys.stderr.write(f"unknown command: {cmd}\n\n{_USAGE}")
|
|
42
|
+
sys.exit(2)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
if __name__ == "__main__":
|
|
46
|
+
main()
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""stille-hunter: canary generator engine.
|
|
4
|
+
|
|
5
|
+
Generates fake-but-realistic canaries to seed as leak tripwires. WHAT kind of
|
|
6
|
+
canary is produced is a plugin — a CanaryType. The engine here only ties the
|
|
7
|
+
pieces together: pick a type, let it generate records, and write the two output
|
|
8
|
+
files (the seeded data you plant, and the secret ledger that maps each canary's
|
|
9
|
+
token back to where you planted it).
|
|
10
|
+
|
|
11
|
+
Canary TYPES are the mirror image of check SOURCES: a type subclasses
|
|
12
|
+
CanaryType and registers itself, and the engine never changes to add one.
|
|
13
|
+
Built-in types live in stille_hunter.types; proprietary ones ship as
|
|
14
|
+
stille_hunter_pro.types and are discovered the same way.
|
|
15
|
+
|
|
16
|
+
Design principles (every type must honour these):
|
|
17
|
+
* Everything stays on your own turf. This only *generates* data; it never
|
|
18
|
+
touches anyone else's system.
|
|
19
|
+
* A canary must be realistic but inert — it can never authenticate, charge,
|
|
20
|
+
or resolve to anything real, and it carries a unique token so a leak can be
|
|
21
|
+
traced back to where it was planted.
|
|
22
|
+
* No real PII of real people, no minors, no national-ID / card-CVV storage.
|
|
23
|
+
|
|
24
|
+
Every record carries two bookkeeping markers the engine adds to the ledger:
|
|
25
|
+
* detect = "inbox" -> caught by a catch-all you control (fast, active)
|
|
26
|
+
* detect = "search" -> found by searching leak sources (the default)
|
|
27
|
+
"""
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import argparse
|
|
31
|
+
import csv
|
|
32
|
+
import json
|
|
33
|
+
import secrets
|
|
34
|
+
import sys
|
|
35
|
+
from datetime import datetime, timezone
|
|
36
|
+
from typing import Iterable
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
# ---------------------------------------------------------------------------
|
|
40
|
+
# Shared helpers (used by the engine and by type plugins)
|
|
41
|
+
# ---------------------------------------------------------------------------
|
|
42
|
+
def new_token() -> str:
|
|
43
|
+
"""A short, unique, URL/identifier-safe canary token (6 hex chars)."""
|
|
44
|
+
return secrets.token_hex(3)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def weighted_choice(options, weights):
|
|
48
|
+
r = secrets.randbelow(sum(weights))
|
|
49
|
+
upto = 0
|
|
50
|
+
for opt, w in zip(options, weights, strict=True):
|
|
51
|
+
upto += w
|
|
52
|
+
if r < upto:
|
|
53
|
+
return opt
|
|
54
|
+
return options[-1]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# ---------------------------------------------------------------------------
|
|
58
|
+
# The plugin contract
|
|
59
|
+
# ---------------------------------------------------------------------------
|
|
60
|
+
class CanaryType:
|
|
61
|
+
"""Base class for a kind of canary. Subclass, implement, and @register_type.
|
|
62
|
+
|
|
63
|
+
A type owns everything specific to its shape: the CLI options it needs, how
|
|
64
|
+
it builds one seeded record, and which fields go into the ledger as
|
|
65
|
+
searchable markers. The engine stays generic.
|
|
66
|
+
"""
|
|
67
|
+
#: unique type name, e.g. "member", "aws_key"
|
|
68
|
+
name: str = "base"
|
|
69
|
+
#: one-line description for --list-types
|
|
70
|
+
description: str = ""
|
|
71
|
+
|
|
72
|
+
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
|
|
73
|
+
"""Register this type's own CLI options on the generate parser.
|
|
74
|
+
|
|
75
|
+
Common options (--count, --tag, --format, --out) are added by the
|
|
76
|
+
engine; a type only adds what is specific to it (e.g. --locale)."""
|
|
77
|
+
return
|
|
78
|
+
|
|
79
|
+
def generate(self, args, count: int) -> Iterable[dict]:
|
|
80
|
+
"""Yield `count` records. Each record is a dict with:
|
|
81
|
+
* the plantable fields (what gets written to the seeded file), plus
|
|
82
|
+
* "_detect": "inbox" | "search" for this record, and
|
|
83
|
+
* "_ledger": {marker_name: value, ...} — the searchable tokens to
|
|
84
|
+
record in the ledger (email, user_id, key id, hash, record_id...).
|
|
85
|
+
The engine strips the "_"-prefixed keys from the seeded output and
|
|
86
|
+
writes "_ledger" (plus record_id/seed_tag/detect) to the ledger.
|
|
87
|
+
Every record MUST include "record_id"."""
|
|
88
|
+
raise NotImplementedError
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
# Type registry
|
|
93
|
+
# ---------------------------------------------------------------------------
|
|
94
|
+
_TYPES: dict[str, type[CanaryType]] = {}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def register_type(cls: type[CanaryType]) -> type[CanaryType]:
|
|
98
|
+
if not getattr(cls, "name", None) or cls.name == "base":
|
|
99
|
+
raise ValueError("a CanaryType must set a unique .name")
|
|
100
|
+
if cls.name in _TYPES:
|
|
101
|
+
raise ValueError(f"duplicate canary type: {cls.name}")
|
|
102
|
+
_TYPES[cls.name] = cls
|
|
103
|
+
return cls
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def load_type_plugins(packages=("stille_hunter.types", "stille_hunter_pro.types")
|
|
107
|
+
) -> list[str]:
|
|
108
|
+
"""Import every module in each type package so it self-registers."""
|
|
109
|
+
import importlib
|
|
110
|
+
import pkgutil
|
|
111
|
+
problems: list[str] = []
|
|
112
|
+
for pkg_name in packages:
|
|
113
|
+
try:
|
|
114
|
+
pkg = importlib.import_module(pkg_name)
|
|
115
|
+
except ImportError:
|
|
116
|
+
continue
|
|
117
|
+
for mod in pkgutil.iter_modules(pkg.__path__):
|
|
118
|
+
if mod.name.startswith("_"):
|
|
119
|
+
continue
|
|
120
|
+
try:
|
|
121
|
+
importlib.import_module(f"{pkg_name}.{mod.name}")
|
|
122
|
+
except Exception as e:
|
|
123
|
+
problems.append(f"{pkg_name}.{mod.name}: {e}")
|
|
124
|
+
return problems
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# ---------------------------------------------------------------------------
|
|
128
|
+
# Output
|
|
129
|
+
# ---------------------------------------------------------------------------
|
|
130
|
+
def _seeded_view(rec: dict) -> dict:
|
|
131
|
+
"""The record as planted: drop the engine's bookkeeping keys."""
|
|
132
|
+
return {k: v for k, v in rec.items() if not k.startswith("_")}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _ledger_row(rec: dict, seed_tag: str) -> dict:
|
|
136
|
+
row = {
|
|
137
|
+
"record_id": rec["record_id"],
|
|
138
|
+
"seed_tag": seed_tag,
|
|
139
|
+
"detect": rec.get("_detect", "search"),
|
|
140
|
+
}
|
|
141
|
+
# the type's declared searchable markers
|
|
142
|
+
row.update(rec.get("_ledger", {}))
|
|
143
|
+
row["created_at"] = datetime.now(timezone.utc).isoformat()
|
|
144
|
+
return row
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def write_jsonl(records, seed_tag, sf, lf):
|
|
148
|
+
for rec in records:
|
|
149
|
+
sf.write(json.dumps(_seeded_view(rec), ensure_ascii=False) + "\n")
|
|
150
|
+
lf.write(json.dumps(_ledger_row(rec, seed_tag), ensure_ascii=False) + "\n")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def write_csv(records, seed_tag, sf, lf):
|
|
154
|
+
records = list(records)
|
|
155
|
+
if not records:
|
|
156
|
+
return
|
|
157
|
+
sw = csv.DictWriter(sf, fieldnames=list(_seeded_view(records[0]).keys()))
|
|
158
|
+
sw.writeheader()
|
|
159
|
+
lw = csv.DictWriter(lf, fieldnames=list(_ledger_row(records[0], seed_tag).keys()))
|
|
160
|
+
lw.writeheader()
|
|
161
|
+
for rec in records:
|
|
162
|
+
sw.writerow(_seeded_view(rec))
|
|
163
|
+
lw.writerow(_ledger_row(rec, seed_tag))
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
# ---------------------------------------------------------------------------
|
|
167
|
+
# CLI
|
|
168
|
+
# ---------------------------------------------------------------------------
|
|
169
|
+
def main(argv=None):
|
|
170
|
+
load_type_plugins()
|
|
171
|
+
|
|
172
|
+
# Two-stage parse: first just --type (and the always-available flags), then
|
|
173
|
+
# let the chosen type add its own options.
|
|
174
|
+
pre = argparse.ArgumentParser(add_help=False)
|
|
175
|
+
pre.add_argument("--type", default="member")
|
|
176
|
+
pre.add_argument("--list-types", action="store_true")
|
|
177
|
+
known, _ = pre.parse_known_args(argv)
|
|
178
|
+
|
|
179
|
+
if known.list_types:
|
|
180
|
+
width = max(map(len, _TYPES))
|
|
181
|
+
for name in sorted(_TYPES):
|
|
182
|
+
print(f"{name:{width}} {_TYPES[name].description}")
|
|
183
|
+
return
|
|
184
|
+
|
|
185
|
+
if known.type not in _TYPES:
|
|
186
|
+
sys.stderr.write(
|
|
187
|
+
f"ERROR: unknown canary type '{known.type}'. "
|
|
188
|
+
f"Available: {', '.join(sorted(_TYPES))}\n")
|
|
189
|
+
sys.exit(2)
|
|
190
|
+
|
|
191
|
+
canary_type = _TYPES[known.type]()
|
|
192
|
+
|
|
193
|
+
p = argparse.ArgumentParser(
|
|
194
|
+
description="Generate leak canaries (stille-hunter).")
|
|
195
|
+
p.add_argument("--type", default="member",
|
|
196
|
+
help="Canary type: " + ", ".join(sorted(_TYPES)) + " (default: member)")
|
|
197
|
+
p.add_argument("--list-types", action="store_true",
|
|
198
|
+
help="List available canary types and exit.")
|
|
199
|
+
p.add_argument("--count", type=int, default=10)
|
|
200
|
+
p.add_argument("--tag", default="default",
|
|
201
|
+
help="Seed tag: where these canaries will be planted.")
|
|
202
|
+
p.add_argument("--format", default="jsonl", choices=["jsonl", "csv"])
|
|
203
|
+
p.add_argument("--out", default="canary_seeded")
|
|
204
|
+
canary_type.add_arguments(p) # the type's own options
|
|
205
|
+
args = p.parse_args(argv)
|
|
206
|
+
|
|
207
|
+
try:
|
|
208
|
+
recs = list(canary_type.generate(args, args.count))
|
|
209
|
+
except ValueError as e:
|
|
210
|
+
sys.stderr.write(f"ERROR: {e}\n")
|
|
211
|
+
sys.exit(2)
|
|
212
|
+
|
|
213
|
+
ext = args.format
|
|
214
|
+
seeded_path, ledger_path = f"{args.out}.{ext}", f"{args.out}.ledger.{ext}"
|
|
215
|
+
inbox_n = sum(1 for r in recs if r.get("_detect") == "inbox")
|
|
216
|
+
|
|
217
|
+
with open(seeded_path, "w", encoding="utf-8", newline="") as sf, \
|
|
218
|
+
open(ledger_path, "w", encoding="utf-8", newline="") as lf:
|
|
219
|
+
(write_jsonl if ext == "jsonl" else write_csv)(recs, args.tag, sf, lf)
|
|
220
|
+
|
|
221
|
+
detail = (f"({inbox_n} inbox-detect, {len(recs)-inbox_n} search-only)"
|
|
222
|
+
if inbox_n else "(search-only)")
|
|
223
|
+
print(f"wrote {len(recs)} {known.type} canaries {detail}:")
|
|
224
|
+
print(f" seeded data : {seeded_path} (plant this in your DB/export)")
|
|
225
|
+
print(f" ledger : {ledger_path}")
|
|
226
|
+
print(" !! The ledger is the token -> seed-location map. Move it OUT of this")
|
|
227
|
+
print(" directory to private storage. NEVER commit it, and NEVER plant it")
|
|
228
|
+
print(" anywhere a leak could expose it — it unmasks every canary at once.")
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
if __name__ == "__main__":
|
|
232
|
+
main()
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""Built-in leak-check sources for stille-hunter.
|
|
3
|
+
|
|
4
|
+
Each module here defines one or more CheckSource subclasses decorated with
|
|
5
|
+
@register. They are imported and registered automatically by
|
|
6
|
+
stille_hunter.check.load_plugins(). Add a source by dropping a module here
|
|
7
|
+
(or, for proprietary sources, in the stille_hunter_pro.sources package).
|
|
8
|
+
"""
|