starshield-forensics 0.3.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- starshield/__init__.py +3 -0
- starshield/cli.py +743 -0
- starshield/data/__init__.py +1 -0
- starshield/data/synthetic_case_benchmark.json +73 -0
- starshield/image_containers.py +436 -0
- starshield/ioc_validators.py +229 -0
- starshield/leveldb_log.py +795 -0
- starshield_forensics-0.3.7.dist-info/METADATA +119 -0
- starshield_forensics-0.3.7.dist-info/RECORD +13 -0
- starshield_forensics-0.3.7.dist-info/WHEEL +5 -0
- starshield_forensics-0.3.7.dist-info/entry_points.txt +2 -0
- starshield_forensics-0.3.7.dist-info/licenses/LICENSE +21 -0
- starshield_forensics-0.3.7.dist-info/top_level.txt +1 -0
starshield/cli.py
ADDED
|
@@ -0,0 +1,743 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
StarShield Forensics CLI v0.3.6 (`starshield`)
|
|
5
|
+
Author: Ziqi Xiao (@Ziqi)
|
|
6
|
+
License: MIT
|
|
7
|
+
|
|
8
|
+
Deterministic, strictly offline browser cache, image container, and financial
|
|
9
|
+
indicator triage toolkit for digital forensics and commercial payment-dispute review.
|
|
10
|
+
|
|
11
|
+
Every analytical claim is tagged with an explicit epistemic tier:
|
|
12
|
+
- OBSERVED : Directly present in raw input bytes/fields
|
|
13
|
+
- DERIVED : Deterministically computed from raw input (CRC32C, Luhn, Mod-97, deltas)
|
|
14
|
+
- INFERRED : Triage lead requiring external corroboration (never standalone proof)
|
|
15
|
+
- UNKNOWN : Outside the evidentiary boundary of local client artifacts
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import hashlib
|
|
22
|
+
import hmac
|
|
23
|
+
import html
|
|
24
|
+
import importlib.resources
|
|
25
|
+
import json
|
|
26
|
+
import re
|
|
27
|
+
import secrets
|
|
28
|
+
import statistics
|
|
29
|
+
import sys
|
|
30
|
+
from datetime import datetime, timezone
|
|
31
|
+
from pathlib import Path
|
|
32
|
+
from typing import Any, Dict, List
|
|
33
|
+
|
|
34
|
+
from starshield import __version__
|
|
35
|
+
from starshield.image_containers import (
|
|
36
|
+
build_valid_jpeg_with_icc_and_8bim,
|
|
37
|
+
build_valid_png_with_iccp,
|
|
38
|
+
inspect_image_container,
|
|
39
|
+
)
|
|
40
|
+
from starshield.ioc_validators import (
|
|
41
|
+
SUPPORTED_HMAC_DIGESTS,
|
|
42
|
+
empty_ioc_dict,
|
|
43
|
+
extract_validated_iocs,
|
|
44
|
+
get_active_hmac_callable,
|
|
45
|
+
luhn_valid,
|
|
46
|
+
mask_identifier,
|
|
47
|
+
merge_ioc_dicts,
|
|
48
|
+
set_active_hmac_digest,
|
|
49
|
+
)
|
|
50
|
+
from starshield.leveldb_log import (
|
|
51
|
+
VALID_SCHEMA_MODES,
|
|
52
|
+
build_writebatch,
|
|
53
|
+
classify_carved_json_object,
|
|
54
|
+
encode_logical_record_across_blocks,
|
|
55
|
+
inspect_chromium_storage_path,
|
|
56
|
+
normalize_carved_record_fields,
|
|
57
|
+
parse_leveldb_wal,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
MAX_DEFAULT_MB = 64
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def sha256_bytes(data: bytes) -> str:
|
|
64
|
+
return hashlib.sha256(data).hexdigest()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def redact_timestamp(ts: Any) -> Any:
|
|
68
|
+
"""Suppress exact dates and times in default masked mode to prevent timeline linkage."""
|
|
69
|
+
if ts is None or ts == "":
|
|
70
|
+
return None
|
|
71
|
+
epoch = parse_strict_utc_ts(ts)
|
|
72
|
+
if epoch is None:
|
|
73
|
+
return "[REDACTED_UNPARSED_TIMESTAMP]"
|
|
74
|
+
return "[REDACTED_UTC_TIMESTAMP]"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def redact_msg_id(msg_id: str, salt: bytes) -> str:
|
|
78
|
+
"""Keyed per-run HMAC message token with configurable digest agility."""
|
|
79
|
+
if not msg_id:
|
|
80
|
+
return ""
|
|
81
|
+
digest = hmac.new(salt, msg_id.encode("utf-8"), get_active_hmac_callable()).hexdigest()[:12]
|
|
82
|
+
return f"msg_hmac_{digest}"
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def build_acquisition_manifest(
|
|
86
|
+
raw: bytes,
|
|
87
|
+
source_label: str,
|
|
88
|
+
redact: bool = True,
|
|
89
|
+
operator: str = "unspecified",
|
|
90
|
+
) -> Dict[str, Any]:
|
|
91
|
+
now_utc = datetime.now(timezone.utc)
|
|
92
|
+
return {
|
|
93
|
+
"source_label": "[REDACTED_SOURCE_LABEL]" if redact else source_label,
|
|
94
|
+
"input_byte_length": None if redact else len(raw),
|
|
95
|
+
"input_sha256": (
|
|
96
|
+
"[OMITTED_IN_MASKED_MODE — pass --unredacted in controlled lab for raw SHA-256]"
|
|
97
|
+
if redact
|
|
98
|
+
else sha256_bytes(raw)
|
|
99
|
+
),
|
|
100
|
+
"processed_utc": now_utc.strftime("%Y-%m") if redact else now_utc.isoformat(),
|
|
101
|
+
"tool_version": f"starshield-forensics/{__version__}",
|
|
102
|
+
"operator": operator,
|
|
103
|
+
"redaction_notice": (
|
|
104
|
+
"Default masked mode applies ephemeral per-run HMAC-SHA256 keyed pseudonymization "
|
|
105
|
+
"(not formal legal anonymization under GDPR/UK GDPR) and suppresses raw file hashes, "
|
|
106
|
+
"exact byte lengths, exact UTC dates, filenames, image free-text chunks, and plaintext "
|
|
107
|
+
"bodies. Pass --unredacted inside authorized offline forensic environments only."
|
|
108
|
+
),
|
|
109
|
+
"provenance_caveat": (
|
|
110
|
+
"Cryptographic hashes attest post-acquisition byte integrity only; upstream device "
|
|
111
|
+
"authenticity and completeness require an independent acquisition log."
|
|
112
|
+
),
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def read_bounded_bytes(path: Path, max_mb: int = MAX_DEFAULT_MB, redact: bool = True) -> bytes:
|
|
117
|
+
if max_mb < 1:
|
|
118
|
+
raise ValueError("--max-mb safety limit must be >= 1 MiB")
|
|
119
|
+
try:
|
|
120
|
+
size = path.stat().st_size
|
|
121
|
+
except OSError as exc:
|
|
122
|
+
if redact:
|
|
123
|
+
raise OSError(
|
|
124
|
+
"Unable to access input file (filename and OS path suppressed in masked mode; "
|
|
125
|
+
"pass --unredacted in an offline lab for path diagnostics)"
|
|
126
|
+
) from None
|
|
127
|
+
raise exc
|
|
128
|
+
limit = max_mb * 1024 * 1024
|
|
129
|
+
if size > limit:
|
|
130
|
+
if redact:
|
|
131
|
+
raise ValueError(
|
|
132
|
+
f"Input file exceeds --max-mb={max_mb} safety limit "
|
|
133
|
+
"(filename and exact byte length suppressed in masked mode)"
|
|
134
|
+
)
|
|
135
|
+
raise ValueError(f"File {path.name} ({size} bytes) exceeds --max-mb={max_mb} safety limit")
|
|
136
|
+
try:
|
|
137
|
+
return path.read_bytes()
|
|
138
|
+
except OSError as exc:
|
|
139
|
+
if redact:
|
|
140
|
+
raise OSError(
|
|
141
|
+
"Unable to read input file bytes (filename and OS path suppressed in masked mode; "
|
|
142
|
+
"pass --unredacted in an offline lab for path diagnostics)"
|
|
143
|
+
) from None
|
|
144
|
+
raise exc
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def strip_markup(raw: str) -> str:
|
|
148
|
+
if not raw:
|
|
149
|
+
return ""
|
|
150
|
+
cleaned = re.sub(r"<br\s*/?>", "\n", str(raw), flags=re.IGNORECASE)
|
|
151
|
+
cleaned = re.sub(r"<[^>]+>", " ", cleaned)
|
|
152
|
+
cleaned = html.unescape(cleaned)
|
|
153
|
+
return re.sub(r"[ \t]+", " ", cleaned).strip()
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def parse_strict_utc_ts(ts: Any) -> float | None:
|
|
157
|
+
"""Parse epoch ms/s or explicit timezone-aware ISO-8601. Rejects naive local timestamps."""
|
|
158
|
+
if ts is None or ts == "":
|
|
159
|
+
return None
|
|
160
|
+
if isinstance(ts, (int, float)):
|
|
161
|
+
val = float(ts)
|
|
162
|
+
return val / 1000.0 if val > 1e11 else val
|
|
163
|
+
s = str(ts).strip()
|
|
164
|
+
if re.fullmatch(r"\d+(?:\.\d+)?", s):
|
|
165
|
+
val = float(s)
|
|
166
|
+
return val / 1000.0 if val > 1e11 else val
|
|
167
|
+
try:
|
|
168
|
+
normalized = s.replace("Z", "+00:00")
|
|
169
|
+
dt = datetime.fromisoformat(normalized)
|
|
170
|
+
if dt.tzinfo is None:
|
|
171
|
+
return None
|
|
172
|
+
return dt.astimezone(timezone.utc).timestamp()
|
|
173
|
+
except ValueError:
|
|
174
|
+
return None
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def apply_wal_record_redaction(
|
|
178
|
+
wal_parsed: Dict[str, Any],
|
|
179
|
+
redact: bool = True,
|
|
180
|
+
salt: bytes | None = None,
|
|
181
|
+
) -> Dict[str, Any]:
|
|
182
|
+
run_salt = salt if salt is not None else secrets.token_bytes(32)
|
|
183
|
+
if redact:
|
|
184
|
+
wal_parsed["file_size_bytes"] = None
|
|
185
|
+
for rec in wal_parsed["recovered_records"]:
|
|
186
|
+
raw_txt = rec["cached_content"]
|
|
187
|
+
rec["extracted_iocs"] = extract_validated_iocs(raw_txt, redact=redact, salt=run_salt)
|
|
188
|
+
if redact:
|
|
189
|
+
rec["key_sha256"] = "[OMITTED_IN_MASKED_MODE]"
|
|
190
|
+
rec["value_sha256"] = "[OMITTED_IN_MASKED_MODE]"
|
|
191
|
+
rec["message_id"] = redact_msg_id(rec["message_id"], salt=run_salt)
|
|
192
|
+
rec["sender_mri"] = mask_identifier(rec["sender_mri"], salt=run_salt)
|
|
193
|
+
rec["composetime"] = redact_timestamp(rec["composetime"])
|
|
194
|
+
rec["deletetime"] = redact_timestamp(rec["deletetime"])
|
|
195
|
+
rec["edittime"] = redact_timestamp(rec["edittime"])
|
|
196
|
+
rec["cached_content"] = "[REDACTED_CONTENT — pass --unredacted to view]"
|
|
197
|
+
return wal_parsed
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
_STANDARD_PNG_CHUNKS = frozenset(
|
|
201
|
+
{
|
|
202
|
+
"IHDR", "PLTE", "IDAT", "IEND", "iCCP", "sRGB", "gAMA", "cHRM",
|
|
203
|
+
"pHYs", "tEXt", "zTXt", "iTXt", "tIME", "bKGD", "tRNS", "sBIT",
|
|
204
|
+
"hIST", "sPLT", "eXIf",
|
|
205
|
+
}
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def apply_image_redaction(
|
|
210
|
+
img_report: Dict[str, Any],
|
|
211
|
+
redact: bool = True,
|
|
212
|
+
salt: bytes | None = None,
|
|
213
|
+
) -> Dict[str, Any]:
|
|
214
|
+
"""Suppress PNG free-text software chunk previews, custom chunk tags, and custom ICC names in default masked mode."""
|
|
215
|
+
if not redact:
|
|
216
|
+
return img_report
|
|
217
|
+
run_salt = salt if salt is not None else secrets.token_bytes(32)
|
|
218
|
+
img_report["artifact"] = "[REDACTED_IMAGE_LABEL]"
|
|
219
|
+
for ch in img_report.get("chunks_summary", []):
|
|
220
|
+
if isinstance(ch, dict):
|
|
221
|
+
if "chunk" in ch and ch.get("chunk") not in _STANDARD_PNG_CHUNKS:
|
|
222
|
+
ch["chunk"] = "[CUSTOM_PNG_CHUNK]"
|
|
223
|
+
if "type" in ch and ch.get("type") not in _STANDARD_PNG_CHUNKS:
|
|
224
|
+
ch["type"] = "[CUSTOM_PNG_CHUNK]"
|
|
225
|
+
for cerr in img_report.get("crc_errors", []):
|
|
226
|
+
if isinstance(cerr, dict):
|
|
227
|
+
if "chunk" in cerr and cerr.get("chunk") not in _STANDARD_PNG_CHUNKS:
|
|
228
|
+
cerr["chunk"] = "[CUSTOM_PNG_CHUNK]"
|
|
229
|
+
if "chunk_type" in cerr and cerr.get("chunk_type") not in _STANDARD_PNG_CHUNKS:
|
|
230
|
+
cerr["chunk_type"] = "[CUSTOM_PNG_CHUNK]"
|
|
231
|
+
for prof in img_report.get("icc_profiles", []):
|
|
232
|
+
if not prof.get("valid_icc_header") and "reason" in prof:
|
|
233
|
+
prof["reason"] = "Invalid or non-ISO-15076-1 ICC profile header (raw signature bytes suppressed in masked mode)"
|
|
234
|
+
if prof.get("png_iccp_name"):
|
|
235
|
+
prof["png_iccp_name"] = mask_identifier(str(prof["png_iccp_name"]), salt=run_salt)
|
|
236
|
+
if prof.get("profile_creator"):
|
|
237
|
+
prof["profile_creator"] = mask_identifier(str(prof["profile_creator"]), salt=run_salt)
|
|
238
|
+
for tag in img_report.get("software_text_chunks", []):
|
|
239
|
+
tag["preview"] = "[REDACTED_TEXT_METADATA — pass --unredacted to view]"
|
|
240
|
+
return img_report
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def triage_exported_chatsvc_json(
|
|
244
|
+
payload: Dict[str, Any],
|
|
245
|
+
redact: bool = True,
|
|
246
|
+
salt: bytes | None = None,
|
|
247
|
+
schema_mode: str = "auto",
|
|
248
|
+
) -> Dict[str, Any]:
|
|
249
|
+
if schema_mode not in VALID_SCHEMA_MODES:
|
|
250
|
+
raise ValueError(f"Unsupported schema_mode={schema_mode!r}; expected one of {VALID_SCHEMA_MODES}")
|
|
251
|
+
if not isinstance(payload, dict):
|
|
252
|
+
raise ValueError("Expected JSON object containing a 'messages' or 'records' list")
|
|
253
|
+
|
|
254
|
+
if "messages" in payload:
|
|
255
|
+
messages = payload.get("messages")
|
|
256
|
+
if not isinstance(messages, list):
|
|
257
|
+
raise ValueError("Expected 'messages' to be a JSON list")
|
|
258
|
+
elif "records" in payload:
|
|
259
|
+
messages = payload.get("records")
|
|
260
|
+
if not isinstance(messages, list):
|
|
261
|
+
raise ValueError("Expected 'records' to be a JSON list")
|
|
262
|
+
else:
|
|
263
|
+
messages = []
|
|
264
|
+
|
|
265
|
+
run_salt = salt if salt is not None else secrets.token_bytes(32)
|
|
266
|
+
retracted: List[Dict[str, Any]] = []
|
|
267
|
+
edited: List[Dict[str, Any]] = []
|
|
268
|
+
corpus_iocs = empty_ioc_dict()
|
|
269
|
+
|
|
270
|
+
for msg in messages:
|
|
271
|
+
if not isinstance(msg, dict):
|
|
272
|
+
continue
|
|
273
|
+
schema_tag = classify_carved_json_object(msg, schema_mode=schema_mode)
|
|
274
|
+
if schema_mode == "chatsvc" and schema_tag != "chatsvc":
|
|
275
|
+
continue
|
|
276
|
+
norm = normalize_carved_record_fields(msg, schema_tag or "generic_json")
|
|
277
|
+
body = strip_markup(norm["cached_content"])
|
|
278
|
+
# Extract IOCs strictly PER MESSAGE so cross-message numbers never pair!
|
|
279
|
+
msg_iocs = extract_validated_iocs(body, redact=redact, salt=run_salt)
|
|
280
|
+
merge_ioc_dicts(corpus_iocs, msg_iocs)
|
|
281
|
+
|
|
282
|
+
delete_ts = norm["deletetime"]
|
|
283
|
+
edit_ts = norm["edittime"]
|
|
284
|
+
raw_id = norm["message_id"]
|
|
285
|
+
rendered_id = redact_msg_id(raw_id, salt=run_salt) if redact else raw_id
|
|
286
|
+
sender = norm["sender_mri"] or "unknown"
|
|
287
|
+
rendered_sender = mask_identifier(sender, salt=run_salt) if redact else sender
|
|
288
|
+
comp_ts = norm["composetime"]
|
|
289
|
+
|
|
290
|
+
if delete_ts:
|
|
291
|
+
retracted.append(
|
|
292
|
+
{
|
|
293
|
+
"epistemic_tier": "OBSERVED",
|
|
294
|
+
"schema_detected": schema_tag or "generic_json",
|
|
295
|
+
"id": rendered_id,
|
|
296
|
+
"sender_mri": rendered_sender,
|
|
297
|
+
"compose_time": redact_timestamp(comp_ts) if redact else comp_ts,
|
|
298
|
+
"delete_time": redact_timestamp(delete_ts) if redact else delete_ts,
|
|
299
|
+
"cached_plaintext": "[REDACTED_CONTENT — pass --unredacted to view]" if redact else body,
|
|
300
|
+
"extracted_iocs": msg_iocs,
|
|
301
|
+
}
|
|
302
|
+
)
|
|
303
|
+
if edit_ts:
|
|
304
|
+
edited.append(
|
|
305
|
+
{
|
|
306
|
+
"epistemic_tier": "OBSERVED",
|
|
307
|
+
"schema_detected": schema_tag or "generic_json",
|
|
308
|
+
"id": rendered_id,
|
|
309
|
+
"sender_mri": rendered_sender,
|
|
310
|
+
"compose_time": redact_timestamp(comp_ts) if redact else comp_ts,
|
|
311
|
+
"edit_time": redact_timestamp(edit_ts) if redact else edit_ts,
|
|
312
|
+
"limitation": "Exported JSON retains post-edit content only; pre-edit versions require uncompacted LevelDB sequence history.",
|
|
313
|
+
}
|
|
314
|
+
)
|
|
315
|
+
|
|
316
|
+
return {
|
|
317
|
+
"input_mode": "exported_message_json",
|
|
318
|
+
"schema_mode": schema_mode,
|
|
319
|
+
"messages_present_in_input": len(messages),
|
|
320
|
+
"soft_deleted_messages_in_input": len(retracted),
|
|
321
|
+
"edited_messages_in_input": len(edited),
|
|
322
|
+
"completeness_caveat": (
|
|
323
|
+
"Counts reflect records present in the supplied export only; already compacted "
|
|
324
|
+
"or garbage-collected LevelDB records cannot be enumerated from client cache alone."
|
|
325
|
+
),
|
|
326
|
+
"soft_deleted_records": retracted,
|
|
327
|
+
"edited_records": edited,
|
|
328
|
+
"corpus_validated_iocs": corpus_iocs,
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def corroborate_timestamps(
|
|
333
|
+
payload: Dict[str, Any],
|
|
334
|
+
redact: bool = True,
|
|
335
|
+
salt: bytes | None = None,
|
|
336
|
+
) -> Dict[str, Any]:
|
|
337
|
+
"""Verify timezone-aware UTC event ordering and clock-skew bounds across generic event pairs."""
|
|
338
|
+
if not isinstance(payload, dict):
|
|
339
|
+
raise ValueError("Expected top-level JSON object for timestamp corroboration")
|
|
340
|
+
run_salt = salt if salt is not None else secrets.token_bytes(32)
|
|
341
|
+
event_pairs = payload.get("timestamp_pairs")
|
|
342
|
+
if event_pairs is None:
|
|
343
|
+
event_pairs = []
|
|
344
|
+
elif not isinstance(event_pairs, list):
|
|
345
|
+
raise ValueError("Expected 'timestamp_pairs' to be a JSON list")
|
|
346
|
+
|
|
347
|
+
routing_tags = payload.get("session_metadata")
|
|
348
|
+
if routing_tags is None:
|
|
349
|
+
routing_tags = []
|
|
350
|
+
elif not isinstance(routing_tags, list):
|
|
351
|
+
raise ValueError("Expected 'session_metadata' to be a JSON list")
|
|
352
|
+
|
|
353
|
+
matrix: List[Dict[str, Any]] = []
|
|
354
|
+
seen_regions: Dict[str, int] = {}
|
|
355
|
+
for ev in routing_tags:
|
|
356
|
+
if not isinstance(ev, dict):
|
|
357
|
+
continue
|
|
358
|
+
tag = str(ev.get("edge_pop_tag", "")).strip()
|
|
359
|
+
if tag:
|
|
360
|
+
rendered_tag = mask_identifier(tag, salt=run_salt) if redact else tag
|
|
361
|
+
seen_regions[rendered_tag] = seen_regions.get(rendered_tag, 0) + 1
|
|
362
|
+
|
|
363
|
+
for r_tag, count in sorted(seen_regions.items(), key=lambda x: -x[1]):
|
|
364
|
+
matrix.append(
|
|
365
|
+
{
|
|
366
|
+
"observation": f"Infrastructure edge tag {r_tag} observed across {count} session record(s).",
|
|
367
|
+
"provenance": "session_metadata[].edge_pop_tag",
|
|
368
|
+
"epistemic_tier": "OBSERVED",
|
|
369
|
+
"alternative_explanations": [
|
|
370
|
+
"Edge PoP tags reflect CDN or media-relay latency routing, NOT confirmed physical endpoint geolocation.",
|
|
371
|
+
"Repeated events within a single persistent session are correlated rather than statistically independent.",
|
|
372
|
+
],
|
|
373
|
+
}
|
|
374
|
+
)
|
|
375
|
+
|
|
376
|
+
intervals: List[Dict[str, Any]] = []
|
|
377
|
+
for idx, pair in enumerate(event_pairs):
|
|
378
|
+
if not isinstance(pair, dict):
|
|
379
|
+
continue
|
|
380
|
+
raw_label = str(pair.get("label", f"pair_{idx + 1}"))
|
|
381
|
+
label = f"pair_{idx + 1}" if redact else raw_label
|
|
382
|
+
t_start = parse_strict_utc_ts(pair.get("start_utc"))
|
|
383
|
+
t_end = parse_strict_utc_ts(pair.get("end_utc"))
|
|
384
|
+
if t_start is None or t_end is None:
|
|
385
|
+
intervals.append(
|
|
386
|
+
{
|
|
387
|
+
"label": label,
|
|
388
|
+
"raw_label": None if redact else raw_label,
|
|
389
|
+
"status": "INVALID_OR_TIMEZONE_NAIVE_TIMESTAMP",
|
|
390
|
+
"epistemic_tier": "UNKNOWN",
|
|
391
|
+
}
|
|
392
|
+
)
|
|
393
|
+
continue
|
|
394
|
+
delta = round(t_end - t_start, 3)
|
|
395
|
+
if delta < 0:
|
|
396
|
+
intervals.append(
|
|
397
|
+
{
|
|
398
|
+
"label": label,
|
|
399
|
+
"raw_label": None if redact else raw_label,
|
|
400
|
+
"delta_seconds": None if redact else delta,
|
|
401
|
+
"status": "REJECTED_NEGATIVE_DELTA_CLOCK_SKEW_OR_BAD_INPUT",
|
|
402
|
+
"epistemic_tier": "UNKNOWN",
|
|
403
|
+
}
|
|
404
|
+
)
|
|
405
|
+
else:
|
|
406
|
+
intervals.append(
|
|
407
|
+
{
|
|
408
|
+
"label": label,
|
|
409
|
+
"raw_label": None if redact else raw_label,
|
|
410
|
+
"delta_seconds": "[REDACTED_INTERVAL]" if redact else delta,
|
|
411
|
+
"status": "DERIVED_NON_NEGATIVE_INTERVAL",
|
|
412
|
+
"epistemic_tier": "DERIVED",
|
|
413
|
+
"caveat": "Cross-system timestamp intervals depend on NTP clock synchronization across endpoints.",
|
|
414
|
+
}
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
return {
|
|
418
|
+
"corroboration_matrix": matrix,
|
|
419
|
+
"timestamp_interval_observations": intervals,
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
def summarize_corpus_linguistics(payload: Dict[str, Any]) -> Dict[str, Any]:
|
|
424
|
+
"""Compute strictly descriptive utterance length & punctuation statistics (ZERO clinical/legal diagnosis)."""
|
|
425
|
+
if not isinstance(payload, dict):
|
|
426
|
+
raise ValueError("Expected top-level JSON object for corpus linguistics summary")
|
|
427
|
+
messages = payload.get("messages")
|
|
428
|
+
if messages is None:
|
|
429
|
+
messages = []
|
|
430
|
+
elif not isinstance(messages, list):
|
|
431
|
+
raise ValueError("Expected 'messages' to be a JSON list")
|
|
432
|
+
subject_id = payload.get("subject_sender_id", "user:accounts_payable_02")
|
|
433
|
+
|
|
434
|
+
utterances = [
|
|
435
|
+
strip_markup(m.get("content") or m.get("text") or m.get("body") or "")
|
|
436
|
+
for m in messages
|
|
437
|
+
if isinstance(m, dict)
|
|
438
|
+
and (m.get("from") == subject_id or m.get("user") == subject_id or m.get("sender") == subject_id)
|
|
439
|
+
and strip_markup(m.get("content") or m.get("text") or m.get("body") or "")
|
|
440
|
+
]
|
|
441
|
+
|
|
442
|
+
n = len(utterances)
|
|
443
|
+
lengths = [len(u) for u in utterances]
|
|
444
|
+
in_3_to_12 = [u for u in utterances if 3 <= len(u) <= 12]
|
|
445
|
+
no_mid_punct = [
|
|
446
|
+
u for u in utterances if not re.search(r"[,.;:!?,。;:!?]", u[:-1] if len(u) > 1 else u)
|
|
447
|
+
]
|
|
448
|
+
|
|
449
|
+
return {
|
|
450
|
+
"epistemic_tier": "DERIVED",
|
|
451
|
+
"sample_size": n,
|
|
452
|
+
"minimum_recommended_sample_met": n >= 30,
|
|
453
|
+
"median_char_length": round(statistics.median(lengths), 2) if lengths else None,
|
|
454
|
+
"utterance_ratio_3_to_12_chars": round(len(in_3_to_12) / n, 4) if n else None,
|
|
455
|
+
"zero_mid_punctuation_ratio": round(len(no_mid_punct) / n, 4) if n else None,
|
|
456
|
+
"methodological_limitation": (
|
|
457
|
+
"Descriptive text statistics only. Short, unpunctuated replies are routine in "
|
|
458
|
+
"workplace and mobile messaging and never establish psychological state or legal vulnerability."
|
|
459
|
+
),
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def build_synthetic_leveldb_wal_bytes() -> bytes:
|
|
464
|
+
"""
|
|
465
|
+
Create a clean-room, specification-only LevelDB `.log` test vector emulating Chromium
|
|
466
|
+
IndexedDB binary prefixes (`\\xff\\x15` Blink header) + cross-32KiB `FIRST/MIDDLE/LAST` blocks.
|
|
467
|
+
Uses standard RFC/ISO textbook test values from a fictional 2025 UK supplier-invoice scenario.
|
|
468
|
+
"""
|
|
469
|
+
blink_prefix = b"\xff\x15\xfe\x00\x22"
|
|
470
|
+
msg1 = blink_prefix + json.dumps(
|
|
471
|
+
{
|
|
472
|
+
"id": "msg_spec_0001",
|
|
473
|
+
"from": "user:impersonated_supplier_01",
|
|
474
|
+
"composetime": "2025-03-18T10:00:00Z",
|
|
475
|
+
"content": (
|
|
476
|
+
"Updated UK Faster Payments supplier account for Invoice INV-2025-0100 (GBP 18,400.00): "
|
|
477
|
+
"Sort Code 00-00-00 Account 00000000."
|
|
478
|
+
),
|
|
479
|
+
"properties": {
|
|
480
|
+
"deletetime": "1742292600000",
|
|
481
|
+
"nested_meta": {"schema_hint": "chromium_idb_spec_vector"},
|
|
482
|
+
},
|
|
483
|
+
}
|
|
484
|
+
).encode("utf-8")
|
|
485
|
+
|
|
486
|
+
# Standard ISO/IEC 7812 Luhn test vector prefix (620000000000000x -> 6200000000000005)
|
|
487
|
+
prefix = "620000000000000"
|
|
488
|
+
valid_test_card = next(prefix + d for d in "0123456789" if luhn_valid(prefix + d))
|
|
489
|
+
# Pad second WriteBatch (>66 KiB) so it spans 3 physical 32KiB blocks via FIRST + MIDDLE + LAST
|
|
490
|
+
padding_blob = "X" * 68000
|
|
491
|
+
msg2 = blink_prefix + json.dumps(
|
|
492
|
+
{
|
|
493
|
+
"id": "msg_spec_0002",
|
|
494
|
+
"from": "user:impersonated_supplier_01",
|
|
495
|
+
"composetime": "2025-03-18T14:30:00Z",
|
|
496
|
+
"content": (
|
|
497
|
+
f"Secondary reserve IBAN GB82WEST12345698765432 (SWIFT NWBKGB2LXXX), "
|
|
498
|
+
f"Norway test IBAN NO9386011117947, and Luhn test card {valid_test_card}."
|
|
499
|
+
),
|
|
500
|
+
"properties": {"deletetime": "1742308800000", "blob_pad": padding_blob},
|
|
501
|
+
}
|
|
502
|
+
).encode("utf-8")
|
|
503
|
+
|
|
504
|
+
wb1 = build_writebatch(1001, [(1, b"\x00\x01\x02\x01idb:msg:spec_0001", msg1)])
|
|
505
|
+
wb2 = build_writebatch(1002, [(1, b"\x00\x01\x02\x01idb:msg:spec_0002", msg2)])
|
|
506
|
+
|
|
507
|
+
part1 = encode_logical_record_across_blocks(wb1, initial_block_used=0)
|
|
508
|
+
part2 = encode_logical_record_across_blocks(wb2, initial_block_used=len(part1))
|
|
509
|
+
return part1 + part2
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def run_demo(redact: bool = True) -> int:
|
|
513
|
+
run_salt = secrets.token_bytes(32)
|
|
514
|
+
try:
|
|
515
|
+
sample_json_bytes = (
|
|
516
|
+
importlib.resources.files("starshield.data")
|
|
517
|
+
.joinpath("synthetic_case_benchmark.json")
|
|
518
|
+
.read_bytes()
|
|
519
|
+
)
|
|
520
|
+
except Exception:
|
|
521
|
+
repo_sample = Path(__file__).resolve().parent.parent / "samples" / "synthetic_case_benchmark.json"
|
|
522
|
+
sample_json_bytes = repo_sample.read_bytes()
|
|
523
|
+
payload = json.loads(sample_json_bytes.decode("utf-8"))
|
|
524
|
+
|
|
525
|
+
wal_bytes = build_synthetic_leveldb_wal_bytes()
|
|
526
|
+
wal_parsed = apply_wal_record_redaction(parse_leveldb_wal(wal_bytes), redact=redact, salt=run_salt)
|
|
527
|
+
|
|
528
|
+
valid_png = build_valid_png_with_iccp(
|
|
529
|
+
"Display P3",
|
|
530
|
+
"APPL",
|
|
531
|
+
software_text="Software\x00QuickShot Raster Editor 4.2",
|
|
532
|
+
)
|
|
533
|
+
valid_jpeg = build_valid_jpeg_with_icc_and_8bim("Display P3", "APPL", split_icc_chunks=2)
|
|
534
|
+
|
|
535
|
+
report = {
|
|
536
|
+
"toolkit": f"starshield-forensics v{__version__} (Deterministic Chromium Cache & Container Provenance Toolkit)",
|
|
537
|
+
"author": "Ziqi Xiao (@Ziqi)",
|
|
538
|
+
"dataset_notice": "100% CLEAN-ROOM SPECIFICATION TEST VECTORS (2025 UK SUPPLIER INVOICE SCENARIO — TLP:CLEAR)",
|
|
539
|
+
"acquisition_manifest": build_acquisition_manifest(
|
|
540
|
+
sample_json_bytes,
|
|
541
|
+
"synthetic_case_benchmark.json",
|
|
542
|
+
redact=redact,
|
|
543
|
+
),
|
|
544
|
+
"engine_1_raw_leveldb_wal_carver": wal_parsed,
|
|
545
|
+
"engine_2_exported_json_triage": triage_exported_chatsvc_json(payload, redact=redact, salt=run_salt),
|
|
546
|
+
"engine_3_timestamp_and_session_corroboration": corroborate_timestamps(payload, redact=redact, salt=run_salt),
|
|
547
|
+
"engine_4a_structural_png_metadata": apply_image_redaction(
|
|
548
|
+
inspect_image_container(valid_png, "spec_valid_iccp_sample.png"),
|
|
549
|
+
redact=redact,
|
|
550
|
+
salt=run_salt,
|
|
551
|
+
),
|
|
552
|
+
"engine_4b_structural_jpeg_metadata": apply_image_redaction(
|
|
553
|
+
inspect_image_container(valid_jpeg, "spec_valid_app2_8bim_sample.jpg"),
|
|
554
|
+
redact=redact,
|
|
555
|
+
salt=run_salt,
|
|
556
|
+
),
|
|
557
|
+
"engine_5_descriptive_linguistics": summarize_corpus_linguistics(payload),
|
|
558
|
+
}
|
|
559
|
+
print(json.dumps(report, indent=2, ensure_ascii=False))
|
|
560
|
+
return 0
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
def main(argv: List[str] | None = None) -> int:
|
|
564
|
+
parser = argparse.ArgumentParser(
|
|
565
|
+
prog="starshield",
|
|
566
|
+
description=f"StarShield Forensics v{__version__} — Deterministic Chromium Cache & Container Triage CLI",
|
|
567
|
+
)
|
|
568
|
+
parser.add_argument(
|
|
569
|
+
"--unredacted",
|
|
570
|
+
action="store_true",
|
|
571
|
+
help="Include unmasked identifiers, raw SHA-256, exact timestamps, and plaintext bodies (offline lab use only)",
|
|
572
|
+
)
|
|
573
|
+
parser.add_argument(
|
|
574
|
+
"--max-mb",
|
|
575
|
+
type=int,
|
|
576
|
+
default=MAX_DEFAULT_MB,
|
|
577
|
+
help=f"Maximum input file size in MiB (default: {MAX_DEFAULT_MB})",
|
|
578
|
+
)
|
|
579
|
+
parser.add_argument(
|
|
580
|
+
"--hmac-digest",
|
|
581
|
+
choices=tuple(SUPPORTED_HMAC_DIGESTS.keys()),
|
|
582
|
+
default="sha256",
|
|
583
|
+
help="HMAC digest algorithm for keyed pseudonymization agility (default: sha256)",
|
|
584
|
+
)
|
|
585
|
+
parser.add_argument(
|
|
586
|
+
"--hmac-key-file",
|
|
587
|
+
type=Path,
|
|
588
|
+
default=None,
|
|
589
|
+
help="Optional external secret key file (>=16 bytes) for cross-run pseudonymization without hardcoded keys",
|
|
590
|
+
)
|
|
591
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
592
|
+
|
|
593
|
+
sub.add_parser("demo", help="Run deterministic triage pipeline on clean-room specification test vectors")
|
|
594
|
+
|
|
595
|
+
p_pre = sub.add_parser(
|
|
596
|
+
"precheck",
|
|
597
|
+
help="Zero-PII local GO/PARTIAL/NO-GO feasibility check on a Chromium storage folder or .log/.ldb file",
|
|
598
|
+
)
|
|
599
|
+
p_pre.add_argument("target_path", type=Path, help="Read-only Chromium IndexedDB/LevelDB folder or .log/.ldb file")
|
|
600
|
+
p_pre.add_argument(
|
|
601
|
+
"--schema",
|
|
602
|
+
choices=VALID_SCHEMA_MODES,
|
|
603
|
+
default="auto",
|
|
604
|
+
help="JSON record schema filter (default: auto — supports Teams/Skype, Slack, WhatsApp Web & B2B portals)",
|
|
605
|
+
)
|
|
606
|
+
|
|
607
|
+
p_wal = sub.add_parser("parse-leveldb-log", help="Parse Chromium IndexedDB LevelDB .log file with byte offsets & CRC32C")
|
|
608
|
+
p_wal.add_argument("log_file", type=Path, help="Read-only copy of LevelDB .log write-ahead log")
|
|
609
|
+
p_wal.add_argument(
|
|
610
|
+
"--schema",
|
|
611
|
+
choices=VALID_SCHEMA_MODES,
|
|
612
|
+
default="auto",
|
|
613
|
+
help="JSON record schema filter (default: auto — supports Teams/Skype, Slack, WhatsApp Web & B2B portals)",
|
|
614
|
+
)
|
|
615
|
+
|
|
616
|
+
p_rec = sub.add_parser("triage-json", help="Triage exported message JSON for soft-deleted records & validated IOCs")
|
|
617
|
+
p_rec.add_argument("json_file", type=Path, help="Path to exported message JSON")
|
|
618
|
+
p_rec.add_argument(
|
|
619
|
+
"--schema",
|
|
620
|
+
choices=VALID_SCHEMA_MODES,
|
|
621
|
+
default="auto",
|
|
622
|
+
help="JSON record schema filter (default: auto)",
|
|
623
|
+
)
|
|
624
|
+
|
|
625
|
+
p_sig = sub.add_parser("corroborate-timestamps", help="Verify timezone-aware UTC intervals and session metadata")
|
|
626
|
+
p_sig.add_argument("json_file", type=Path, help="Path to timestamp/session JSON")
|
|
627
|
+
|
|
628
|
+
p_img = sub.add_parser("inspect-image", help="Parse RFC 2083 PNG / ISO 10918 JPEG ICC & APP13 container metadata")
|
|
629
|
+
p_img.add_argument("image_file", type=Path, help="Path to PNG or JPEG image")
|
|
630
|
+
|
|
631
|
+
p_dur = sub.add_parser("summarize-linguistics", help="Compute descriptive utterance length & punctuation statistics")
|
|
632
|
+
p_dur.add_argument("json_file", type=Path, help="Path to chat transcript JSON")
|
|
633
|
+
|
|
634
|
+
args = parser.parse_args(argv)
|
|
635
|
+
redact = not args.unredacted
|
|
636
|
+
set_active_hmac_digest(args.hmac_digest)
|
|
637
|
+
|
|
638
|
+
try:
|
|
639
|
+
if args.hmac_key_file is not None:
|
|
640
|
+
raw_key = read_bounded_bytes(args.hmac_key_file, max_mb=1, redact=redact).strip()
|
|
641
|
+
if len(raw_key) < 16:
|
|
642
|
+
raise ValueError("External --hmac-key-file must contain at least 16 secret bytes")
|
|
643
|
+
run_salt = raw_key
|
|
644
|
+
else:
|
|
645
|
+
run_salt = secrets.token_bytes(32)
|
|
646
|
+
if args.command == "demo":
|
|
647
|
+
return run_demo(redact=redact)
|
|
648
|
+
|
|
649
|
+
if args.command == "precheck":
|
|
650
|
+
pre_report = inspect_chromium_storage_path(
|
|
651
|
+
args.target_path,
|
|
652
|
+
max_mb=args.max_mb,
|
|
653
|
+
schema_mode=args.schema,
|
|
654
|
+
redact=redact,
|
|
655
|
+
)
|
|
656
|
+
print(json.dumps({"precheck_manifest": pre_report}, indent=2, ensure_ascii=False))
|
|
657
|
+
return 0
|
|
658
|
+
|
|
659
|
+
if args.command == "parse-leveldb-log":
|
|
660
|
+
raw = read_bounded_bytes(args.log_file, args.max_mb, redact=redact)
|
|
661
|
+
parsed = apply_wal_record_redaction(
|
|
662
|
+
parse_leveldb_wal(raw, schema_mode=args.schema),
|
|
663
|
+
redact=redact,
|
|
664
|
+
salt=run_salt,
|
|
665
|
+
)
|
|
666
|
+
out = {
|
|
667
|
+
"acquisition_manifest": build_acquisition_manifest(raw, args.log_file.name, redact=redact),
|
|
668
|
+
"leveldb_wal_analysis": parsed,
|
|
669
|
+
}
|
|
670
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
671
|
+
return 0
|
|
672
|
+
|
|
673
|
+
if args.command == "triage-json":
|
|
674
|
+
raw = read_bounded_bytes(args.json_file, args.max_mb, redact=redact)
|
|
675
|
+
data = json.loads(raw.decode("utf-8"))
|
|
676
|
+
out = {
|
|
677
|
+
"acquisition_manifest": build_acquisition_manifest(raw, args.json_file.name, redact=redact),
|
|
678
|
+
"triage_result": triage_exported_chatsvc_json(
|
|
679
|
+
data,
|
|
680
|
+
redact=redact,
|
|
681
|
+
salt=run_salt,
|
|
682
|
+
schema_mode=args.schema,
|
|
683
|
+
),
|
|
684
|
+
}
|
|
685
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
686
|
+
return 0
|
|
687
|
+
|
|
688
|
+
if args.command == "corroborate-timestamps":
|
|
689
|
+
raw = read_bounded_bytes(args.json_file, args.max_mb, redact=redact)
|
|
690
|
+
data = json.loads(raw.decode("utf-8"))
|
|
691
|
+
out = {
|
|
692
|
+
"acquisition_manifest": build_acquisition_manifest(raw, args.json_file.name, redact=redact),
|
|
693
|
+
"corroboration": corroborate_timestamps(data, redact=redact, salt=run_salt),
|
|
694
|
+
}
|
|
695
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
696
|
+
return 0
|
|
697
|
+
|
|
698
|
+
if args.command == "inspect-image":
|
|
699
|
+
raw = read_bounded_bytes(args.image_file, args.max_mb, redact=redact)
|
|
700
|
+
label = "[REDACTED_IMAGE_LABEL]" if redact else args.image_file.name
|
|
701
|
+
out = {
|
|
702
|
+
"acquisition_manifest": build_acquisition_manifest(raw, args.image_file.name, redact=redact),
|
|
703
|
+
"image_container_analysis": apply_image_redaction(
|
|
704
|
+
inspect_image_container(raw, label),
|
|
705
|
+
redact=redact,
|
|
706
|
+
salt=run_salt,
|
|
707
|
+
),
|
|
708
|
+
}
|
|
709
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
710
|
+
return 0
|
|
711
|
+
|
|
712
|
+
if args.command == "summarize-linguistics":
|
|
713
|
+
raw = read_bounded_bytes(args.json_file, args.max_mb, redact=redact)
|
|
714
|
+
data = json.loads(raw.decode("utf-8"))
|
|
715
|
+
out = {
|
|
716
|
+
"acquisition_manifest": build_acquisition_manifest(raw, args.json_file.name, redact=redact),
|
|
717
|
+
"descriptive_linguistics": summarize_corpus_linguistics(data),
|
|
718
|
+
}
|
|
719
|
+
print(json.dumps(out, indent=2, ensure_ascii=False))
|
|
720
|
+
return 0
|
|
721
|
+
except (OSError, ValueError, TypeError, KeyError, UnicodeDecodeError) as exc:
|
|
722
|
+
if redact:
|
|
723
|
+
if isinstance(exc, UnicodeDecodeError):
|
|
724
|
+
safe_msg = "Input file is not valid UTF-8 text (raw byte context suppressed in masked mode)"
|
|
725
|
+
elif isinstance(exc, json.JSONDecodeError):
|
|
726
|
+
safe_msg = "Malformed JSON input (line/column and snippet suppressed in masked mode)"
|
|
727
|
+
else:
|
|
728
|
+
safe_msg = str(exc)
|
|
729
|
+
for attr in ("target_path", "log_file", "json_file", "image_file"):
|
|
730
|
+
cand_path = getattr(args, attr, None)
|
|
731
|
+
if isinstance(cand_path, Path):
|
|
732
|
+
safe_msg = safe_msg.replace(str(cand_path), "[REDACTED_PATH]")
|
|
733
|
+
safe_msg = safe_msg.replace(cand_path.name, "[REDACTED_FILENAME]")
|
|
734
|
+
sys.stderr.write(json.dumps({"error": safe_msg, "masked_mode": True}) + "\n")
|
|
735
|
+
else:
|
|
736
|
+
sys.stderr.write(json.dumps({"error": str(exc), "masked_mode": False}) + "\n")
|
|
737
|
+
return 2
|
|
738
|
+
|
|
739
|
+
return 1
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
if __name__ == "__main__":
|
|
743
|
+
raise SystemExit(main())
|