starshield-forensics 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
starshield/cli.py ADDED
@@ -0,0 +1,743 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ StarShield Forensics CLI v0.3.6 (`starshield`)
5
+ Author: Ziqi Xiao (@Ziqi)
6
+ License: MIT
7
+
8
+ Deterministic, strictly offline browser cache, image container, and financial
9
+ indicator triage toolkit for digital forensics and commercial payment-dispute review.
10
+
11
+ Every analytical claim is tagged with an explicit epistemic tier:
12
+ - OBSERVED : Directly present in raw input bytes/fields
13
+ - DERIVED : Deterministically computed from raw input (CRC32C, Luhn, Mod-97, deltas)
14
+ - INFERRED : Triage lead requiring external corroboration (never standalone proof)
15
+ - UNKNOWN : Outside the evidentiary boundary of local client artifacts
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import argparse
21
+ import hashlib
22
+ import hmac
23
+ import html
24
+ import importlib.resources
25
+ import json
26
+ import re
27
+ import secrets
28
+ import statistics
29
+ import sys
30
+ from datetime import datetime, timezone
31
+ from pathlib import Path
32
+ from typing import Any, Dict, List
33
+
34
+ from starshield import __version__
35
+ from starshield.image_containers import (
36
+ build_valid_jpeg_with_icc_and_8bim,
37
+ build_valid_png_with_iccp,
38
+ inspect_image_container,
39
+ )
40
+ from starshield.ioc_validators import (
41
+ SUPPORTED_HMAC_DIGESTS,
42
+ empty_ioc_dict,
43
+ extract_validated_iocs,
44
+ get_active_hmac_callable,
45
+ luhn_valid,
46
+ mask_identifier,
47
+ merge_ioc_dicts,
48
+ set_active_hmac_digest,
49
+ )
50
+ from starshield.leveldb_log import (
51
+ VALID_SCHEMA_MODES,
52
+ build_writebatch,
53
+ classify_carved_json_object,
54
+ encode_logical_record_across_blocks,
55
+ inspect_chromium_storage_path,
56
+ normalize_carved_record_fields,
57
+ parse_leveldb_wal,
58
+ )
59
+
60
+ MAX_DEFAULT_MB = 64
61
+
62
+
63
+ def sha256_bytes(data: bytes) -> str:
64
+ return hashlib.sha256(data).hexdigest()
65
+
66
+
67
+ def redact_timestamp(ts: Any) -> Any:
68
+ """Suppress exact dates and times in default masked mode to prevent timeline linkage."""
69
+ if ts is None or ts == "":
70
+ return None
71
+ epoch = parse_strict_utc_ts(ts)
72
+ if epoch is None:
73
+ return "[REDACTED_UNPARSED_TIMESTAMP]"
74
+ return "[REDACTED_UTC_TIMESTAMP]"
75
+
76
+
77
+ def redact_msg_id(msg_id: str, salt: bytes) -> str:
78
+ """Keyed per-run HMAC message token with configurable digest agility."""
79
+ if not msg_id:
80
+ return ""
81
+ digest = hmac.new(salt, msg_id.encode("utf-8"), get_active_hmac_callable()).hexdigest()[:12]
82
+ return f"msg_hmac_{digest}"
83
+
84
+
85
+ def build_acquisition_manifest(
86
+ raw: bytes,
87
+ source_label: str,
88
+ redact: bool = True,
89
+ operator: str = "unspecified",
90
+ ) -> Dict[str, Any]:
91
+ now_utc = datetime.now(timezone.utc)
92
+ return {
93
+ "source_label": "[REDACTED_SOURCE_LABEL]" if redact else source_label,
94
+ "input_byte_length": None if redact else len(raw),
95
+ "input_sha256": (
96
+ "[OMITTED_IN_MASKED_MODE — pass --unredacted in controlled lab for raw SHA-256]"
97
+ if redact
98
+ else sha256_bytes(raw)
99
+ ),
100
+ "processed_utc": now_utc.strftime("%Y-%m") if redact else now_utc.isoformat(),
101
+ "tool_version": f"starshield-forensics/{__version__}",
102
+ "operator": operator,
103
+ "redaction_notice": (
104
+ "Default masked mode applies ephemeral per-run HMAC-SHA256 keyed pseudonymization "
105
+ "(not formal legal anonymization under GDPR/UK GDPR) and suppresses raw file hashes, "
106
+ "exact byte lengths, exact UTC dates, filenames, image free-text chunks, and plaintext "
107
+ "bodies. Pass --unredacted inside authorized offline forensic environments only."
108
+ ),
109
+ "provenance_caveat": (
110
+ "Cryptographic hashes attest post-acquisition byte integrity only; upstream device "
111
+ "authenticity and completeness require an independent acquisition log."
112
+ ),
113
+ }
114
+
115
+
116
+ def read_bounded_bytes(path: Path, max_mb: int = MAX_DEFAULT_MB, redact: bool = True) -> bytes:
117
+ if max_mb < 1:
118
+ raise ValueError("--max-mb safety limit must be >= 1 MiB")
119
+ try:
120
+ size = path.stat().st_size
121
+ except OSError as exc:
122
+ if redact:
123
+ raise OSError(
124
+ "Unable to access input file (filename and OS path suppressed in masked mode; "
125
+ "pass --unredacted in an offline lab for path diagnostics)"
126
+ ) from None
127
+ raise exc
128
+ limit = max_mb * 1024 * 1024
129
+ if size > limit:
130
+ if redact:
131
+ raise ValueError(
132
+ f"Input file exceeds --max-mb={max_mb} safety limit "
133
+ "(filename and exact byte length suppressed in masked mode)"
134
+ )
135
+ raise ValueError(f"File {path.name} ({size} bytes) exceeds --max-mb={max_mb} safety limit")
136
+ try:
137
+ return path.read_bytes()
138
+ except OSError as exc:
139
+ if redact:
140
+ raise OSError(
141
+ "Unable to read input file bytes (filename and OS path suppressed in masked mode; "
142
+ "pass --unredacted in an offline lab for path diagnostics)"
143
+ ) from None
144
+ raise exc
145
+
146
+
147
+ def strip_markup(raw: str) -> str:
148
+ if not raw:
149
+ return ""
150
+ cleaned = re.sub(r"<br\s*/?>", "\n", str(raw), flags=re.IGNORECASE)
151
+ cleaned = re.sub(r"<[^>]+>", " ", cleaned)
152
+ cleaned = html.unescape(cleaned)
153
+ return re.sub(r"[ \t]+", " ", cleaned).strip()
154
+
155
+
156
+ def parse_strict_utc_ts(ts: Any) -> float | None:
157
+ """Parse epoch ms/s or explicit timezone-aware ISO-8601. Rejects naive local timestamps."""
158
+ if ts is None or ts == "":
159
+ return None
160
+ if isinstance(ts, (int, float)):
161
+ val = float(ts)
162
+ return val / 1000.0 if val > 1e11 else val
163
+ s = str(ts).strip()
164
+ if re.fullmatch(r"\d+(?:\.\d+)?", s):
165
+ val = float(s)
166
+ return val / 1000.0 if val > 1e11 else val
167
+ try:
168
+ normalized = s.replace("Z", "+00:00")
169
+ dt = datetime.fromisoformat(normalized)
170
+ if dt.tzinfo is None:
171
+ return None
172
+ return dt.astimezone(timezone.utc).timestamp()
173
+ except ValueError:
174
+ return None
175
+
176
+
177
+ def apply_wal_record_redaction(
178
+ wal_parsed: Dict[str, Any],
179
+ redact: bool = True,
180
+ salt: bytes | None = None,
181
+ ) -> Dict[str, Any]:
182
+ run_salt = salt if salt is not None else secrets.token_bytes(32)
183
+ if redact:
184
+ wal_parsed["file_size_bytes"] = None
185
+ for rec in wal_parsed["recovered_records"]:
186
+ raw_txt = rec["cached_content"]
187
+ rec["extracted_iocs"] = extract_validated_iocs(raw_txt, redact=redact, salt=run_salt)
188
+ if redact:
189
+ rec["key_sha256"] = "[OMITTED_IN_MASKED_MODE]"
190
+ rec["value_sha256"] = "[OMITTED_IN_MASKED_MODE]"
191
+ rec["message_id"] = redact_msg_id(rec["message_id"], salt=run_salt)
192
+ rec["sender_mri"] = mask_identifier(rec["sender_mri"], salt=run_salt)
193
+ rec["composetime"] = redact_timestamp(rec["composetime"])
194
+ rec["deletetime"] = redact_timestamp(rec["deletetime"])
195
+ rec["edittime"] = redact_timestamp(rec["edittime"])
196
+ rec["cached_content"] = "[REDACTED_CONTENT — pass --unredacted to view]"
197
+ return wal_parsed
198
+
199
+
200
+ _STANDARD_PNG_CHUNKS = frozenset(
201
+ {
202
+ "IHDR", "PLTE", "IDAT", "IEND", "iCCP", "sRGB", "gAMA", "cHRM",
203
+ "pHYs", "tEXt", "zTXt", "iTXt", "tIME", "bKGD", "tRNS", "sBIT",
204
+ "hIST", "sPLT", "eXIf",
205
+ }
206
+ )
207
+
208
+
209
+ def apply_image_redaction(
210
+ img_report: Dict[str, Any],
211
+ redact: bool = True,
212
+ salt: bytes | None = None,
213
+ ) -> Dict[str, Any]:
214
+ """Suppress PNG free-text software chunk previews, custom chunk tags, and custom ICC names in default masked mode."""
215
+ if not redact:
216
+ return img_report
217
+ run_salt = salt if salt is not None else secrets.token_bytes(32)
218
+ img_report["artifact"] = "[REDACTED_IMAGE_LABEL]"
219
+ for ch in img_report.get("chunks_summary", []):
220
+ if isinstance(ch, dict):
221
+ if "chunk" in ch and ch.get("chunk") not in _STANDARD_PNG_CHUNKS:
222
+ ch["chunk"] = "[CUSTOM_PNG_CHUNK]"
223
+ if "type" in ch and ch.get("type") not in _STANDARD_PNG_CHUNKS:
224
+ ch["type"] = "[CUSTOM_PNG_CHUNK]"
225
+ for cerr in img_report.get("crc_errors", []):
226
+ if isinstance(cerr, dict):
227
+ if "chunk" in cerr and cerr.get("chunk") not in _STANDARD_PNG_CHUNKS:
228
+ cerr["chunk"] = "[CUSTOM_PNG_CHUNK]"
229
+ if "chunk_type" in cerr and cerr.get("chunk_type") not in _STANDARD_PNG_CHUNKS:
230
+ cerr["chunk_type"] = "[CUSTOM_PNG_CHUNK]"
231
+ for prof in img_report.get("icc_profiles", []):
232
+ if not prof.get("valid_icc_header") and "reason" in prof:
233
+ prof["reason"] = "Invalid or non-ISO-15076-1 ICC profile header (raw signature bytes suppressed in masked mode)"
234
+ if prof.get("png_iccp_name"):
235
+ prof["png_iccp_name"] = mask_identifier(str(prof["png_iccp_name"]), salt=run_salt)
236
+ if prof.get("profile_creator"):
237
+ prof["profile_creator"] = mask_identifier(str(prof["profile_creator"]), salt=run_salt)
238
+ for tag in img_report.get("software_text_chunks", []):
239
+ tag["preview"] = "[REDACTED_TEXT_METADATA — pass --unredacted to view]"
240
+ return img_report
241
+
242
+
243
+ def triage_exported_chatsvc_json(
244
+ payload: Dict[str, Any],
245
+ redact: bool = True,
246
+ salt: bytes | None = None,
247
+ schema_mode: str = "auto",
248
+ ) -> Dict[str, Any]:
249
+ if schema_mode not in VALID_SCHEMA_MODES:
250
+ raise ValueError(f"Unsupported schema_mode={schema_mode!r}; expected one of {VALID_SCHEMA_MODES}")
251
+ if not isinstance(payload, dict):
252
+ raise ValueError("Expected JSON object containing a 'messages' or 'records' list")
253
+
254
+ if "messages" in payload:
255
+ messages = payload.get("messages")
256
+ if not isinstance(messages, list):
257
+ raise ValueError("Expected 'messages' to be a JSON list")
258
+ elif "records" in payload:
259
+ messages = payload.get("records")
260
+ if not isinstance(messages, list):
261
+ raise ValueError("Expected 'records' to be a JSON list")
262
+ else:
263
+ messages = []
264
+
265
+ run_salt = salt if salt is not None else secrets.token_bytes(32)
266
+ retracted: List[Dict[str, Any]] = []
267
+ edited: List[Dict[str, Any]] = []
268
+ corpus_iocs = empty_ioc_dict()
269
+
270
+ for msg in messages:
271
+ if not isinstance(msg, dict):
272
+ continue
273
+ schema_tag = classify_carved_json_object(msg, schema_mode=schema_mode)
274
+ if schema_mode == "chatsvc" and schema_tag != "chatsvc":
275
+ continue
276
+ norm = normalize_carved_record_fields(msg, schema_tag or "generic_json")
277
+ body = strip_markup(norm["cached_content"])
278
+ # Extract IOCs strictly PER MESSAGE so cross-message numbers never pair!
279
+ msg_iocs = extract_validated_iocs(body, redact=redact, salt=run_salt)
280
+ merge_ioc_dicts(corpus_iocs, msg_iocs)
281
+
282
+ delete_ts = norm["deletetime"]
283
+ edit_ts = norm["edittime"]
284
+ raw_id = norm["message_id"]
285
+ rendered_id = redact_msg_id(raw_id, salt=run_salt) if redact else raw_id
286
+ sender = norm["sender_mri"] or "unknown"
287
+ rendered_sender = mask_identifier(sender, salt=run_salt) if redact else sender
288
+ comp_ts = norm["composetime"]
289
+
290
+ if delete_ts:
291
+ retracted.append(
292
+ {
293
+ "epistemic_tier": "OBSERVED",
294
+ "schema_detected": schema_tag or "generic_json",
295
+ "id": rendered_id,
296
+ "sender_mri": rendered_sender,
297
+ "compose_time": redact_timestamp(comp_ts) if redact else comp_ts,
298
+ "delete_time": redact_timestamp(delete_ts) if redact else delete_ts,
299
+ "cached_plaintext": "[REDACTED_CONTENT — pass --unredacted to view]" if redact else body,
300
+ "extracted_iocs": msg_iocs,
301
+ }
302
+ )
303
+ if edit_ts:
304
+ edited.append(
305
+ {
306
+ "epistemic_tier": "OBSERVED",
307
+ "schema_detected": schema_tag or "generic_json",
308
+ "id": rendered_id,
309
+ "sender_mri": rendered_sender,
310
+ "compose_time": redact_timestamp(comp_ts) if redact else comp_ts,
311
+ "edit_time": redact_timestamp(edit_ts) if redact else edit_ts,
312
+ "limitation": "Exported JSON retains post-edit content only; pre-edit versions require uncompacted LevelDB sequence history.",
313
+ }
314
+ )
315
+
316
+ return {
317
+ "input_mode": "exported_message_json",
318
+ "schema_mode": schema_mode,
319
+ "messages_present_in_input": len(messages),
320
+ "soft_deleted_messages_in_input": len(retracted),
321
+ "edited_messages_in_input": len(edited),
322
+ "completeness_caveat": (
323
+ "Counts reflect records present in the supplied export only; already compacted "
324
+ "or garbage-collected LevelDB records cannot be enumerated from client cache alone."
325
+ ),
326
+ "soft_deleted_records": retracted,
327
+ "edited_records": edited,
328
+ "corpus_validated_iocs": corpus_iocs,
329
+ }
330
+
331
+
332
+ def corroborate_timestamps(
333
+ payload: Dict[str, Any],
334
+ redact: bool = True,
335
+ salt: bytes | None = None,
336
+ ) -> Dict[str, Any]:
337
+ """Verify timezone-aware UTC event ordering and clock-skew bounds across generic event pairs."""
338
+ if not isinstance(payload, dict):
339
+ raise ValueError("Expected top-level JSON object for timestamp corroboration")
340
+ run_salt = salt if salt is not None else secrets.token_bytes(32)
341
+ event_pairs = payload.get("timestamp_pairs")
342
+ if event_pairs is None:
343
+ event_pairs = []
344
+ elif not isinstance(event_pairs, list):
345
+ raise ValueError("Expected 'timestamp_pairs' to be a JSON list")
346
+
347
+ routing_tags = payload.get("session_metadata")
348
+ if routing_tags is None:
349
+ routing_tags = []
350
+ elif not isinstance(routing_tags, list):
351
+ raise ValueError("Expected 'session_metadata' to be a JSON list")
352
+
353
+ matrix: List[Dict[str, Any]] = []
354
+ seen_regions: Dict[str, int] = {}
355
+ for ev in routing_tags:
356
+ if not isinstance(ev, dict):
357
+ continue
358
+ tag = str(ev.get("edge_pop_tag", "")).strip()
359
+ if tag:
360
+ rendered_tag = mask_identifier(tag, salt=run_salt) if redact else tag
361
+ seen_regions[rendered_tag] = seen_regions.get(rendered_tag, 0) + 1
362
+
363
+ for r_tag, count in sorted(seen_regions.items(), key=lambda x: -x[1]):
364
+ matrix.append(
365
+ {
366
+ "observation": f"Infrastructure edge tag {r_tag} observed across {count} session record(s).",
367
+ "provenance": "session_metadata[].edge_pop_tag",
368
+ "epistemic_tier": "OBSERVED",
369
+ "alternative_explanations": [
370
+ "Edge PoP tags reflect CDN or media-relay latency routing, NOT confirmed physical endpoint geolocation.",
371
+ "Repeated events within a single persistent session are correlated rather than statistically independent.",
372
+ ],
373
+ }
374
+ )
375
+
376
+ intervals: List[Dict[str, Any]] = []
377
+ for idx, pair in enumerate(event_pairs):
378
+ if not isinstance(pair, dict):
379
+ continue
380
+ raw_label = str(pair.get("label", f"pair_{idx + 1}"))
381
+ label = f"pair_{idx + 1}" if redact else raw_label
382
+ t_start = parse_strict_utc_ts(pair.get("start_utc"))
383
+ t_end = parse_strict_utc_ts(pair.get("end_utc"))
384
+ if t_start is None or t_end is None:
385
+ intervals.append(
386
+ {
387
+ "label": label,
388
+ "raw_label": None if redact else raw_label,
389
+ "status": "INVALID_OR_TIMEZONE_NAIVE_TIMESTAMP",
390
+ "epistemic_tier": "UNKNOWN",
391
+ }
392
+ )
393
+ continue
394
+ delta = round(t_end - t_start, 3)
395
+ if delta < 0:
396
+ intervals.append(
397
+ {
398
+ "label": label,
399
+ "raw_label": None if redact else raw_label,
400
+ "delta_seconds": None if redact else delta,
401
+ "status": "REJECTED_NEGATIVE_DELTA_CLOCK_SKEW_OR_BAD_INPUT",
402
+ "epistemic_tier": "UNKNOWN",
403
+ }
404
+ )
405
+ else:
406
+ intervals.append(
407
+ {
408
+ "label": label,
409
+ "raw_label": None if redact else raw_label,
410
+ "delta_seconds": "[REDACTED_INTERVAL]" if redact else delta,
411
+ "status": "DERIVED_NON_NEGATIVE_INTERVAL",
412
+ "epistemic_tier": "DERIVED",
413
+ "caveat": "Cross-system timestamp intervals depend on NTP clock synchronization across endpoints.",
414
+ }
415
+ )
416
+
417
+ return {
418
+ "corroboration_matrix": matrix,
419
+ "timestamp_interval_observations": intervals,
420
+ }
421
+
422
+
423
+ def summarize_corpus_linguistics(payload: Dict[str, Any]) -> Dict[str, Any]:
424
+ """Compute strictly descriptive utterance length & punctuation statistics (ZERO clinical/legal diagnosis)."""
425
+ if not isinstance(payload, dict):
426
+ raise ValueError("Expected top-level JSON object for corpus linguistics summary")
427
+ messages = payload.get("messages")
428
+ if messages is None:
429
+ messages = []
430
+ elif not isinstance(messages, list):
431
+ raise ValueError("Expected 'messages' to be a JSON list")
432
+ subject_id = payload.get("subject_sender_id", "user:accounts_payable_02")
433
+
434
+ utterances = [
435
+ strip_markup(m.get("content") or m.get("text") or m.get("body") or "")
436
+ for m in messages
437
+ if isinstance(m, dict)
438
+ and (m.get("from") == subject_id or m.get("user") == subject_id or m.get("sender") == subject_id)
439
+ and strip_markup(m.get("content") or m.get("text") or m.get("body") or "")
440
+ ]
441
+
442
+ n = len(utterances)
443
+ lengths = [len(u) for u in utterances]
444
+ in_3_to_12 = [u for u in utterances if 3 <= len(u) <= 12]
445
+ no_mid_punct = [
446
+ u for u in utterances if not re.search(r"[,.;:!?,。;:!?]", u[:-1] if len(u) > 1 else u)
447
+ ]
448
+
449
+ return {
450
+ "epistemic_tier": "DERIVED",
451
+ "sample_size": n,
452
+ "minimum_recommended_sample_met": n >= 30,
453
+ "median_char_length": round(statistics.median(lengths), 2) if lengths else None,
454
+ "utterance_ratio_3_to_12_chars": round(len(in_3_to_12) / n, 4) if n else None,
455
+ "zero_mid_punctuation_ratio": round(len(no_mid_punct) / n, 4) if n else None,
456
+ "methodological_limitation": (
457
+ "Descriptive text statistics only. Short, unpunctuated replies are routine in "
458
+ "workplace and mobile messaging and never establish psychological state or legal vulnerability."
459
+ ),
460
+ }
461
+
462
+
463
+ def build_synthetic_leveldb_wal_bytes() -> bytes:
464
+ """
465
+ Create a clean-room, specification-only LevelDB `.log` test vector emulating Chromium
466
+ IndexedDB binary prefixes (`\\xff\\x15` Blink header) + cross-32KiB `FIRST/MIDDLE/LAST` blocks.
467
+ Uses standard RFC/ISO textbook test values from a fictional 2025 UK supplier-invoice scenario.
468
+ """
469
+ blink_prefix = b"\xff\x15\xfe\x00\x22"
470
+ msg1 = blink_prefix + json.dumps(
471
+ {
472
+ "id": "msg_spec_0001",
473
+ "from": "user:impersonated_supplier_01",
474
+ "composetime": "2025-03-18T10:00:00Z",
475
+ "content": (
476
+ "Updated UK Faster Payments supplier account for Invoice INV-2025-0100 (GBP 18,400.00): "
477
+ "Sort Code 00-00-00 Account 00000000."
478
+ ),
479
+ "properties": {
480
+ "deletetime": "1742292600000",
481
+ "nested_meta": {"schema_hint": "chromium_idb_spec_vector"},
482
+ },
483
+ }
484
+ ).encode("utf-8")
485
+
486
+ # Standard ISO/IEC 7812 Luhn test vector prefix (620000000000000x -> 6200000000000005)
487
+ prefix = "620000000000000"
488
+ valid_test_card = next(prefix + d for d in "0123456789" if luhn_valid(prefix + d))
489
+ # Pad second WriteBatch (>66 KiB) so it spans 3 physical 32KiB blocks via FIRST + MIDDLE + LAST
490
+ padding_blob = "X" * 68000
491
+ msg2 = blink_prefix + json.dumps(
492
+ {
493
+ "id": "msg_spec_0002",
494
+ "from": "user:impersonated_supplier_01",
495
+ "composetime": "2025-03-18T14:30:00Z",
496
+ "content": (
497
+ f"Secondary reserve IBAN GB82WEST12345698765432 (SWIFT NWBKGB2LXXX), "
498
+ f"Norway test IBAN NO9386011117947, and Luhn test card {valid_test_card}."
499
+ ),
500
+ "properties": {"deletetime": "1742308800000", "blob_pad": padding_blob},
501
+ }
502
+ ).encode("utf-8")
503
+
504
+ wb1 = build_writebatch(1001, [(1, b"\x00\x01\x02\x01idb:msg:spec_0001", msg1)])
505
+ wb2 = build_writebatch(1002, [(1, b"\x00\x01\x02\x01idb:msg:spec_0002", msg2)])
506
+
507
+ part1 = encode_logical_record_across_blocks(wb1, initial_block_used=0)
508
+ part2 = encode_logical_record_across_blocks(wb2, initial_block_used=len(part1))
509
+ return part1 + part2
510
+
511
+
512
+ def run_demo(redact: bool = True) -> int:
513
+ run_salt = secrets.token_bytes(32)
514
+ try:
515
+ sample_json_bytes = (
516
+ importlib.resources.files("starshield.data")
517
+ .joinpath("synthetic_case_benchmark.json")
518
+ .read_bytes()
519
+ )
520
+ except Exception:
521
+ repo_sample = Path(__file__).resolve().parent.parent / "samples" / "synthetic_case_benchmark.json"
522
+ sample_json_bytes = repo_sample.read_bytes()
523
+ payload = json.loads(sample_json_bytes.decode("utf-8"))
524
+
525
+ wal_bytes = build_synthetic_leveldb_wal_bytes()
526
+ wal_parsed = apply_wal_record_redaction(parse_leveldb_wal(wal_bytes), redact=redact, salt=run_salt)
527
+
528
+ valid_png = build_valid_png_with_iccp(
529
+ "Display P3",
530
+ "APPL",
531
+ software_text="Software\x00QuickShot Raster Editor 4.2",
532
+ )
533
+ valid_jpeg = build_valid_jpeg_with_icc_and_8bim("Display P3", "APPL", split_icc_chunks=2)
534
+
535
+ report = {
536
+ "toolkit": f"starshield-forensics v{__version__} (Deterministic Chromium Cache & Container Provenance Toolkit)",
537
+ "author": "Ziqi Xiao (@Ziqi)",
538
+ "dataset_notice": "100% CLEAN-ROOM SPECIFICATION TEST VECTORS (2025 UK SUPPLIER INVOICE SCENARIO — TLP:CLEAR)",
539
+ "acquisition_manifest": build_acquisition_manifest(
540
+ sample_json_bytes,
541
+ "synthetic_case_benchmark.json",
542
+ redact=redact,
543
+ ),
544
+ "engine_1_raw_leveldb_wal_carver": wal_parsed,
545
+ "engine_2_exported_json_triage": triage_exported_chatsvc_json(payload, redact=redact, salt=run_salt),
546
+ "engine_3_timestamp_and_session_corroboration": corroborate_timestamps(payload, redact=redact, salt=run_salt),
547
+ "engine_4a_structural_png_metadata": apply_image_redaction(
548
+ inspect_image_container(valid_png, "spec_valid_iccp_sample.png"),
549
+ redact=redact,
550
+ salt=run_salt,
551
+ ),
552
+ "engine_4b_structural_jpeg_metadata": apply_image_redaction(
553
+ inspect_image_container(valid_jpeg, "spec_valid_app2_8bim_sample.jpg"),
554
+ redact=redact,
555
+ salt=run_salt,
556
+ ),
557
+ "engine_5_descriptive_linguistics": summarize_corpus_linguistics(payload),
558
+ }
559
+ print(json.dumps(report, indent=2, ensure_ascii=False))
560
+ return 0
561
+
562
+
563
+ def main(argv: List[str] | None = None) -> int:
564
+ parser = argparse.ArgumentParser(
565
+ prog="starshield",
566
+ description=f"StarShield Forensics v{__version__} — Deterministic Chromium Cache & Container Triage CLI",
567
+ )
568
+ parser.add_argument(
569
+ "--unredacted",
570
+ action="store_true",
571
+ help="Include unmasked identifiers, raw SHA-256, exact timestamps, and plaintext bodies (offline lab use only)",
572
+ )
573
+ parser.add_argument(
574
+ "--max-mb",
575
+ type=int,
576
+ default=MAX_DEFAULT_MB,
577
+ help=f"Maximum input file size in MiB (default: {MAX_DEFAULT_MB})",
578
+ )
579
+ parser.add_argument(
580
+ "--hmac-digest",
581
+ choices=tuple(SUPPORTED_HMAC_DIGESTS.keys()),
582
+ default="sha256",
583
+ help="HMAC digest algorithm for keyed pseudonymization agility (default: sha256)",
584
+ )
585
+ parser.add_argument(
586
+ "--hmac-key-file",
587
+ type=Path,
588
+ default=None,
589
+ help="Optional external secret key file (>=16 bytes) for cross-run pseudonymization without hardcoded keys",
590
+ )
591
+ sub = parser.add_subparsers(dest="command", required=True)
592
+
593
+ sub.add_parser("demo", help="Run deterministic triage pipeline on clean-room specification test vectors")
594
+
595
+ p_pre = sub.add_parser(
596
+ "precheck",
597
+ help="Zero-PII local GO/PARTIAL/NO-GO feasibility check on a Chromium storage folder or .log/.ldb file",
598
+ )
599
+ p_pre.add_argument("target_path", type=Path, help="Read-only Chromium IndexedDB/LevelDB folder or .log/.ldb file")
600
+ p_pre.add_argument(
601
+ "--schema",
602
+ choices=VALID_SCHEMA_MODES,
603
+ default="auto",
604
+ help="JSON record schema filter (default: auto — supports Teams/Skype, Slack, WhatsApp Web & B2B portals)",
605
+ )
606
+
607
+ p_wal = sub.add_parser("parse-leveldb-log", help="Parse Chromium IndexedDB LevelDB .log file with byte offsets & CRC32C")
608
+ p_wal.add_argument("log_file", type=Path, help="Read-only copy of LevelDB .log write-ahead log")
609
+ p_wal.add_argument(
610
+ "--schema",
611
+ choices=VALID_SCHEMA_MODES,
612
+ default="auto",
613
+ help="JSON record schema filter (default: auto — supports Teams/Skype, Slack, WhatsApp Web & B2B portals)",
614
+ )
615
+
616
+ p_rec = sub.add_parser("triage-json", help="Triage exported message JSON for soft-deleted records & validated IOCs")
617
+ p_rec.add_argument("json_file", type=Path, help="Path to exported message JSON")
618
+ p_rec.add_argument(
619
+ "--schema",
620
+ choices=VALID_SCHEMA_MODES,
621
+ default="auto",
622
+ help="JSON record schema filter (default: auto)",
623
+ )
624
+
625
+ p_sig = sub.add_parser("corroborate-timestamps", help="Verify timezone-aware UTC intervals and session metadata")
626
+ p_sig.add_argument("json_file", type=Path, help="Path to timestamp/session JSON")
627
+
628
+ p_img = sub.add_parser("inspect-image", help="Parse RFC 2083 PNG / ISO 10918 JPEG ICC & APP13 container metadata")
629
+ p_img.add_argument("image_file", type=Path, help="Path to PNG or JPEG image")
630
+
631
+ p_dur = sub.add_parser("summarize-linguistics", help="Compute descriptive utterance length & punctuation statistics")
632
+ p_dur.add_argument("json_file", type=Path, help="Path to chat transcript JSON")
633
+
634
+ args = parser.parse_args(argv)
635
+ redact = not args.unredacted
636
+ set_active_hmac_digest(args.hmac_digest)
637
+
638
+ try:
639
+ if args.hmac_key_file is not None:
640
+ raw_key = read_bounded_bytes(args.hmac_key_file, max_mb=1, redact=redact).strip()
641
+ if len(raw_key) < 16:
642
+ raise ValueError("External --hmac-key-file must contain at least 16 secret bytes")
643
+ run_salt = raw_key
644
+ else:
645
+ run_salt = secrets.token_bytes(32)
646
+ if args.command == "demo":
647
+ return run_demo(redact=redact)
648
+
649
+ if args.command == "precheck":
650
+ pre_report = inspect_chromium_storage_path(
651
+ args.target_path,
652
+ max_mb=args.max_mb,
653
+ schema_mode=args.schema,
654
+ redact=redact,
655
+ )
656
+ print(json.dumps({"precheck_manifest": pre_report}, indent=2, ensure_ascii=False))
657
+ return 0
658
+
659
+ if args.command == "parse-leveldb-log":
660
+ raw = read_bounded_bytes(args.log_file, args.max_mb, redact=redact)
661
+ parsed = apply_wal_record_redaction(
662
+ parse_leveldb_wal(raw, schema_mode=args.schema),
663
+ redact=redact,
664
+ salt=run_salt,
665
+ )
666
+ out = {
667
+ "acquisition_manifest": build_acquisition_manifest(raw, args.log_file.name, redact=redact),
668
+ "leveldb_wal_analysis": parsed,
669
+ }
670
+ print(json.dumps(out, indent=2, ensure_ascii=False))
671
+ return 0
672
+
673
+ if args.command == "triage-json":
674
+ raw = read_bounded_bytes(args.json_file, args.max_mb, redact=redact)
675
+ data = json.loads(raw.decode("utf-8"))
676
+ out = {
677
+ "acquisition_manifest": build_acquisition_manifest(raw, args.json_file.name, redact=redact),
678
+ "triage_result": triage_exported_chatsvc_json(
679
+ data,
680
+ redact=redact,
681
+ salt=run_salt,
682
+ schema_mode=args.schema,
683
+ ),
684
+ }
685
+ print(json.dumps(out, indent=2, ensure_ascii=False))
686
+ return 0
687
+
688
+ if args.command == "corroborate-timestamps":
689
+ raw = read_bounded_bytes(args.json_file, args.max_mb, redact=redact)
690
+ data = json.loads(raw.decode("utf-8"))
691
+ out = {
692
+ "acquisition_manifest": build_acquisition_manifest(raw, args.json_file.name, redact=redact),
693
+ "corroboration": corroborate_timestamps(data, redact=redact, salt=run_salt),
694
+ }
695
+ print(json.dumps(out, indent=2, ensure_ascii=False))
696
+ return 0
697
+
698
+ if args.command == "inspect-image":
699
+ raw = read_bounded_bytes(args.image_file, args.max_mb, redact=redact)
700
+ label = "[REDACTED_IMAGE_LABEL]" if redact else args.image_file.name
701
+ out = {
702
+ "acquisition_manifest": build_acquisition_manifest(raw, args.image_file.name, redact=redact),
703
+ "image_container_analysis": apply_image_redaction(
704
+ inspect_image_container(raw, label),
705
+ redact=redact,
706
+ salt=run_salt,
707
+ ),
708
+ }
709
+ print(json.dumps(out, indent=2, ensure_ascii=False))
710
+ return 0
711
+
712
+ if args.command == "summarize-linguistics":
713
+ raw = read_bounded_bytes(args.json_file, args.max_mb, redact=redact)
714
+ data = json.loads(raw.decode("utf-8"))
715
+ out = {
716
+ "acquisition_manifest": build_acquisition_manifest(raw, args.json_file.name, redact=redact),
717
+ "descriptive_linguistics": summarize_corpus_linguistics(data),
718
+ }
719
+ print(json.dumps(out, indent=2, ensure_ascii=False))
720
+ return 0
721
+ except (OSError, ValueError, TypeError, KeyError, UnicodeDecodeError) as exc:
722
+ if redact:
723
+ if isinstance(exc, UnicodeDecodeError):
724
+ safe_msg = "Input file is not valid UTF-8 text (raw byte context suppressed in masked mode)"
725
+ elif isinstance(exc, json.JSONDecodeError):
726
+ safe_msg = "Malformed JSON input (line/column and snippet suppressed in masked mode)"
727
+ else:
728
+ safe_msg = str(exc)
729
+ for attr in ("target_path", "log_file", "json_file", "image_file"):
730
+ cand_path = getattr(args, attr, None)
731
+ if isinstance(cand_path, Path):
732
+ safe_msg = safe_msg.replace(str(cand_path), "[REDACTED_PATH]")
733
+ safe_msg = safe_msg.replace(cand_path.name, "[REDACTED_FILENAME]")
734
+ sys.stderr.write(json.dumps({"error": safe_msg, "masked_mode": True}) + "\n")
735
+ else:
736
+ sys.stderr.write(json.dumps({"error": str(exc), "masked_mode": False}) + "\n")
737
+ return 2
738
+
739
+ return 1
740
+
741
+
742
+ if __name__ == "__main__":
743
+ raise SystemExit(main())