starshield-forensics 0.3.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,795 @@
1
+ #!/usr/bin/env python3
2
+ # -*- coding: utf-8 -*-
3
+ """
4
+ Google LevelDB Write-Ahead Log (`.log`) Binary Frame, Multi-Schema JSON Carver
5
+ & Local Storage Precheck Gate for Chromium IndexedDB / Web Application Caches.
6
+
7
+ Implements the official LevelDB log format specification:
8
+ - 32,768-byte (32 KiB) physical block boundaries with strict zero-padding trailer checks
9
+ - 7-byte record header: [masked_crc32c: uint32_le][length: uint16_le][type: uint8]
10
+ where type in {1: FULL, 2: FIRST, 3: MIDDLE, 4: LAST}
11
+ - Castagnoli CRC32C (polynomial 0x82F63B78) + LevelDB mask delta 0xA282EAD8
12
+ - Strict LevelDB WriteBatch validation: [sequence: uint64_le][count: uint32_le] followed
13
+ by varint-prefixed kTypeValue (0x01) and kTypeDeletion (0x00) entries, rejecting
14
+ truncated payloads, op-count mismatches, trailing garbage bytes, and unclosed EOF fragments.
15
+ - Top-level `json.JSONDecoder.raw_decode` scanning across `chatsvc` (Teams/Skype), Slack Web,
16
+ WhatsApp/Web Messenger, and B2B supplier/invoice portal JSON schemas (`--schema auto`).
17
+ - Offline `inspect_chromium_storage_path` feasibility precheck (`GO` / `PARTIAL` /
18
+ `NO_GO_COMPACTED_OR_UNSUPPORTED`) detecting `.ldb` SSTable footers (`0xdb4775248b80fb57`),
19
+ IndexedDB `.blob` directories, and Blink V8 `0xFF` headers with zero PII leakage.
20
+
21
+ Scope & Honest Limitations:
22
+ - Targets uncompressed LevelDB `.log` write-ahead logs and embedded UTF-8/UTF-16LE JSON
23
+ slices inside Chromium IndexedDB / Local Storage WriteBatch values.
24
+ - Does NOT decompress Snappy-compressed `.ldb` SSTable data blocks or decode binary-only
25
+ Blink V8 StructuredClone primitives (scheduled for v0.4).
26
+ """
27
+
28
+ from __future__ import annotations
29
+
30
+ import hashlib
31
+ import json
32
+ import struct
33
+ from pathlib import Path
34
+ from typing import Any, Dict, List, Tuple
35
+
36
+ BLOCK_SIZE = 32768
37
+ HEADER_SIZE = 7
38
+ MAX_LOGICAL_RECORD_BYTES = 32 * 1024 * 1024 # 32 MiB anti-DoS cap per reassembled batch
39
+ LEVELDB_SSTABLE_MAGIC_LE = struct.pack("<Q", 0xDB4775248B80FB57)
40
+
41
+ REC_ZERO = 0
42
+ REC_FULL = 1
43
+ REC_FIRST = 2
44
+ REC_MIDDLE = 3
45
+ REC_LAST = 4
46
+
47
+ MASK_DELTA = 0xA282EAD8
48
+
49
+ VALID_SCHEMA_MODES = ("auto", "chatsvc", "generic-json")
50
+
51
+ _PORTAL_ID_KEYS = (
52
+ "id",
53
+ "message_id",
54
+ "msg_id",
55
+ "client_msg_id",
56
+ "invoice_id",
57
+ "payment_id",
58
+ "tx_id",
59
+ "transaction_id",
60
+ "reference",
61
+ "uuid",
62
+ "record_id",
63
+ "event_id",
64
+ "order_id",
65
+ )
66
+
67
+ _PORTAL_CONTENT_KEYS = (
68
+ "content",
69
+ "text",
70
+ "body",
71
+ "message",
72
+ "memo",
73
+ "note",
74
+ "description",
75
+ "payment_instructions",
76
+ "iban",
77
+ "beneficiary_iban",
78
+ "account_number",
79
+ "sort_code",
80
+ "beneficiary",
81
+ "payee",
82
+ "amount",
83
+ )
84
+
85
+
86
+ def _build_crc32c_table() -> List[int]:
87
+ poly = 0x82F63B78
88
+ table: List[int] = []
89
+ for i in range(256):
90
+ crc = i
91
+ for _ in range(8):
92
+ if crc & 1:
93
+ crc = (crc >> 1) ^ poly
94
+ else:
95
+ crc >>= 1
96
+ table.append(crc)
97
+ return table
98
+
99
+
100
+ _CRC32C_TABLE = _build_crc32c_table()
101
+
102
+
103
+ def crc32c(data: bytes) -> int:
104
+ crc = 0xFFFFFFFF
105
+ for b in data:
106
+ crc = _CRC32C_TABLE[(crc ^ b) & 0xFF] ^ (crc >> 8)
107
+ return crc ^ 0xFFFFFFFF
108
+
109
+
110
+ def mask_crc32c(crc: int) -> int:
111
+ return (((crc >> 15) | ((crc << 17) & 0xFFFFFFFF)) + MASK_DELTA) & 0xFFFFFFFF
112
+
113
+
114
+ def encode_leveldb_log_record(rec_type: int, payload: bytes) -> bytes:
115
+ if len(payload) > 0xFFFF:
116
+ raise ValueError("Single physical LevelDB log record payload cannot exceed 65535 bytes")
117
+ raw_crc = crc32c(bytes([rec_type]) + payload)
118
+ masked = mask_crc32c(raw_crc)
119
+ return struct.pack("<IHB", masked, len(payload), rec_type) + payload
120
+
121
+
122
+ def encode_logical_record_across_blocks(payload: bytes, initial_block_used: int = 0) -> bytes:
123
+ """Split a logical WriteBatch across 32KiB physical LevelDB blocks using FULL/FIRST/MIDDLE/LAST."""
124
+ out = bytearray()
125
+ rem_in_block = BLOCK_SIZE - (initial_block_used % BLOCK_SIZE)
126
+ if rem_in_block < HEADER_SIZE:
127
+ out.extend(b"\x00" * rem_in_block)
128
+ rem_in_block = BLOCK_SIZE
129
+
130
+ offset = 0
131
+ first_piece = True
132
+ total = len(payload)
133
+
134
+ if total == 0:
135
+ return bytes(out) + encode_leveldb_log_record(REC_FULL, b"")
136
+
137
+ while offset < total:
138
+ if rem_in_block < HEADER_SIZE:
139
+ out.extend(b"\x00" * rem_in_block)
140
+ rem_in_block = BLOCK_SIZE
141
+
142
+ avail_data = rem_in_block - HEADER_SIZE
143
+ take = min(avail_data, total - offset)
144
+ chunk = payload[offset : offset + take]
145
+ is_last = (offset + take) == total
146
+
147
+ if first_piece and is_last:
148
+ rtype = REC_FULL
149
+ elif first_piece:
150
+ rtype = REC_FIRST
151
+ elif is_last:
152
+ rtype = REC_LAST
153
+ else:
154
+ rtype = REC_MIDDLE
155
+
156
+ frame = encode_leveldb_log_record(rtype, chunk)
157
+ out.extend(frame)
158
+ offset += take
159
+ rem_in_block -= len(frame)
160
+ first_piece = False
161
+
162
+ return bytes(out)
163
+
164
+
165
+ def encode_varint(val: int) -> bytes:
166
+ if val < 0:
167
+ raise ValueError("Varint must be non-negative")
168
+ out = bytearray()
169
+ while val >= 0x80:
170
+ out.append((val & 0x7F) | 0x80)
171
+ val >>= 7
172
+ out.append(val)
173
+ return bytes(out)
174
+
175
+
176
+ def decode_varint(buf: bytes, pos: int) -> Tuple[int, int]:
177
+ shift = 0
178
+ result = 0
179
+ while pos < len(buf):
180
+ b = buf[pos]
181
+ pos += 1
182
+ result |= (b & 0x7F) << shift
183
+ if not (b & 0x80):
184
+ return result, pos
185
+ shift += 7
186
+ if shift > 63:
187
+ raise ValueError("Varint overflow")
188
+ raise ValueError("Truncated varint")
189
+
190
+
191
+ def build_writebatch(sequence: int, operations: List[Tuple[int, bytes, bytes]]) -> bytes:
192
+ """Build a spec-compliant LevelDB WriteBatch payload for fixtures & tests."""
193
+ body = bytearray(struct.pack("<QI", sequence, len(operations)))
194
+ for op_type, key, val in operations:
195
+ body.append(op_type)
196
+ body.extend(encode_varint(len(key)))
197
+ body.extend(key)
198
+ if op_type == 1:
199
+ body.extend(encode_varint(len(val)))
200
+ body.extend(val)
201
+ elif op_type != 0:
202
+ raise ValueError(f"Unsupported op_type {op_type}")
203
+ return bytes(body)
204
+
205
+
206
+ def _is_valid_chatsvc_message_obj(obj: Any) -> bool:
207
+ """Ensure carved JSON object is a top-level chatsvc message record, not a non-chatsvc messenger or inner sub-dict."""
208
+ if not isinstance(obj, dict):
209
+ return False
210
+ msg_id = obj.get("id")
211
+ if msg_id is None or str(msg_id).strip() == "":
212
+ return False
213
+ if any(k in obj for k in ("content", "composetime", "messagetype", "conversationid")):
214
+ return True
215
+ if "from" in obj and not any(k in obj for k in ("body", "text", "t", "ts")):
216
+ return True
217
+ return False
218
+
219
+
220
+ def classify_carved_json_object(obj: Any, schema_mode: str = "auto") -> str | None:
221
+ """
222
+ Classify a top-level JSON dictionary under the requested `schema_mode`
223
+ (`auto`, `chatsvc`, or `generic-json`). Returns schema tag or None if filtered out.
224
+ """
225
+ if schema_mode not in VALID_SCHEMA_MODES:
226
+ raise ValueError(f"Unsupported schema_mode={schema_mode!r}; expected one of {VALID_SCHEMA_MODES}")
227
+ if not isinstance(obj, dict) or not obj:
228
+ return None
229
+
230
+ if _is_valid_chatsvc_message_obj(obj):
231
+ return "chatsvc"
232
+
233
+ if schema_mode == "chatsvc":
234
+ return None
235
+
236
+ lower_keys = {str(k).lower() for k in obj.keys()}
237
+
238
+ # Slack Web / Electron IndexedDB & Redux cache shape
239
+ if ("ts" in lower_keys or "client_msg_id" in lower_keys) and (
240
+ lower_keys & {"text", "user", "blocks", "subtype", "deleted_ts", "channel"}
241
+ ):
242
+ return "slack_web"
243
+
244
+ # WhatsApp Web / Signal / Generic Web Messenger shape
245
+ if (lower_keys & {"id", "msg_id", "t", "timestamp"}) and (
246
+ lower_keys & {"body", "text", "message", "author", "from", "sender", "ack"}
247
+ ):
248
+ return "web_messenger"
249
+
250
+ # B2B supplier, ERP, or banking invoice/payment portal shape
251
+ if (lower_keys & set(_PORTAL_ID_KEYS)) and (lower_keys & set(_PORTAL_CONTENT_KEYS)):
252
+ return "b2b_portal_json"
253
+
254
+ if schema_mode == "generic-json":
255
+ return "generic_json"
256
+
257
+ return None
258
+
259
+
260
+ def _collect_string_scalars(node: Any, cap: int = 64) -> List[str]:
261
+ """Deterministically collect bounded string/number leaves for generic-json IOC & content normalization."""
262
+ acc: List[str] = []
263
+
264
+ def _walk(cur: Any) -> None:
265
+ if len(acc) >= cap or cur is None or isinstance(cur, bool):
266
+ return
267
+ if isinstance(cur, str):
268
+ s = cur.strip()
269
+ if s:
270
+ acc.append(s)
271
+ elif isinstance(cur, (int, float)):
272
+ acc.append(str(cur))
273
+ elif isinstance(cur, dict):
274
+ for k in sorted(cur.keys(), key=str):
275
+ kl = str(k).lower()
276
+ if kl in ("blob_pad", "avatar", "thumbnail", "raw_html"):
277
+ continue
278
+ v = cur[k]
279
+ if isinstance(v, (str, int, float)) and not isinstance(v, bool):
280
+ vs = str(v).strip()
281
+ if vs:
282
+ acc.append(f"{k}: {vs}")
283
+ if len(acc) >= cap:
284
+ return
285
+ else:
286
+ _walk(v)
287
+ elif isinstance(cur, list):
288
+ for item in cur:
289
+ _walk(item)
290
+
291
+ _walk(node)
292
+ return acc
293
+
294
+
295
+ def normalize_carved_record_fields(obj: Dict[str, Any], schema_tag: str) -> Dict[str, Any]:
296
+ """Map multi-platform web cache JSON fields onto StarShield's canonical record layout."""
297
+ props = obj.get("properties") if isinstance(obj.get("properties"), dict) else {}
298
+ edited_sub = obj.get("edited") if isinstance(obj.get("edited"), dict) else {}
299
+
300
+ msg_id = ""
301
+ for k in _PORTAL_ID_KEYS + ("ts", "t"):
302
+ v = obj.get(k)
303
+ if v is not None and str(v).strip() != "":
304
+ msg_id = str(v).strip()
305
+ break
306
+
307
+ sender = ""
308
+ for k in ("from", "user", "sender", "author", "actor", "vendor", "supplier", "beneficiary"):
309
+ v = obj.get(k)
310
+ if v is not None and str(v).strip() != "":
311
+ sender = str(v).strip()
312
+ break
313
+
314
+ compose_ts = None
315
+ for k in ("composetime", "ts", "t", "timestamp", "created_at", "issued_at", "date"):
316
+ if obj.get(k) not in (None, ""):
317
+ compose_ts = obj.get(k)
318
+ break
319
+
320
+ del_ts = (
321
+ props.get("deletetime")
322
+ or obj.get("deletetime")
323
+ or obj.get("deleted_ts")
324
+ or obj.get("deleted_at")
325
+ or obj.get("retracted_at")
326
+ or ("true" if obj.get("subtype") == "message_deleted" or obj.get("is_deleted") is True else None)
327
+ )
328
+ edit_ts = (
329
+ props.get("edittime")
330
+ or obj.get("edittime")
331
+ or edited_sub.get("ts")
332
+ or obj.get("edited_at")
333
+ or obj.get("updated_at")
334
+ )
335
+
336
+ if schema_tag == "chatsvc":
337
+ content_str = str(obj.get("content", ""))
338
+ else:
339
+ primary_parts: List[str] = []
340
+ for k in ("content", "text", "body", "message", "payment_instructions", "memo", "note", "description"):
341
+ v = obj.get(k)
342
+ if isinstance(v, str) and v.strip():
343
+ primary_parts.append(v.strip())
344
+ # Append structured banking/routing fields if present outside free-text body
345
+ for k in ("beneficiary", "payee", "iban", "beneficiary_iban", "sort_code", "account_number", "swift", "bic", "amount"):
346
+ v = obj.get(k)
347
+ if v is not None and str(v).strip():
348
+ primary_parts.append(f"{k}={str(v).strip()}")
349
+ if primary_parts:
350
+ content_str = " | ".join(primary_parts)
351
+ else:
352
+ content_str = " | ".join(_collect_string_scalars(obj))
353
+
354
+ return {
355
+ "message_id": msg_id,
356
+ "sender_mri": sender,
357
+ "composetime": compose_ts,
358
+ "deletetime": del_ts,
359
+ "edittime": edit_ts,
360
+ "cached_content": content_str,
361
+ }
362
+
363
+
364
+ def _extract_json_objects_from_bytes(
365
+ raw_val: bytes,
366
+ schema_mode: str = "auto",
367
+ ) -> List[Tuple[Dict[str, Any], str]]:
368
+ """
369
+ Scan UTF-8 (and UTF-16LE if NUL bytes exist) using `json.JSONDecoder.raw_decode`.
370
+ Advances past the full span of each decoded top-level JSON object so nested child
371
+ objects (e.g., `properties: {deletetime: ...}`) are NEVER mis-carved as standalone records.
372
+ """
373
+ candidates: List[str] = [raw_val.decode("utf-8", errors="ignore")]
374
+ if b"\x00" in raw_val:
375
+ candidates.append(raw_val.decode("utf-16-le", errors="ignore"))
376
+
377
+ decoder = json.JSONDecoder()
378
+ found: List[Tuple[Dict[str, Any], str]] = []
379
+ seen_hashes = set()
380
+
381
+ for text in candidates:
382
+ pos = 0
383
+ n = len(text)
384
+ while pos < n:
385
+ brace_idx = text.find("{", pos)
386
+ if brace_idx == -1:
387
+ break
388
+ try:
389
+ obj, end_idx = decoder.raw_decode(text, brace_idx)
390
+ except json.JSONDecodeError:
391
+ pos = brace_idx + 1
392
+ continue
393
+
394
+ # Advance past the entire top-level JSON object so nested dicts are skipped
395
+ pos = max(end_idx, brace_idx + 1)
396
+ schema_tag = classify_carved_json_object(obj, schema_mode=schema_mode)
397
+ if schema_tag is not None:
398
+ canonical = json.dumps(obj, sort_keys=True, ensure_ascii=False)
399
+ h = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
400
+ if h not in seen_hashes:
401
+ seen_hashes.add(h)
402
+ found.append((obj, schema_tag))
403
+
404
+ return found
405
+
406
+
407
+ def parse_leveldb_wal(raw_bytes: bytes, schema_mode: str = "auto") -> Dict[str, Any]:
408
+ """Parse a Chromium LevelDB `.log` file, verifying CRC32C, WriteBatch bounds, and EOF completeness."""
409
+ if schema_mode not in VALID_SCHEMA_MODES:
410
+ raise ValueError(f"Unsupported schema_mode={schema_mode!r}; expected one of {VALID_SCHEMA_MODES}")
411
+
412
+ total_len = len(raw_bytes)
413
+ physical_records_ok = 0
414
+ corrupted_records: List[Dict[str, Any]] = []
415
+ recovered_entries: List[Dict[str, Any]] = []
416
+ stats: Dict[str, int] = {"value_ops_total": 0, "blink_v8_binary_only_values": 0}
417
+
418
+ frag_buf = bytearray()
419
+ frag_start_offset: int | None = None
420
+
421
+ block_idx = 0
422
+ while block_idx * BLOCK_SIZE < total_len:
423
+ block_start = block_idx * BLOCK_SIZE
424
+ block_end = min(block_start + BLOCK_SIZE, total_len)
425
+ pos = block_start
426
+
427
+ while pos + HEADER_SIZE <= block_end:
428
+ stored_crc, length, rec_type = struct.unpack_from("<IHB", raw_bytes, pos)
429
+ if stored_crc == 0 and length == 0 and rec_type == REC_ZERO:
430
+ rem = raw_bytes[pos:block_end]
431
+ if any(b != 0 for b in rem):
432
+ corrupted_records.append(
433
+ {
434
+ "byte_offset": pos,
435
+ "reason": "Non-zero bytes following zero-header preallocation region in 32KiB block",
436
+ }
437
+ )
438
+ pos = block_end
439
+ break
440
+
441
+ rec_offset = pos
442
+ data_start = pos + HEADER_SIZE
443
+ data_end = data_start + length
444
+ if data_end > block_end:
445
+ corrupted_records.append(
446
+ {
447
+ "byte_offset": rec_offset,
448
+ "reason": "Record length exceeds 32KiB LevelDB block boundary or truncated EOF",
449
+ }
450
+ )
451
+ frag_buf.clear()
452
+ frag_start_offset = None
453
+ pos = block_end
454
+ break
455
+
456
+ chunk = raw_bytes[data_start:data_end]
457
+ pos = data_end
458
+
459
+ expected_masked = mask_crc32c(crc32c(bytes([rec_type]) + chunk))
460
+ if stored_crc != expected_masked:
461
+ corrupted_records.append(
462
+ {
463
+ "byte_offset": rec_offset,
464
+ "record_type": rec_type,
465
+ "reason": f"CRC32C mismatch (stored=0x{stored_crc:08x}, computed=0x{expected_masked:08x})",
466
+ }
467
+ )
468
+ frag_buf.clear()
469
+ frag_start_offset = None
470
+ continue
471
+
472
+ physical_records_ok += 1
473
+
474
+ if rec_type == REC_FULL:
475
+ if frag_start_offset is not None:
476
+ corrupted_records.append(
477
+ {
478
+ "byte_offset": frag_start_offset,
479
+ "reason": "Unclosed FIRST/MIDDLE fragmented record interrupted by FULL record",
480
+ }
481
+ )
482
+ frag_buf.clear()
483
+ frag_start_offset = None
484
+ logical_bytes = chunk
485
+ logical_offset = rec_offset
486
+ elif rec_type == REC_FIRST:
487
+ if frag_start_offset is not None:
488
+ corrupted_records.append(
489
+ {
490
+ "byte_offset": frag_start_offset,
491
+ "reason": "Unclosed FIRST/MIDDLE fragmented record interrupted by new FIRST record",
492
+ }
493
+ )
494
+ frag_buf = bytearray(chunk)
495
+ frag_start_offset = rec_offset
496
+ continue
497
+ elif rec_type == REC_MIDDLE:
498
+ if frag_start_offset is None:
499
+ corrupted_records.append(
500
+ {"byte_offset": rec_offset, "reason": "Orphan MIDDLE fragment without FIRST"}
501
+ )
502
+ continue
503
+ if len(frag_buf) + len(chunk) > MAX_LOGICAL_RECORD_BYTES:
504
+ corrupted_records.append(
505
+ {"byte_offset": frag_start_offset, "reason": "Fragmented record exceeds 32MiB safety limit"}
506
+ )
507
+ frag_buf.clear()
508
+ frag_start_offset = None
509
+ continue
510
+ frag_buf.extend(chunk)
511
+ continue
512
+ elif rec_type == REC_LAST:
513
+ if frag_start_offset is None:
514
+ corrupted_records.append(
515
+ {"byte_offset": rec_offset, "reason": "Orphan LAST fragment without FIRST"}
516
+ )
517
+ continue
518
+ if len(frag_buf) + len(chunk) > MAX_LOGICAL_RECORD_BYTES:
519
+ corrupted_records.append(
520
+ {"byte_offset": frag_start_offset, "reason": "Fragmented record exceeds 32MiB safety limit"}
521
+ )
522
+ frag_buf.clear()
523
+ frag_start_offset = None
524
+ continue
525
+ frag_buf.extend(chunk)
526
+ logical_bytes = bytes(frag_buf)
527
+ logical_offset = frag_start_offset
528
+ frag_buf.clear()
529
+ frag_start_offset = None
530
+ else:
531
+ corrupted_records.append(
532
+ {"byte_offset": rec_offset, "reason": f"Unknown LevelDB record type {rec_type}"}
533
+ )
534
+ frag_buf.clear()
535
+ frag_start_offset = None
536
+ continue
537
+
538
+ _decode_writebatch_into(
539
+ logical_bytes,
540
+ logical_offset,
541
+ recovered_entries,
542
+ corrupted_records,
543
+ schema_mode=schema_mode,
544
+ stats=stats,
545
+ )
546
+
547
+ if pos < block_end:
548
+ trailer = raw_bytes[pos:block_end]
549
+ if any(b != 0 for b in trailer):
550
+ corrupted_records.append(
551
+ {
552
+ "byte_offset": pos,
553
+ "reason": f"Non-zero truncated header/trailer ({len(trailer)} bytes < 7-byte header)",
554
+ }
555
+ )
556
+
557
+ block_idx += 1
558
+
559
+ if frag_start_offset is not None:
560
+ corrupted_records.append(
561
+ {
562
+ "byte_offset": frag_start_offset,
563
+ "reason": "Truncated logical record: unclosed FIRST/MIDDLE fragment at EOF",
564
+ }
565
+ )
566
+
567
+ chatsvc_count = sum(1 for r in recovered_entries if r.get("schema_detected") == "chatsvc")
568
+ return {
569
+ "parser": "leveldb_wal_32k_crc32c_v0.3.6",
570
+ "schema_mode": schema_mode,
571
+ "file_size_bytes": total_len,
572
+ "valid_physical_frames": physical_records_ok,
573
+ "writebatch_value_ops_seen": stats["value_ops_total"],
574
+ "blink_v8_non_json_values_skipped": stats["blink_v8_binary_only_values"],
575
+ "corrupted_frames_count": len(corrupted_records),
576
+ "corrupted_frames": corrupted_records,
577
+ "recovered_records_count": len(recovered_entries),
578
+ "recovered_chatsvc_records_count": chatsvc_count if schema_mode == "chatsvc" else len(recovered_entries),
579
+ "recovered_records": recovered_entries,
580
+ }
581
+
582
+
583
+ def _decode_writebatch_into(
584
+ batch: bytes,
585
+ logical_offset: int,
586
+ out_entries: List[Dict[str, Any]],
587
+ corrupted: List[Dict[str, Any]],
588
+ schema_mode: str = "auto",
589
+ stats: Dict[str, int] | None = None,
590
+ ) -> None:
591
+ if len(batch) < 12:
592
+ corrupted.append({"byte_offset": logical_offset, "reason": "WriteBatch shorter than 12-byte header"})
593
+ return
594
+
595
+ seq_num, declared_count = struct.unpack_from("<QI", batch, 0)
596
+ idx = 12
597
+ parsed_ops = 0
598
+ staged_entries: List[Dict[str, Any]] = []
599
+ local_value_ops = 0
600
+ local_v8_binary_only = 0
601
+
602
+ try:
603
+ while idx < len(batch) and parsed_ops < declared_count:
604
+ op_type = batch[idx]
605
+ idx += 1
606
+ key_len, idx = decode_varint(batch, idx)
607
+ if idx + key_len > len(batch):
608
+ raise ValueError(f"Truncated key bytes (declared={key_len}, remaining={len(batch) - idx})")
609
+ key_bytes = batch[idx : idx + key_len]
610
+ idx += key_len
611
+
612
+ if op_type == 1:
613
+ val_len, idx = decode_varint(batch, idx)
614
+ if idx + val_len > len(batch):
615
+ raise ValueError(f"Truncated value bytes (declared={val_len}, remaining={len(batch) - idx})")
616
+ val_bytes = batch[idx : idx + val_len]
617
+ idx += val_len
618
+ local_value_ops += 1
619
+
620
+ carved_objs = _extract_json_objects_from_bytes(val_bytes, schema_mode=schema_mode)
621
+ if not carved_objs and b"\xff" in val_bytes[:8]:
622
+ local_v8_binary_only += 1
623
+
624
+ for obj, schema_tag in carved_objs:
625
+ norm = normalize_carved_record_fields(obj, schema_tag)
626
+ del_ts = norm["deletetime"]
627
+ edit_ts = norm["edittime"]
628
+ staged_entries.append(
629
+ {
630
+ "epistemic_tier": "OBSERVED",
631
+ "schema_detected": schema_tag,
632
+ "leveldb_batch_byte_offset": logical_offset,
633
+ "leveldb_sequence_number": seq_num,
634
+ "key_sha256": hashlib.sha256(key_bytes).hexdigest(),
635
+ "value_sha256": hashlib.sha256(val_bytes).hexdigest(),
636
+ "message_id": norm["message_id"],
637
+ "sender_mri": norm["sender_mri"],
638
+ "composetime": norm["composetime"],
639
+ "deletetime": del_ts,
640
+ "edittime": edit_ts,
641
+ "is_soft_deleted_in_ui": bool(del_ts),
642
+ "note_on_edittime": (
643
+ "Present value reflects post-edit state unless an earlier sequence_number "
644
+ "record with the same message_id exists in uncompacted log frames."
645
+ if edit_ts
646
+ else None
647
+ ),
648
+ "cached_content": norm["cached_content"],
649
+ }
650
+ )
651
+ elif op_type == 0:
652
+ pass
653
+ else:
654
+ raise ValueError(f"Unsupported WriteBatch op_type=0x{op_type:02x}")
655
+ parsed_ops += 1
656
+
657
+ if parsed_ops != declared_count:
658
+ raise ValueError(f"WriteBatch op count mismatch (declared={declared_count}, parsed={parsed_ops})")
659
+ if idx != len(batch):
660
+ raise ValueError(f"Trailing unparsed bytes in WriteBatch ({len(batch) - idx} bytes remaining)")
661
+
662
+ except Exception as exc:
663
+ corrupted.append({"byte_offset": logical_offset, "reason": f"Malformed WriteBatch entry: {exc}"})
664
+ return
665
+
666
+ if stats is not None:
667
+ stats["value_ops_total"] = stats.get("value_ops_total", 0) + local_value_ops
668
+ stats["blink_v8_binary_only_values"] = stats.get("blink_v8_binary_only_values", 0) + local_v8_binary_only
669
+ out_entries.extend(staged_entries)
670
+
671
+
672
+ def _has_leveldb_sstable_magic(path: Path) -> bool:
673
+ """Verify the 8-byte LevelDB SSTable footer magic (`0xdb4775248b80fb57`) at `EOF - 8`."""
674
+ try:
675
+ size = path.stat().st_size
676
+ if size < 48:
677
+ return False
678
+ with path.open("rb") as fh:
679
+ fh.seek(size - 8)
680
+ return fh.read(8) == LEVELDB_SSTABLE_MAGIC_LE
681
+ except OSError:
682
+ return False
683
+
684
+
685
+ def inspect_chromium_storage_path(
686
+ target: Path,
687
+ max_mb: int = 64,
688
+ schema_mode: str = "auto",
689
+ redact: bool = True,
690
+ ) -> Dict[str, Any]:
691
+ """
692
+ Zero-PII local precheck gate (`GO` / `PARTIAL` / `NO_GO_COMPACTED_OR_UNSUPPORTED`)
693
+ for law-firm IT or forensic intake teams before instructing a paid triage pilot.
694
+ Never emits plaintext message content, filenames, or raw hashes in default masked mode.
695
+ """
696
+ if max_mb < 1:
697
+ raise ValueError("--max-mb safety limit must be >= 1 MiB")
698
+ if not target.exists():
699
+ if redact:
700
+ raise OSError(
701
+ "Unable to access target path (OS path suppressed in masked mode; "
702
+ "pass --unredacted in an offline lab for path diagnostics)"
703
+ )
704
+ raise FileNotFoundError(f"Target path does not exist: {target}")
705
+
706
+ limit_bytes = max_mb * 1024 * 1024
707
+ log_files: List[Path] = []
708
+ ldb_files: List[Path] = []
709
+ blob_files_count = 0
710
+
711
+ if target.is_file():
712
+ suf = target.suffix.lower()
713
+ if suf == ".log":
714
+ log_files.append(target)
715
+ elif suf in (".ldb", ".sst"):
716
+ ldb_files.append(target)
717
+ elif target.is_dir():
718
+ for p in sorted(target.rglob("*")):
719
+ if not p.is_file():
720
+ continue
721
+ suf = p.suffix.lower()
722
+ if suf == ".log":
723
+ log_files.append(p)
724
+ elif suf in (".ldb", ".sst"):
725
+ ldb_files.append(p)
726
+ elif ".blob" in str(p.parent).lower() or suf == ".blob":
727
+ blob_files_count += 1
728
+
729
+ valid_wal_frames = 0
730
+ carvable_records = 0
731
+ soft_deleted_records = 0
732
+ v8_binary_only_values = 0
733
+ oversize_logs_skipped = 0
734
+ schemas_seen: Dict[str, int] = {}
735
+
736
+ for lf in log_files:
737
+ try:
738
+ sz = lf.stat().st_size
739
+ except OSError:
740
+ continue
741
+ if sz > limit_bytes:
742
+ oversize_logs_skipped += 1
743
+ continue
744
+ if sz == 0:
745
+ continue
746
+ parsed = parse_leveldb_wal(lf.read_bytes(), schema_mode=schema_mode)
747
+ valid_wal_frames += parsed["valid_physical_frames"]
748
+ carvable_records += parsed["recovered_records_count"]
749
+ v8_binary_only_values += parsed["blink_v8_non_json_values_skipped"]
750
+ for rec in parsed["recovered_records"]:
751
+ st = str(rec.get("schema_detected", "unknown"))
752
+ schemas_seen[st] = schemas_seen.get(st, 0) + 1
753
+ if rec.get("is_soft_deleted_in_ui"):
754
+ soft_deleted_records += 1
755
+
756
+ verified_sstables = sum(1 for f in ldb_files if _has_leveldb_sstable_magic(f))
757
+
758
+ if carvable_records > 0 and verified_sstables == 0 and v8_binary_only_values == 0 and blob_files_count == 0:
759
+ verdict = "GO"
760
+ guidance = (
761
+ "Uncompacted LevelDB .log frames contain carvable JSON records. "
762
+ "Eligible for immediate byte-offset-cited extraction under v0.3.6."
763
+ )
764
+ elif carvable_records > 0:
765
+ verdict = "PARTIAL"
766
+ guidance = (
767
+ "Carvable JSON records are recoverable from uncompacted .log frames, alongside compacted "
768
+ ".ldb SSTables, external .blob files, or binary-only Blink V8 values that fall outside v0.3.6 scope."
769
+ )
770
+ else:
771
+ verdict = "NO_GO_COMPACTED_OR_UNSUPPORTED"
772
+ guidance = (
773
+ "Zero carvable JSON records found in uncompacted .log files (storage may already be compacted "
774
+ "into Snappy .ldb SSTables, encoded solely in binary Blink V8 StructuredClone, or empty). "
775
+ "Under StarShield SOW terms, £0 pilot fee applies."
776
+ )
777
+
778
+ return {
779
+ "epistemic_tier": "OBSERVED",
780
+ "tool_version": "starshield-forensics/0.3.6",
781
+ "target_label": "[REDACTED_TARGET_PATH]" if redact else str(target),
782
+ "schema_mode": schema_mode,
783
+ "feasibility_verdict": verdict,
784
+ "wal_log_files_detected": len(log_files),
785
+ "wal_oversize_files_skipped": oversize_logs_skipped,
786
+ "valid_wal_physical_frames": valid_wal_frames,
787
+ "carvable_json_records_count": carvable_records,
788
+ "soft_deleted_json_records_count": soft_deleted_records,
789
+ "schemas_detected_counts": schemas_seen,
790
+ "compacted_sstable_files_detected": len(ldb_files),
791
+ "verified_leveldb_sstables_count": verified_sstables,
792
+ "indexeddb_blob_files_detected": blob_files_count,
793
+ "blink_v8_non_json_values_detected": v8_binary_only_values,
794
+ "intake_guidance": guidance,
795
+ }