starshield-forensics 0.3.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- starshield/__init__.py +3 -0
- starshield/cli.py +743 -0
- starshield/data/__init__.py +1 -0
- starshield/data/synthetic_case_benchmark.json +73 -0
- starshield/image_containers.py +436 -0
- starshield/ioc_validators.py +229 -0
- starshield/leveldb_log.py +795 -0
- starshield_forensics-0.3.7.dist-info/METADATA +119 -0
- starshield_forensics-0.3.7.dist-info/RECORD +13 -0
- starshield_forensics-0.3.7.dist-info/WHEEL +5 -0
- starshield_forensics-0.3.7.dist-info/entry_points.txt +2 -0
- starshield_forensics-0.3.7.dist-info/licenses/LICENSE +21 -0
- starshield_forensics-0.3.7.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,795 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Google LevelDB Write-Ahead Log (`.log`) Binary Frame, Multi-Schema JSON Carver
|
|
5
|
+
& Local Storage Precheck Gate for Chromium IndexedDB / Web Application Caches.
|
|
6
|
+
|
|
7
|
+
Implements the official LevelDB log format specification:
|
|
8
|
+
- 32,768-byte (32 KiB) physical block boundaries with strict zero-padding trailer checks
|
|
9
|
+
- 7-byte record header: [masked_crc32c: uint32_le][length: uint16_le][type: uint8]
|
|
10
|
+
where type in {1: FULL, 2: FIRST, 3: MIDDLE, 4: LAST}
|
|
11
|
+
- Castagnoli CRC32C (polynomial 0x82F63B78) + LevelDB mask delta 0xA282EAD8
|
|
12
|
+
- Strict LevelDB WriteBatch validation: [sequence: uint64_le][count: uint32_le] followed
|
|
13
|
+
by varint-prefixed kTypeValue (0x01) and kTypeDeletion (0x00) entries, rejecting
|
|
14
|
+
truncated payloads, op-count mismatches, trailing garbage bytes, and unclosed EOF fragments.
|
|
15
|
+
- Top-level `json.JSONDecoder.raw_decode` scanning across `chatsvc` (Teams/Skype), Slack Web,
|
|
16
|
+
WhatsApp/Web Messenger, and B2B supplier/invoice portal JSON schemas (`--schema auto`).
|
|
17
|
+
- Offline `inspect_chromium_storage_path` feasibility precheck (`GO` / `PARTIAL` /
|
|
18
|
+
`NO_GO_COMPACTED_OR_UNSUPPORTED`) detecting `.ldb` SSTable footers (`0xdb4775248b80fb57`),
|
|
19
|
+
IndexedDB `.blob` directories, and Blink V8 `0xFF` headers with zero PII leakage.
|
|
20
|
+
|
|
21
|
+
Scope & Honest Limitations:
|
|
22
|
+
- Targets uncompressed LevelDB `.log` write-ahead logs and embedded UTF-8/UTF-16LE JSON
|
|
23
|
+
slices inside Chromium IndexedDB / Local Storage WriteBatch values.
|
|
24
|
+
- Does NOT decompress Snappy-compressed `.ldb` SSTable data blocks or decode binary-only
|
|
25
|
+
Blink V8 StructuredClone primitives (scheduled for v0.4).
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
from __future__ import annotations
|
|
29
|
+
|
|
30
|
+
import hashlib
|
|
31
|
+
import json
|
|
32
|
+
import struct
|
|
33
|
+
from pathlib import Path
|
|
34
|
+
from typing import Any, Dict, List, Tuple
|
|
35
|
+
|
|
36
|
+
BLOCK_SIZE = 32768
|
|
37
|
+
HEADER_SIZE = 7
|
|
38
|
+
MAX_LOGICAL_RECORD_BYTES = 32 * 1024 * 1024 # 32 MiB anti-DoS cap per reassembled batch
|
|
39
|
+
LEVELDB_SSTABLE_MAGIC_LE = struct.pack("<Q", 0xDB4775248B80FB57)
|
|
40
|
+
|
|
41
|
+
REC_ZERO = 0
|
|
42
|
+
REC_FULL = 1
|
|
43
|
+
REC_FIRST = 2
|
|
44
|
+
REC_MIDDLE = 3
|
|
45
|
+
REC_LAST = 4
|
|
46
|
+
|
|
47
|
+
MASK_DELTA = 0xA282EAD8
|
|
48
|
+
|
|
49
|
+
VALID_SCHEMA_MODES = ("auto", "chatsvc", "generic-json")
|
|
50
|
+
|
|
51
|
+
_PORTAL_ID_KEYS = (
|
|
52
|
+
"id",
|
|
53
|
+
"message_id",
|
|
54
|
+
"msg_id",
|
|
55
|
+
"client_msg_id",
|
|
56
|
+
"invoice_id",
|
|
57
|
+
"payment_id",
|
|
58
|
+
"tx_id",
|
|
59
|
+
"transaction_id",
|
|
60
|
+
"reference",
|
|
61
|
+
"uuid",
|
|
62
|
+
"record_id",
|
|
63
|
+
"event_id",
|
|
64
|
+
"order_id",
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
_PORTAL_CONTENT_KEYS = (
|
|
68
|
+
"content",
|
|
69
|
+
"text",
|
|
70
|
+
"body",
|
|
71
|
+
"message",
|
|
72
|
+
"memo",
|
|
73
|
+
"note",
|
|
74
|
+
"description",
|
|
75
|
+
"payment_instructions",
|
|
76
|
+
"iban",
|
|
77
|
+
"beneficiary_iban",
|
|
78
|
+
"account_number",
|
|
79
|
+
"sort_code",
|
|
80
|
+
"beneficiary",
|
|
81
|
+
"payee",
|
|
82
|
+
"amount",
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _build_crc32c_table() -> List[int]:
|
|
87
|
+
poly = 0x82F63B78
|
|
88
|
+
table: List[int] = []
|
|
89
|
+
for i in range(256):
|
|
90
|
+
crc = i
|
|
91
|
+
for _ in range(8):
|
|
92
|
+
if crc & 1:
|
|
93
|
+
crc = (crc >> 1) ^ poly
|
|
94
|
+
else:
|
|
95
|
+
crc >>= 1
|
|
96
|
+
table.append(crc)
|
|
97
|
+
return table
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
_CRC32C_TABLE = _build_crc32c_table()
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def crc32c(data: bytes) -> int:
|
|
104
|
+
crc = 0xFFFFFFFF
|
|
105
|
+
for b in data:
|
|
106
|
+
crc = _CRC32C_TABLE[(crc ^ b) & 0xFF] ^ (crc >> 8)
|
|
107
|
+
return crc ^ 0xFFFFFFFF
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def mask_crc32c(crc: int) -> int:
|
|
111
|
+
return (((crc >> 15) | ((crc << 17) & 0xFFFFFFFF)) + MASK_DELTA) & 0xFFFFFFFF
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def encode_leveldb_log_record(rec_type: int, payload: bytes) -> bytes:
|
|
115
|
+
if len(payload) > 0xFFFF:
|
|
116
|
+
raise ValueError("Single physical LevelDB log record payload cannot exceed 65535 bytes")
|
|
117
|
+
raw_crc = crc32c(bytes([rec_type]) + payload)
|
|
118
|
+
masked = mask_crc32c(raw_crc)
|
|
119
|
+
return struct.pack("<IHB", masked, len(payload), rec_type) + payload
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def encode_logical_record_across_blocks(payload: bytes, initial_block_used: int = 0) -> bytes:
|
|
123
|
+
"""Split a logical WriteBatch across 32KiB physical LevelDB blocks using FULL/FIRST/MIDDLE/LAST."""
|
|
124
|
+
out = bytearray()
|
|
125
|
+
rem_in_block = BLOCK_SIZE - (initial_block_used % BLOCK_SIZE)
|
|
126
|
+
if rem_in_block < HEADER_SIZE:
|
|
127
|
+
out.extend(b"\x00" * rem_in_block)
|
|
128
|
+
rem_in_block = BLOCK_SIZE
|
|
129
|
+
|
|
130
|
+
offset = 0
|
|
131
|
+
first_piece = True
|
|
132
|
+
total = len(payload)
|
|
133
|
+
|
|
134
|
+
if total == 0:
|
|
135
|
+
return bytes(out) + encode_leveldb_log_record(REC_FULL, b"")
|
|
136
|
+
|
|
137
|
+
while offset < total:
|
|
138
|
+
if rem_in_block < HEADER_SIZE:
|
|
139
|
+
out.extend(b"\x00" * rem_in_block)
|
|
140
|
+
rem_in_block = BLOCK_SIZE
|
|
141
|
+
|
|
142
|
+
avail_data = rem_in_block - HEADER_SIZE
|
|
143
|
+
take = min(avail_data, total - offset)
|
|
144
|
+
chunk = payload[offset : offset + take]
|
|
145
|
+
is_last = (offset + take) == total
|
|
146
|
+
|
|
147
|
+
if first_piece and is_last:
|
|
148
|
+
rtype = REC_FULL
|
|
149
|
+
elif first_piece:
|
|
150
|
+
rtype = REC_FIRST
|
|
151
|
+
elif is_last:
|
|
152
|
+
rtype = REC_LAST
|
|
153
|
+
else:
|
|
154
|
+
rtype = REC_MIDDLE
|
|
155
|
+
|
|
156
|
+
frame = encode_leveldb_log_record(rtype, chunk)
|
|
157
|
+
out.extend(frame)
|
|
158
|
+
offset += take
|
|
159
|
+
rem_in_block -= len(frame)
|
|
160
|
+
first_piece = False
|
|
161
|
+
|
|
162
|
+
return bytes(out)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def encode_varint(val: int) -> bytes:
|
|
166
|
+
if val < 0:
|
|
167
|
+
raise ValueError("Varint must be non-negative")
|
|
168
|
+
out = bytearray()
|
|
169
|
+
while val >= 0x80:
|
|
170
|
+
out.append((val & 0x7F) | 0x80)
|
|
171
|
+
val >>= 7
|
|
172
|
+
out.append(val)
|
|
173
|
+
return bytes(out)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def decode_varint(buf: bytes, pos: int) -> Tuple[int, int]:
|
|
177
|
+
shift = 0
|
|
178
|
+
result = 0
|
|
179
|
+
while pos < len(buf):
|
|
180
|
+
b = buf[pos]
|
|
181
|
+
pos += 1
|
|
182
|
+
result |= (b & 0x7F) << shift
|
|
183
|
+
if not (b & 0x80):
|
|
184
|
+
return result, pos
|
|
185
|
+
shift += 7
|
|
186
|
+
if shift > 63:
|
|
187
|
+
raise ValueError("Varint overflow")
|
|
188
|
+
raise ValueError("Truncated varint")
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def build_writebatch(sequence: int, operations: List[Tuple[int, bytes, bytes]]) -> bytes:
|
|
192
|
+
"""Build a spec-compliant LevelDB WriteBatch payload for fixtures & tests."""
|
|
193
|
+
body = bytearray(struct.pack("<QI", sequence, len(operations)))
|
|
194
|
+
for op_type, key, val in operations:
|
|
195
|
+
body.append(op_type)
|
|
196
|
+
body.extend(encode_varint(len(key)))
|
|
197
|
+
body.extend(key)
|
|
198
|
+
if op_type == 1:
|
|
199
|
+
body.extend(encode_varint(len(val)))
|
|
200
|
+
body.extend(val)
|
|
201
|
+
elif op_type != 0:
|
|
202
|
+
raise ValueError(f"Unsupported op_type {op_type}")
|
|
203
|
+
return bytes(body)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _is_valid_chatsvc_message_obj(obj: Any) -> bool:
|
|
207
|
+
"""Ensure carved JSON object is a top-level chatsvc message record, not a non-chatsvc messenger or inner sub-dict."""
|
|
208
|
+
if not isinstance(obj, dict):
|
|
209
|
+
return False
|
|
210
|
+
msg_id = obj.get("id")
|
|
211
|
+
if msg_id is None or str(msg_id).strip() == "":
|
|
212
|
+
return False
|
|
213
|
+
if any(k in obj for k in ("content", "composetime", "messagetype", "conversationid")):
|
|
214
|
+
return True
|
|
215
|
+
if "from" in obj and not any(k in obj for k in ("body", "text", "t", "ts")):
|
|
216
|
+
return True
|
|
217
|
+
return False
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def classify_carved_json_object(obj: Any, schema_mode: str = "auto") -> str | None:
|
|
221
|
+
"""
|
|
222
|
+
Classify a top-level JSON dictionary under the requested `schema_mode`
|
|
223
|
+
(`auto`, `chatsvc`, or `generic-json`). Returns schema tag or None if filtered out.
|
|
224
|
+
"""
|
|
225
|
+
if schema_mode not in VALID_SCHEMA_MODES:
|
|
226
|
+
raise ValueError(f"Unsupported schema_mode={schema_mode!r}; expected one of {VALID_SCHEMA_MODES}")
|
|
227
|
+
if not isinstance(obj, dict) or not obj:
|
|
228
|
+
return None
|
|
229
|
+
|
|
230
|
+
if _is_valid_chatsvc_message_obj(obj):
|
|
231
|
+
return "chatsvc"
|
|
232
|
+
|
|
233
|
+
if schema_mode == "chatsvc":
|
|
234
|
+
return None
|
|
235
|
+
|
|
236
|
+
lower_keys = {str(k).lower() for k in obj.keys()}
|
|
237
|
+
|
|
238
|
+
# Slack Web / Electron IndexedDB & Redux cache shape
|
|
239
|
+
if ("ts" in lower_keys or "client_msg_id" in lower_keys) and (
|
|
240
|
+
lower_keys & {"text", "user", "blocks", "subtype", "deleted_ts", "channel"}
|
|
241
|
+
):
|
|
242
|
+
return "slack_web"
|
|
243
|
+
|
|
244
|
+
# WhatsApp Web / Signal / Generic Web Messenger shape
|
|
245
|
+
if (lower_keys & {"id", "msg_id", "t", "timestamp"}) and (
|
|
246
|
+
lower_keys & {"body", "text", "message", "author", "from", "sender", "ack"}
|
|
247
|
+
):
|
|
248
|
+
return "web_messenger"
|
|
249
|
+
|
|
250
|
+
# B2B supplier, ERP, or banking invoice/payment portal shape
|
|
251
|
+
if (lower_keys & set(_PORTAL_ID_KEYS)) and (lower_keys & set(_PORTAL_CONTENT_KEYS)):
|
|
252
|
+
return "b2b_portal_json"
|
|
253
|
+
|
|
254
|
+
if schema_mode == "generic-json":
|
|
255
|
+
return "generic_json"
|
|
256
|
+
|
|
257
|
+
return None
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _collect_string_scalars(node: Any, cap: int = 64) -> List[str]:
|
|
261
|
+
"""Deterministically collect bounded string/number leaves for generic-json IOC & content normalization."""
|
|
262
|
+
acc: List[str] = []
|
|
263
|
+
|
|
264
|
+
def _walk(cur: Any) -> None:
|
|
265
|
+
if len(acc) >= cap or cur is None or isinstance(cur, bool):
|
|
266
|
+
return
|
|
267
|
+
if isinstance(cur, str):
|
|
268
|
+
s = cur.strip()
|
|
269
|
+
if s:
|
|
270
|
+
acc.append(s)
|
|
271
|
+
elif isinstance(cur, (int, float)):
|
|
272
|
+
acc.append(str(cur))
|
|
273
|
+
elif isinstance(cur, dict):
|
|
274
|
+
for k in sorted(cur.keys(), key=str):
|
|
275
|
+
kl = str(k).lower()
|
|
276
|
+
if kl in ("blob_pad", "avatar", "thumbnail", "raw_html"):
|
|
277
|
+
continue
|
|
278
|
+
v = cur[k]
|
|
279
|
+
if isinstance(v, (str, int, float)) and not isinstance(v, bool):
|
|
280
|
+
vs = str(v).strip()
|
|
281
|
+
if vs:
|
|
282
|
+
acc.append(f"{k}: {vs}")
|
|
283
|
+
if len(acc) >= cap:
|
|
284
|
+
return
|
|
285
|
+
else:
|
|
286
|
+
_walk(v)
|
|
287
|
+
elif isinstance(cur, list):
|
|
288
|
+
for item in cur:
|
|
289
|
+
_walk(item)
|
|
290
|
+
|
|
291
|
+
_walk(node)
|
|
292
|
+
return acc
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def normalize_carved_record_fields(obj: Dict[str, Any], schema_tag: str) -> Dict[str, Any]:
|
|
296
|
+
"""Map multi-platform web cache JSON fields onto StarShield's canonical record layout."""
|
|
297
|
+
props = obj.get("properties") if isinstance(obj.get("properties"), dict) else {}
|
|
298
|
+
edited_sub = obj.get("edited") if isinstance(obj.get("edited"), dict) else {}
|
|
299
|
+
|
|
300
|
+
msg_id = ""
|
|
301
|
+
for k in _PORTAL_ID_KEYS + ("ts", "t"):
|
|
302
|
+
v = obj.get(k)
|
|
303
|
+
if v is not None and str(v).strip() != "":
|
|
304
|
+
msg_id = str(v).strip()
|
|
305
|
+
break
|
|
306
|
+
|
|
307
|
+
sender = ""
|
|
308
|
+
for k in ("from", "user", "sender", "author", "actor", "vendor", "supplier", "beneficiary"):
|
|
309
|
+
v = obj.get(k)
|
|
310
|
+
if v is not None and str(v).strip() != "":
|
|
311
|
+
sender = str(v).strip()
|
|
312
|
+
break
|
|
313
|
+
|
|
314
|
+
compose_ts = None
|
|
315
|
+
for k in ("composetime", "ts", "t", "timestamp", "created_at", "issued_at", "date"):
|
|
316
|
+
if obj.get(k) not in (None, ""):
|
|
317
|
+
compose_ts = obj.get(k)
|
|
318
|
+
break
|
|
319
|
+
|
|
320
|
+
del_ts = (
|
|
321
|
+
props.get("deletetime")
|
|
322
|
+
or obj.get("deletetime")
|
|
323
|
+
or obj.get("deleted_ts")
|
|
324
|
+
or obj.get("deleted_at")
|
|
325
|
+
or obj.get("retracted_at")
|
|
326
|
+
or ("true" if obj.get("subtype") == "message_deleted" or obj.get("is_deleted") is True else None)
|
|
327
|
+
)
|
|
328
|
+
edit_ts = (
|
|
329
|
+
props.get("edittime")
|
|
330
|
+
or obj.get("edittime")
|
|
331
|
+
or edited_sub.get("ts")
|
|
332
|
+
or obj.get("edited_at")
|
|
333
|
+
or obj.get("updated_at")
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
if schema_tag == "chatsvc":
|
|
337
|
+
content_str = str(obj.get("content", ""))
|
|
338
|
+
else:
|
|
339
|
+
primary_parts: List[str] = []
|
|
340
|
+
for k in ("content", "text", "body", "message", "payment_instructions", "memo", "note", "description"):
|
|
341
|
+
v = obj.get(k)
|
|
342
|
+
if isinstance(v, str) and v.strip():
|
|
343
|
+
primary_parts.append(v.strip())
|
|
344
|
+
# Append structured banking/routing fields if present outside free-text body
|
|
345
|
+
for k in ("beneficiary", "payee", "iban", "beneficiary_iban", "sort_code", "account_number", "swift", "bic", "amount"):
|
|
346
|
+
v = obj.get(k)
|
|
347
|
+
if v is not None and str(v).strip():
|
|
348
|
+
primary_parts.append(f"{k}={str(v).strip()}")
|
|
349
|
+
if primary_parts:
|
|
350
|
+
content_str = " | ".join(primary_parts)
|
|
351
|
+
else:
|
|
352
|
+
content_str = " | ".join(_collect_string_scalars(obj))
|
|
353
|
+
|
|
354
|
+
return {
|
|
355
|
+
"message_id": msg_id,
|
|
356
|
+
"sender_mri": sender,
|
|
357
|
+
"composetime": compose_ts,
|
|
358
|
+
"deletetime": del_ts,
|
|
359
|
+
"edittime": edit_ts,
|
|
360
|
+
"cached_content": content_str,
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def _extract_json_objects_from_bytes(
|
|
365
|
+
raw_val: bytes,
|
|
366
|
+
schema_mode: str = "auto",
|
|
367
|
+
) -> List[Tuple[Dict[str, Any], str]]:
|
|
368
|
+
"""
|
|
369
|
+
Scan UTF-8 (and UTF-16LE if NUL bytes exist) using `json.JSONDecoder.raw_decode`.
|
|
370
|
+
Advances past the full span of each decoded top-level JSON object so nested child
|
|
371
|
+
objects (e.g., `properties: {deletetime: ...}`) are NEVER mis-carved as standalone records.
|
|
372
|
+
"""
|
|
373
|
+
candidates: List[str] = [raw_val.decode("utf-8", errors="ignore")]
|
|
374
|
+
if b"\x00" in raw_val:
|
|
375
|
+
candidates.append(raw_val.decode("utf-16-le", errors="ignore"))
|
|
376
|
+
|
|
377
|
+
decoder = json.JSONDecoder()
|
|
378
|
+
found: List[Tuple[Dict[str, Any], str]] = []
|
|
379
|
+
seen_hashes = set()
|
|
380
|
+
|
|
381
|
+
for text in candidates:
|
|
382
|
+
pos = 0
|
|
383
|
+
n = len(text)
|
|
384
|
+
while pos < n:
|
|
385
|
+
brace_idx = text.find("{", pos)
|
|
386
|
+
if brace_idx == -1:
|
|
387
|
+
break
|
|
388
|
+
try:
|
|
389
|
+
obj, end_idx = decoder.raw_decode(text, brace_idx)
|
|
390
|
+
except json.JSONDecodeError:
|
|
391
|
+
pos = brace_idx + 1
|
|
392
|
+
continue
|
|
393
|
+
|
|
394
|
+
# Advance past the entire top-level JSON object so nested dicts are skipped
|
|
395
|
+
pos = max(end_idx, brace_idx + 1)
|
|
396
|
+
schema_tag = classify_carved_json_object(obj, schema_mode=schema_mode)
|
|
397
|
+
if schema_tag is not None:
|
|
398
|
+
canonical = json.dumps(obj, sort_keys=True, ensure_ascii=False)
|
|
399
|
+
h = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
400
|
+
if h not in seen_hashes:
|
|
401
|
+
seen_hashes.add(h)
|
|
402
|
+
found.append((obj, schema_tag))
|
|
403
|
+
|
|
404
|
+
return found
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
def parse_leveldb_wal(raw_bytes: bytes, schema_mode: str = "auto") -> Dict[str, Any]:
|
|
408
|
+
"""Parse a Chromium LevelDB `.log` file, verifying CRC32C, WriteBatch bounds, and EOF completeness."""
|
|
409
|
+
if schema_mode not in VALID_SCHEMA_MODES:
|
|
410
|
+
raise ValueError(f"Unsupported schema_mode={schema_mode!r}; expected one of {VALID_SCHEMA_MODES}")
|
|
411
|
+
|
|
412
|
+
total_len = len(raw_bytes)
|
|
413
|
+
physical_records_ok = 0
|
|
414
|
+
corrupted_records: List[Dict[str, Any]] = []
|
|
415
|
+
recovered_entries: List[Dict[str, Any]] = []
|
|
416
|
+
stats: Dict[str, int] = {"value_ops_total": 0, "blink_v8_binary_only_values": 0}
|
|
417
|
+
|
|
418
|
+
frag_buf = bytearray()
|
|
419
|
+
frag_start_offset: int | None = None
|
|
420
|
+
|
|
421
|
+
block_idx = 0
|
|
422
|
+
while block_idx * BLOCK_SIZE < total_len:
|
|
423
|
+
block_start = block_idx * BLOCK_SIZE
|
|
424
|
+
block_end = min(block_start + BLOCK_SIZE, total_len)
|
|
425
|
+
pos = block_start
|
|
426
|
+
|
|
427
|
+
while pos + HEADER_SIZE <= block_end:
|
|
428
|
+
stored_crc, length, rec_type = struct.unpack_from("<IHB", raw_bytes, pos)
|
|
429
|
+
if stored_crc == 0 and length == 0 and rec_type == REC_ZERO:
|
|
430
|
+
rem = raw_bytes[pos:block_end]
|
|
431
|
+
if any(b != 0 for b in rem):
|
|
432
|
+
corrupted_records.append(
|
|
433
|
+
{
|
|
434
|
+
"byte_offset": pos,
|
|
435
|
+
"reason": "Non-zero bytes following zero-header preallocation region in 32KiB block",
|
|
436
|
+
}
|
|
437
|
+
)
|
|
438
|
+
pos = block_end
|
|
439
|
+
break
|
|
440
|
+
|
|
441
|
+
rec_offset = pos
|
|
442
|
+
data_start = pos + HEADER_SIZE
|
|
443
|
+
data_end = data_start + length
|
|
444
|
+
if data_end > block_end:
|
|
445
|
+
corrupted_records.append(
|
|
446
|
+
{
|
|
447
|
+
"byte_offset": rec_offset,
|
|
448
|
+
"reason": "Record length exceeds 32KiB LevelDB block boundary or truncated EOF",
|
|
449
|
+
}
|
|
450
|
+
)
|
|
451
|
+
frag_buf.clear()
|
|
452
|
+
frag_start_offset = None
|
|
453
|
+
pos = block_end
|
|
454
|
+
break
|
|
455
|
+
|
|
456
|
+
chunk = raw_bytes[data_start:data_end]
|
|
457
|
+
pos = data_end
|
|
458
|
+
|
|
459
|
+
expected_masked = mask_crc32c(crc32c(bytes([rec_type]) + chunk))
|
|
460
|
+
if stored_crc != expected_masked:
|
|
461
|
+
corrupted_records.append(
|
|
462
|
+
{
|
|
463
|
+
"byte_offset": rec_offset,
|
|
464
|
+
"record_type": rec_type,
|
|
465
|
+
"reason": f"CRC32C mismatch (stored=0x{stored_crc:08x}, computed=0x{expected_masked:08x})",
|
|
466
|
+
}
|
|
467
|
+
)
|
|
468
|
+
frag_buf.clear()
|
|
469
|
+
frag_start_offset = None
|
|
470
|
+
continue
|
|
471
|
+
|
|
472
|
+
physical_records_ok += 1
|
|
473
|
+
|
|
474
|
+
if rec_type == REC_FULL:
|
|
475
|
+
if frag_start_offset is not None:
|
|
476
|
+
corrupted_records.append(
|
|
477
|
+
{
|
|
478
|
+
"byte_offset": frag_start_offset,
|
|
479
|
+
"reason": "Unclosed FIRST/MIDDLE fragmented record interrupted by FULL record",
|
|
480
|
+
}
|
|
481
|
+
)
|
|
482
|
+
frag_buf.clear()
|
|
483
|
+
frag_start_offset = None
|
|
484
|
+
logical_bytes = chunk
|
|
485
|
+
logical_offset = rec_offset
|
|
486
|
+
elif rec_type == REC_FIRST:
|
|
487
|
+
if frag_start_offset is not None:
|
|
488
|
+
corrupted_records.append(
|
|
489
|
+
{
|
|
490
|
+
"byte_offset": frag_start_offset,
|
|
491
|
+
"reason": "Unclosed FIRST/MIDDLE fragmented record interrupted by new FIRST record",
|
|
492
|
+
}
|
|
493
|
+
)
|
|
494
|
+
frag_buf = bytearray(chunk)
|
|
495
|
+
frag_start_offset = rec_offset
|
|
496
|
+
continue
|
|
497
|
+
elif rec_type == REC_MIDDLE:
|
|
498
|
+
if frag_start_offset is None:
|
|
499
|
+
corrupted_records.append(
|
|
500
|
+
{"byte_offset": rec_offset, "reason": "Orphan MIDDLE fragment without FIRST"}
|
|
501
|
+
)
|
|
502
|
+
continue
|
|
503
|
+
if len(frag_buf) + len(chunk) > MAX_LOGICAL_RECORD_BYTES:
|
|
504
|
+
corrupted_records.append(
|
|
505
|
+
{"byte_offset": frag_start_offset, "reason": "Fragmented record exceeds 32MiB safety limit"}
|
|
506
|
+
)
|
|
507
|
+
frag_buf.clear()
|
|
508
|
+
frag_start_offset = None
|
|
509
|
+
continue
|
|
510
|
+
frag_buf.extend(chunk)
|
|
511
|
+
continue
|
|
512
|
+
elif rec_type == REC_LAST:
|
|
513
|
+
if frag_start_offset is None:
|
|
514
|
+
corrupted_records.append(
|
|
515
|
+
{"byte_offset": rec_offset, "reason": "Orphan LAST fragment without FIRST"}
|
|
516
|
+
)
|
|
517
|
+
continue
|
|
518
|
+
if len(frag_buf) + len(chunk) > MAX_LOGICAL_RECORD_BYTES:
|
|
519
|
+
corrupted_records.append(
|
|
520
|
+
{"byte_offset": frag_start_offset, "reason": "Fragmented record exceeds 32MiB safety limit"}
|
|
521
|
+
)
|
|
522
|
+
frag_buf.clear()
|
|
523
|
+
frag_start_offset = None
|
|
524
|
+
continue
|
|
525
|
+
frag_buf.extend(chunk)
|
|
526
|
+
logical_bytes = bytes(frag_buf)
|
|
527
|
+
logical_offset = frag_start_offset
|
|
528
|
+
frag_buf.clear()
|
|
529
|
+
frag_start_offset = None
|
|
530
|
+
else:
|
|
531
|
+
corrupted_records.append(
|
|
532
|
+
{"byte_offset": rec_offset, "reason": f"Unknown LevelDB record type {rec_type}"}
|
|
533
|
+
)
|
|
534
|
+
frag_buf.clear()
|
|
535
|
+
frag_start_offset = None
|
|
536
|
+
continue
|
|
537
|
+
|
|
538
|
+
_decode_writebatch_into(
|
|
539
|
+
logical_bytes,
|
|
540
|
+
logical_offset,
|
|
541
|
+
recovered_entries,
|
|
542
|
+
corrupted_records,
|
|
543
|
+
schema_mode=schema_mode,
|
|
544
|
+
stats=stats,
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
if pos < block_end:
|
|
548
|
+
trailer = raw_bytes[pos:block_end]
|
|
549
|
+
if any(b != 0 for b in trailer):
|
|
550
|
+
corrupted_records.append(
|
|
551
|
+
{
|
|
552
|
+
"byte_offset": pos,
|
|
553
|
+
"reason": f"Non-zero truncated header/trailer ({len(trailer)} bytes < 7-byte header)",
|
|
554
|
+
}
|
|
555
|
+
)
|
|
556
|
+
|
|
557
|
+
block_idx += 1
|
|
558
|
+
|
|
559
|
+
if frag_start_offset is not None:
|
|
560
|
+
corrupted_records.append(
|
|
561
|
+
{
|
|
562
|
+
"byte_offset": frag_start_offset,
|
|
563
|
+
"reason": "Truncated logical record: unclosed FIRST/MIDDLE fragment at EOF",
|
|
564
|
+
}
|
|
565
|
+
)
|
|
566
|
+
|
|
567
|
+
chatsvc_count = sum(1 for r in recovered_entries if r.get("schema_detected") == "chatsvc")
|
|
568
|
+
return {
|
|
569
|
+
"parser": "leveldb_wal_32k_crc32c_v0.3.6",
|
|
570
|
+
"schema_mode": schema_mode,
|
|
571
|
+
"file_size_bytes": total_len,
|
|
572
|
+
"valid_physical_frames": physical_records_ok,
|
|
573
|
+
"writebatch_value_ops_seen": stats["value_ops_total"],
|
|
574
|
+
"blink_v8_non_json_values_skipped": stats["blink_v8_binary_only_values"],
|
|
575
|
+
"corrupted_frames_count": len(corrupted_records),
|
|
576
|
+
"corrupted_frames": corrupted_records,
|
|
577
|
+
"recovered_records_count": len(recovered_entries),
|
|
578
|
+
"recovered_chatsvc_records_count": chatsvc_count if schema_mode == "chatsvc" else len(recovered_entries),
|
|
579
|
+
"recovered_records": recovered_entries,
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def _decode_writebatch_into(
|
|
584
|
+
batch: bytes,
|
|
585
|
+
logical_offset: int,
|
|
586
|
+
out_entries: List[Dict[str, Any]],
|
|
587
|
+
corrupted: List[Dict[str, Any]],
|
|
588
|
+
schema_mode: str = "auto",
|
|
589
|
+
stats: Dict[str, int] | None = None,
|
|
590
|
+
) -> None:
|
|
591
|
+
if len(batch) < 12:
|
|
592
|
+
corrupted.append({"byte_offset": logical_offset, "reason": "WriteBatch shorter than 12-byte header"})
|
|
593
|
+
return
|
|
594
|
+
|
|
595
|
+
seq_num, declared_count = struct.unpack_from("<QI", batch, 0)
|
|
596
|
+
idx = 12
|
|
597
|
+
parsed_ops = 0
|
|
598
|
+
staged_entries: List[Dict[str, Any]] = []
|
|
599
|
+
local_value_ops = 0
|
|
600
|
+
local_v8_binary_only = 0
|
|
601
|
+
|
|
602
|
+
try:
|
|
603
|
+
while idx < len(batch) and parsed_ops < declared_count:
|
|
604
|
+
op_type = batch[idx]
|
|
605
|
+
idx += 1
|
|
606
|
+
key_len, idx = decode_varint(batch, idx)
|
|
607
|
+
if idx + key_len > len(batch):
|
|
608
|
+
raise ValueError(f"Truncated key bytes (declared={key_len}, remaining={len(batch) - idx})")
|
|
609
|
+
key_bytes = batch[idx : idx + key_len]
|
|
610
|
+
idx += key_len
|
|
611
|
+
|
|
612
|
+
if op_type == 1:
|
|
613
|
+
val_len, idx = decode_varint(batch, idx)
|
|
614
|
+
if idx + val_len > len(batch):
|
|
615
|
+
raise ValueError(f"Truncated value bytes (declared={val_len}, remaining={len(batch) - idx})")
|
|
616
|
+
val_bytes = batch[idx : idx + val_len]
|
|
617
|
+
idx += val_len
|
|
618
|
+
local_value_ops += 1
|
|
619
|
+
|
|
620
|
+
carved_objs = _extract_json_objects_from_bytes(val_bytes, schema_mode=schema_mode)
|
|
621
|
+
if not carved_objs and b"\xff" in val_bytes[:8]:
|
|
622
|
+
local_v8_binary_only += 1
|
|
623
|
+
|
|
624
|
+
for obj, schema_tag in carved_objs:
|
|
625
|
+
norm = normalize_carved_record_fields(obj, schema_tag)
|
|
626
|
+
del_ts = norm["deletetime"]
|
|
627
|
+
edit_ts = norm["edittime"]
|
|
628
|
+
staged_entries.append(
|
|
629
|
+
{
|
|
630
|
+
"epistemic_tier": "OBSERVED",
|
|
631
|
+
"schema_detected": schema_tag,
|
|
632
|
+
"leveldb_batch_byte_offset": logical_offset,
|
|
633
|
+
"leveldb_sequence_number": seq_num,
|
|
634
|
+
"key_sha256": hashlib.sha256(key_bytes).hexdigest(),
|
|
635
|
+
"value_sha256": hashlib.sha256(val_bytes).hexdigest(),
|
|
636
|
+
"message_id": norm["message_id"],
|
|
637
|
+
"sender_mri": norm["sender_mri"],
|
|
638
|
+
"composetime": norm["composetime"],
|
|
639
|
+
"deletetime": del_ts,
|
|
640
|
+
"edittime": edit_ts,
|
|
641
|
+
"is_soft_deleted_in_ui": bool(del_ts),
|
|
642
|
+
"note_on_edittime": (
|
|
643
|
+
"Present value reflects post-edit state unless an earlier sequence_number "
|
|
644
|
+
"record with the same message_id exists in uncompacted log frames."
|
|
645
|
+
if edit_ts
|
|
646
|
+
else None
|
|
647
|
+
),
|
|
648
|
+
"cached_content": norm["cached_content"],
|
|
649
|
+
}
|
|
650
|
+
)
|
|
651
|
+
elif op_type == 0:
|
|
652
|
+
pass
|
|
653
|
+
else:
|
|
654
|
+
raise ValueError(f"Unsupported WriteBatch op_type=0x{op_type:02x}")
|
|
655
|
+
parsed_ops += 1
|
|
656
|
+
|
|
657
|
+
if parsed_ops != declared_count:
|
|
658
|
+
raise ValueError(f"WriteBatch op count mismatch (declared={declared_count}, parsed={parsed_ops})")
|
|
659
|
+
if idx != len(batch):
|
|
660
|
+
raise ValueError(f"Trailing unparsed bytes in WriteBatch ({len(batch) - idx} bytes remaining)")
|
|
661
|
+
|
|
662
|
+
except Exception as exc:
|
|
663
|
+
corrupted.append({"byte_offset": logical_offset, "reason": f"Malformed WriteBatch entry: {exc}"})
|
|
664
|
+
return
|
|
665
|
+
|
|
666
|
+
if stats is not None:
|
|
667
|
+
stats["value_ops_total"] = stats.get("value_ops_total", 0) + local_value_ops
|
|
668
|
+
stats["blink_v8_binary_only_values"] = stats.get("blink_v8_binary_only_values", 0) + local_v8_binary_only
|
|
669
|
+
out_entries.extend(staged_entries)
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def _has_leveldb_sstable_magic(path: Path) -> bool:
|
|
673
|
+
"""Verify the 8-byte LevelDB SSTable footer magic (`0xdb4775248b80fb57`) at `EOF - 8`."""
|
|
674
|
+
try:
|
|
675
|
+
size = path.stat().st_size
|
|
676
|
+
if size < 48:
|
|
677
|
+
return False
|
|
678
|
+
with path.open("rb") as fh:
|
|
679
|
+
fh.seek(size - 8)
|
|
680
|
+
return fh.read(8) == LEVELDB_SSTABLE_MAGIC_LE
|
|
681
|
+
except OSError:
|
|
682
|
+
return False
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
def inspect_chromium_storage_path(
|
|
686
|
+
target: Path,
|
|
687
|
+
max_mb: int = 64,
|
|
688
|
+
schema_mode: str = "auto",
|
|
689
|
+
redact: bool = True,
|
|
690
|
+
) -> Dict[str, Any]:
|
|
691
|
+
"""
|
|
692
|
+
Zero-PII local precheck gate (`GO` / `PARTIAL` / `NO_GO_COMPACTED_OR_UNSUPPORTED`)
|
|
693
|
+
for law-firm IT or forensic intake teams before instructing a paid triage pilot.
|
|
694
|
+
Never emits plaintext message content, filenames, or raw hashes in default masked mode.
|
|
695
|
+
"""
|
|
696
|
+
if max_mb < 1:
|
|
697
|
+
raise ValueError("--max-mb safety limit must be >= 1 MiB")
|
|
698
|
+
if not target.exists():
|
|
699
|
+
if redact:
|
|
700
|
+
raise OSError(
|
|
701
|
+
"Unable to access target path (OS path suppressed in masked mode; "
|
|
702
|
+
"pass --unredacted in an offline lab for path diagnostics)"
|
|
703
|
+
)
|
|
704
|
+
raise FileNotFoundError(f"Target path does not exist: {target}")
|
|
705
|
+
|
|
706
|
+
limit_bytes = max_mb * 1024 * 1024
|
|
707
|
+
log_files: List[Path] = []
|
|
708
|
+
ldb_files: List[Path] = []
|
|
709
|
+
blob_files_count = 0
|
|
710
|
+
|
|
711
|
+
if target.is_file():
|
|
712
|
+
suf = target.suffix.lower()
|
|
713
|
+
if suf == ".log":
|
|
714
|
+
log_files.append(target)
|
|
715
|
+
elif suf in (".ldb", ".sst"):
|
|
716
|
+
ldb_files.append(target)
|
|
717
|
+
elif target.is_dir():
|
|
718
|
+
for p in sorted(target.rglob("*")):
|
|
719
|
+
if not p.is_file():
|
|
720
|
+
continue
|
|
721
|
+
suf = p.suffix.lower()
|
|
722
|
+
if suf == ".log":
|
|
723
|
+
log_files.append(p)
|
|
724
|
+
elif suf in (".ldb", ".sst"):
|
|
725
|
+
ldb_files.append(p)
|
|
726
|
+
elif ".blob" in str(p.parent).lower() or suf == ".blob":
|
|
727
|
+
blob_files_count += 1
|
|
728
|
+
|
|
729
|
+
valid_wal_frames = 0
|
|
730
|
+
carvable_records = 0
|
|
731
|
+
soft_deleted_records = 0
|
|
732
|
+
v8_binary_only_values = 0
|
|
733
|
+
oversize_logs_skipped = 0
|
|
734
|
+
schemas_seen: Dict[str, int] = {}
|
|
735
|
+
|
|
736
|
+
for lf in log_files:
|
|
737
|
+
try:
|
|
738
|
+
sz = lf.stat().st_size
|
|
739
|
+
except OSError:
|
|
740
|
+
continue
|
|
741
|
+
if sz > limit_bytes:
|
|
742
|
+
oversize_logs_skipped += 1
|
|
743
|
+
continue
|
|
744
|
+
if sz == 0:
|
|
745
|
+
continue
|
|
746
|
+
parsed = parse_leveldb_wal(lf.read_bytes(), schema_mode=schema_mode)
|
|
747
|
+
valid_wal_frames += parsed["valid_physical_frames"]
|
|
748
|
+
carvable_records += parsed["recovered_records_count"]
|
|
749
|
+
v8_binary_only_values += parsed["blink_v8_non_json_values_skipped"]
|
|
750
|
+
for rec in parsed["recovered_records"]:
|
|
751
|
+
st = str(rec.get("schema_detected", "unknown"))
|
|
752
|
+
schemas_seen[st] = schemas_seen.get(st, 0) + 1
|
|
753
|
+
if rec.get("is_soft_deleted_in_ui"):
|
|
754
|
+
soft_deleted_records += 1
|
|
755
|
+
|
|
756
|
+
verified_sstables = sum(1 for f in ldb_files if _has_leveldb_sstable_magic(f))
|
|
757
|
+
|
|
758
|
+
if carvable_records > 0 and verified_sstables == 0 and v8_binary_only_values == 0 and blob_files_count == 0:
|
|
759
|
+
verdict = "GO"
|
|
760
|
+
guidance = (
|
|
761
|
+
"Uncompacted LevelDB .log frames contain carvable JSON records. "
|
|
762
|
+
"Eligible for immediate byte-offset-cited extraction under v0.3.6."
|
|
763
|
+
)
|
|
764
|
+
elif carvable_records > 0:
|
|
765
|
+
verdict = "PARTIAL"
|
|
766
|
+
guidance = (
|
|
767
|
+
"Carvable JSON records are recoverable from uncompacted .log frames, alongside compacted "
|
|
768
|
+
".ldb SSTables, external .blob files, or binary-only Blink V8 values that fall outside v0.3.6 scope."
|
|
769
|
+
)
|
|
770
|
+
else:
|
|
771
|
+
verdict = "NO_GO_COMPACTED_OR_UNSUPPORTED"
|
|
772
|
+
guidance = (
|
|
773
|
+
"Zero carvable JSON records found in uncompacted .log files (storage may already be compacted "
|
|
774
|
+
"into Snappy .ldb SSTables, encoded solely in binary Blink V8 StructuredClone, or empty). "
|
|
775
|
+
"Under StarShield SOW terms, £0 pilot fee applies."
|
|
776
|
+
)
|
|
777
|
+
|
|
778
|
+
return {
|
|
779
|
+
"epistemic_tier": "OBSERVED",
|
|
780
|
+
"tool_version": "starshield-forensics/0.3.6",
|
|
781
|
+
"target_label": "[REDACTED_TARGET_PATH]" if redact else str(target),
|
|
782
|
+
"schema_mode": schema_mode,
|
|
783
|
+
"feasibility_verdict": verdict,
|
|
784
|
+
"wal_log_files_detected": len(log_files),
|
|
785
|
+
"wal_oversize_files_skipped": oversize_logs_skipped,
|
|
786
|
+
"valid_wal_physical_frames": valid_wal_frames,
|
|
787
|
+
"carvable_json_records_count": carvable_records,
|
|
788
|
+
"soft_deleted_json_records_count": soft_deleted_records,
|
|
789
|
+
"schemas_detected_counts": schemas_seen,
|
|
790
|
+
"compacted_sstable_files_detected": len(ldb_files),
|
|
791
|
+
"verified_leveldb_sstables_count": verified_sstables,
|
|
792
|
+
"indexeddb_blob_files_detected": blob_files_count,
|
|
793
|
+
"blink_v8_non_json_values_detected": v8_binary_only_values,
|
|
794
|
+
"intake_guidance": guidance,
|
|
795
|
+
}
|