dfindexeddb 20241031__py3-none-any.whl → 20251109__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dfindexeddb/indexeddb/chromium/blink.py +116 -74
- dfindexeddb/indexeddb/chromium/definitions.py +152 -124
- dfindexeddb/indexeddb/chromium/record.py +536 -348
- dfindexeddb/indexeddb/chromium/v8.py +112 -141
- dfindexeddb/indexeddb/cli.py +125 -114
- dfindexeddb/indexeddb/firefox/definitions.py +7 -4
- dfindexeddb/indexeddb/firefox/gecko.py +103 -79
- dfindexeddb/indexeddb/firefox/record.py +66 -24
- dfindexeddb/indexeddb/safari/definitions.py +12 -10
- dfindexeddb/indexeddb/safari/record.py +68 -51
- dfindexeddb/indexeddb/safari/webkit.py +112 -189
- dfindexeddb/indexeddb/types.py +5 -2
- dfindexeddb/leveldb/cli.py +146 -131
- dfindexeddb/leveldb/definitions.py +6 -2
- dfindexeddb/leveldb/descriptor.py +75 -45
- dfindexeddb/leveldb/ldb.py +39 -30
- dfindexeddb/leveldb/log.py +44 -27
- dfindexeddb/leveldb/plugins/chrome_notifications.py +30 -18
- dfindexeddb/leveldb/plugins/interface.py +5 -6
- dfindexeddb/leveldb/plugins/manager.py +11 -10
- dfindexeddb/leveldb/record.py +71 -62
- dfindexeddb/leveldb/utils.py +21 -13
- dfindexeddb/utils.py +35 -30
- dfindexeddb/version.py +2 -2
- dfindexeddb-20251109.dist-info/METADATA +222 -0
- dfindexeddb-20251109.dist-info/RECORD +40 -0
- {dfindexeddb-20241031.dist-info → dfindexeddb-20251109.dist-info}/WHEEL +1 -1
- dfindexeddb-20241031.dist-info/AUTHORS +0 -12
- dfindexeddb-20241031.dist-info/METADATA +0 -424
- dfindexeddb-20241031.dist-info/RECORD +0 -41
- {dfindexeddb-20241031.dist-info → dfindexeddb-20251109.dist-info}/entry_points.txt +0 -0
- {dfindexeddb-20241031.dist-info → dfindexeddb-20251109.dist-info/licenses}/LICENSE +0 -0
- {dfindexeddb-20241031.dist-info → dfindexeddb-20251109.dist-info}/top_level.txt +0 -0
|
@@ -23,8 +23,7 @@ from typing import Any, List, Tuple, Union
|
|
|
23
23
|
|
|
24
24
|
import snappy
|
|
25
25
|
|
|
26
|
-
from dfindexeddb import errors
|
|
27
|
-
from dfindexeddb import utils
|
|
26
|
+
from dfindexeddb import errors, utils
|
|
28
27
|
from dfindexeddb.indexeddb import types
|
|
29
28
|
from dfindexeddb.indexeddb.firefox import definitions
|
|
30
29
|
|
|
@@ -38,13 +37,15 @@ class IDBKey(utils.FromDecoderMixin):
|
|
|
38
37
|
type: the type of the key.
|
|
39
38
|
value: the value of the key.
|
|
40
39
|
"""
|
|
40
|
+
|
|
41
41
|
offset: int
|
|
42
42
|
type: definitions.IndexedDBKeyType
|
|
43
|
-
value: Union[bytes, datetime.datetime, float, list, str, None]
|
|
43
|
+
value: Union[bytes, datetime.datetime, float, list[Any], str, None]
|
|
44
44
|
|
|
45
45
|
@classmethod
|
|
46
46
|
def FromDecoder(
|
|
47
|
-
|
|
47
|
+
cls, decoder: utils.StreamDecoder, base_offset: int = 0
|
|
48
|
+
) -> IDBKey:
|
|
48
49
|
"""Decodes an IndexedDB key from the current position of the decoder.
|
|
49
50
|
|
|
50
51
|
Args:
|
|
@@ -77,10 +78,7 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
77
78
|
return offset, key_type, value
|
|
78
79
|
|
|
79
80
|
def _DecodeJSValInternal(
|
|
80
|
-
self,
|
|
81
|
-
key_type: int,
|
|
82
|
-
type_offset: int = 0,
|
|
83
|
-
recursion_depth: int = 0
|
|
81
|
+
self, key_type: int, type_offset: int = 0, recursion_depth: int = 0
|
|
84
82
|
) -> Any:
|
|
85
83
|
"""Decodes a Javascript Value.
|
|
86
84
|
|
|
@@ -97,21 +95,26 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
97
95
|
type has been parsed.
|
|
98
96
|
"""
|
|
99
97
|
if recursion_depth == definitions.MAX_RECURSION_DEPTH:
|
|
100
|
-
raise errors.ParserError(
|
|
98
|
+
raise errors.ParserError("Reached Maximum Recursion Depth")
|
|
101
99
|
|
|
100
|
+
value: Any = None
|
|
102
101
|
if key_type - type_offset >= definitions.IndexedDBKeyType.ARRAY:
|
|
103
102
|
type_offset += definitions.IndexedDBKeyType.ARRAY
|
|
104
|
-
if (
|
|
105
|
-
|
|
103
|
+
if (
|
|
104
|
+
type_offset
|
|
105
|
+
== definitions.IndexedDBKeyType.ARRAY * definitions.MAX_ARRAY_COLLAPSE
|
|
106
|
+
):
|
|
106
107
|
_ = self.ReadBytes(1)
|
|
107
108
|
type_offset = 0
|
|
108
109
|
value = []
|
|
109
|
-
while (
|
|
110
|
-
|
|
111
|
-
|
|
110
|
+
while self.NumRemainingBytes() and (key_type - type_offset) != (
|
|
111
|
+
definitions.IndexedDBKeyType.TERMINATOR
|
|
112
|
+
):
|
|
112
113
|
value.append(
|
|
113
114
|
self._DecodeJSValInternal(
|
|
114
|
-
key_type, type_offset, recursion_depth + 1
|
|
115
|
+
key_type, type_offset, recursion_depth + 1
|
|
116
|
+
)
|
|
117
|
+
)
|
|
115
118
|
type_offset = 0
|
|
116
119
|
if self.NumRemainingBytes():
|
|
117
120
|
_, key_bytes = self.PeekBytes(1)
|
|
@@ -122,7 +125,7 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
122
125
|
|
|
123
126
|
elif key_type - type_offset == definitions.IndexedDBKeyType.STRING:
|
|
124
127
|
_, value = self._DecodeAsStringy()
|
|
125
|
-
value = value.decode(
|
|
128
|
+
value = value.decode("utf-8") # TODO: test other text encoding types
|
|
126
129
|
elif key_type - type_offset == definitions.IndexedDBKeyType.DATE:
|
|
127
130
|
_, value = self._DecodeDate()
|
|
128
131
|
elif key_type - type_offset == definitions.IndexedDBKeyType.FLOAT:
|
|
@@ -131,7 +134,8 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
131
134
|
_, value = self._DecodeAsStringy()
|
|
132
135
|
else:
|
|
133
136
|
raise errors.ParserError(
|
|
134
|
-
f
|
|
137
|
+
f"Unknown key type parsed: {key_type - type_offset}"
|
|
138
|
+
)
|
|
135
139
|
return value
|
|
136
140
|
|
|
137
141
|
def _ReadUntilNull(self) -> bytearray:
|
|
@@ -140,13 +144,13 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
140
144
|
num_remaining_bytes = self.NumRemainingBytes()
|
|
141
145
|
while num_remaining_bytes:
|
|
142
146
|
_, c = self.ReadBytes(1)
|
|
143
|
-
if c == b
|
|
147
|
+
if c == b"\x00":
|
|
144
148
|
break
|
|
145
149
|
result += c
|
|
146
150
|
num_remaining_bytes -= 1
|
|
147
151
|
return result
|
|
148
152
|
|
|
149
|
-
def _DecodeAsStringy(self, element_size: int = 1) -> Tuple[int,
|
|
153
|
+
def _DecodeAsStringy(self, element_size: int = 1) -> Tuple[int, bytearray]:
|
|
150
154
|
"""Decodes a string buffer.
|
|
151
155
|
|
|
152
156
|
Args:
|
|
@@ -162,9 +166,11 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
162
166
|
offset, type_int = self.DecodeUint8()
|
|
163
167
|
if type_int % definitions.IndexedDBKeyType.ARRAY.value not in (
|
|
164
168
|
definitions.IndexedDBKeyType.STRING.value,
|
|
165
|
-
definitions.IndexedDBKeyType.BINARY.value
|
|
169
|
+
definitions.IndexedDBKeyType.BINARY.value,
|
|
170
|
+
):
|
|
166
171
|
raise errors.ParserError(
|
|
167
|
-
|
|
172
|
+
"Invalid type", hex(type_int), hex(type_int % 0x50)
|
|
173
|
+
)
|
|
168
174
|
|
|
169
175
|
i = 0
|
|
170
176
|
result = self._ReadUntilNull()
|
|
@@ -172,11 +178,11 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
172
178
|
if not result[i] & 0x80:
|
|
173
179
|
result[i] -= definitions.ONE_BYTE_ADJUST
|
|
174
180
|
elif element_size == 2 or not result[i] & 0x40:
|
|
175
|
-
c = struct.unpack_from(
|
|
181
|
+
c = struct.unpack_from(">H", result, i)[0]
|
|
176
182
|
d = c - 0x8000 - definitions.TWO_BYTE_ADJUST
|
|
177
|
-
struct.pack_into(
|
|
183
|
+
struct.pack_into(">H", result, i, d)
|
|
178
184
|
else:
|
|
179
|
-
raise errors.ParserError(
|
|
185
|
+
raise errors.ParserError("Unsupported byte") # TODO: add support.
|
|
180
186
|
i += 1
|
|
181
187
|
return offset + 1, result
|
|
182
188
|
|
|
@@ -191,24 +197,28 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
191
197
|
"""
|
|
192
198
|
_, type_byte = self.DecodeUint8()
|
|
193
199
|
if type_byte % definitions.IndexedDBKeyType.ARRAY not in (
|
|
194
|
-
definitions.IndexedDBKeyType.FLOAT,
|
|
200
|
+
definitions.IndexedDBKeyType.FLOAT,
|
|
201
|
+
definitions.IndexedDBKeyType.DATE,
|
|
202
|
+
):
|
|
195
203
|
raise errors.ParserError(
|
|
196
|
-
|
|
204
|
+
"Invalid type", hex(type_byte), hex(type_byte % 0x50)
|
|
205
|
+
)
|
|
197
206
|
|
|
198
207
|
float_offset, number_bytes = self.ReadBytes(
|
|
199
|
-
min(8, self.NumRemainingBytes())
|
|
208
|
+
min(8, self.NumRemainingBytes())
|
|
209
|
+
)
|
|
200
210
|
|
|
201
211
|
# pad to 64-bits
|
|
202
212
|
if len(number_bytes) != 8:
|
|
203
|
-
number_bytes += b
|
|
213
|
+
number_bytes += b"\x00" * (8 - len(number_bytes))
|
|
204
214
|
|
|
205
|
-
int_value = struct.unpack(
|
|
215
|
+
int_value = struct.unpack(">q", number_bytes)[0]
|
|
206
216
|
|
|
207
217
|
if int_value & 0x8000000000000000:
|
|
208
218
|
int_value = int_value & 0x7FFFFFFFFFFFFFFF
|
|
209
219
|
else:
|
|
210
220
|
int_value = -int_value
|
|
211
|
-
return float_offset, struct.unpack(
|
|
221
|
+
return float_offset, struct.unpack(">d", struct.pack(">q", int_value))[0]
|
|
212
222
|
|
|
213
223
|
def _DecodeBinary(self) -> Tuple[int, bytes]:
|
|
214
224
|
"""Decodes a binary value.
|
|
@@ -220,18 +230,21 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
220
230
|
ParserError: when an invalid type byte is encountered.
|
|
221
231
|
"""
|
|
222
232
|
_, type_byte = self.ReadBytes(1)
|
|
223
|
-
if (
|
|
224
|
-
definitions.IndexedDBKeyType.
|
|
233
|
+
if (
|
|
234
|
+
type_byte[0] % definitions.IndexedDBKeyType.ARRAY
|
|
235
|
+
!= definitions.IndexedDBKeyType.BINARY
|
|
236
|
+
):
|
|
225
237
|
raise errors.ParserError(
|
|
226
|
-
f
|
|
238
|
+
f"Invalid type {type_byte[0] % definitions.IndexedDBKeyType.ARRAY}"
|
|
239
|
+
)
|
|
227
240
|
values = bytearray()
|
|
228
241
|
offset = self.stream.tell()
|
|
229
242
|
while self.NumRemainingBytes():
|
|
230
243
|
_, value = self.ReadBytes(1)
|
|
231
|
-
if value == definitions.IndexedDBKeyType.TERMINATOR:
|
|
244
|
+
if value[0] == definitions.IndexedDBKeyType.TERMINATOR:
|
|
232
245
|
break
|
|
233
246
|
values.append(value[0])
|
|
234
|
-
return offset, values
|
|
247
|
+
return offset, bytes(values)
|
|
235
248
|
|
|
236
249
|
def _DecodeDate(self) -> Tuple[int, datetime.datetime]:
|
|
237
250
|
"""Decodes a date.
|
|
@@ -241,7 +254,8 @@ class KeyDecoder(utils.StreamDecoder):
|
|
|
241
254
|
"""
|
|
242
255
|
offset, value = self._DecodeFloat()
|
|
243
256
|
return offset, datetime.datetime.fromtimestamp(
|
|
244
|
-
value/1000, tz=datetime.timezone.utc
|
|
257
|
+
value / 1000, tz=datetime.timezone.utc
|
|
258
|
+
)
|
|
245
259
|
|
|
246
260
|
|
|
247
261
|
class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
@@ -265,7 +279,7 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
265
279
|
def _PeekTagAndData(self) -> tuple[int, int]:
|
|
266
280
|
"""Peeks Tag and Data values."""
|
|
267
281
|
_, pair_bytes = self.decoder.PeekBytes(8)
|
|
268
|
-
pair = int.from_bytes(pair_bytes, byteorder=
|
|
282
|
+
pair = int.from_bytes(pair_bytes, byteorder="little", signed=False)
|
|
269
283
|
tag = pair >> 32
|
|
270
284
|
data = pair & 0x00000000FFFFFFFF
|
|
271
285
|
return tag, data
|
|
@@ -292,14 +306,14 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
292
306
|
number_chars = data & 0x7FFFFFFF
|
|
293
307
|
latin1 = data & 0x80000000
|
|
294
308
|
if number_chars > definitions.MAX_LENGTH:
|
|
295
|
-
raise errors.ParserError(
|
|
309
|
+
raise errors.ParserError("Bad serialized string length")
|
|
296
310
|
|
|
297
311
|
if latin1:
|
|
298
|
-
_,
|
|
299
|
-
chars =
|
|
312
|
+
_, char_bytes = self.decoder.ReadBytes(number_chars)
|
|
313
|
+
chars = char_bytes.decode("latin-1")
|
|
300
314
|
else:
|
|
301
|
-
_,
|
|
302
|
-
chars =
|
|
315
|
+
_, char_bytes = self.decoder.ReadBytes(number_chars * 2)
|
|
316
|
+
chars = char_bytes.decode("utf-16-le")
|
|
303
317
|
|
|
304
318
|
return chars
|
|
305
319
|
|
|
@@ -316,12 +330,13 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
316
330
|
is_negative = data & 0x80000000
|
|
317
331
|
if length == 0:
|
|
318
332
|
return 0
|
|
319
|
-
contents =
|
|
333
|
+
contents = bytearray()
|
|
320
334
|
for _ in range(length):
|
|
321
335
|
_, element = self.decoder.ReadBytes(8)
|
|
322
336
|
contents.extend(element)
|
|
323
337
|
return int.from_bytes(
|
|
324
|
-
contents, byteorder=
|
|
338
|
+
contents, byteorder="little", signed=bool(is_negative)
|
|
339
|
+
)
|
|
325
340
|
|
|
326
341
|
def _DecodeTypedArray(
|
|
327
342
|
self, array_type: int, number_elements: int, is_version1: bool = False
|
|
@@ -344,18 +359,16 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
344
359
|
number_elements = 0
|
|
345
360
|
|
|
346
361
|
if is_version1:
|
|
347
|
-
|
|
348
|
-
|
|
362
|
+
self._DecodeV1ArrayBuffer(array_type, number_elements)
|
|
363
|
+
byte_offset = 0 # pylint: disable=unused-variable
|
|
349
364
|
else:
|
|
350
365
|
value = self._StartRead()
|
|
351
|
-
|
|
366
|
+
_, byte_offset = self.decoder.DecodeUint64() # pylint: disable=unused-variable
|
|
352
367
|
|
|
353
|
-
self.all_objects[dummy_index] = value
|
|
368
|
+
self.all_objects[dummy_index] = value # pylint: disable=possibly-used-before-assignment
|
|
354
369
|
return value
|
|
355
370
|
|
|
356
|
-
def _DecodeArrayBuffer(
|
|
357
|
-
self, buffer_type: int, data: int
|
|
358
|
-
) -> bytes:
|
|
371
|
+
def _DecodeArrayBuffer(self, buffer_type: int, data: int) -> bytes:
|
|
359
372
|
"""Decodes an ArrayBuffer.
|
|
360
373
|
|
|
361
374
|
Args:
|
|
@@ -367,17 +380,19 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
367
380
|
"""
|
|
368
381
|
if buffer_type == definitions.StructuredDataType.ARRAY_BUFFER_OBJECT:
|
|
369
382
|
_, number_bytes = self.decoder.DecodeUint64()
|
|
370
|
-
elif (
|
|
371
|
-
|
|
383
|
+
elif (
|
|
384
|
+
buffer_type
|
|
385
|
+
== definitions.StructuredDataType.RESIZABLE_ARRAY_BUFFER_OBJECT
|
|
386
|
+
):
|
|
372
387
|
_, number_bytes = self.decoder.DecodeUint64()
|
|
373
|
-
_,
|
|
388
|
+
_, max_bytes = self.decoder.DecodeUint64() # pylint: disable=unused-variable
|
|
374
389
|
else:
|
|
375
390
|
number_bytes = data
|
|
376
391
|
|
|
377
392
|
_, buffer = self.decoder.ReadBytes(number_bytes)
|
|
378
393
|
return buffer
|
|
379
394
|
|
|
380
|
-
def _DecodeV1ArrayBuffer(self, array_type: int, number_elements: int):
|
|
395
|
+
def _DecodeV1ArrayBuffer(self, array_type: int, number_elements: int) -> None:
|
|
381
396
|
"""Decodes a V1 ArrayBuffer.
|
|
382
397
|
|
|
383
398
|
Not currently implemented.
|
|
@@ -385,7 +400,7 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
385
400
|
Raises:
|
|
386
401
|
NotImplementedError: because it's not yet supported.
|
|
387
402
|
"""
|
|
388
|
-
raise NotImplementedError(
|
|
403
|
+
raise NotImplementedError("V1 Array Buffer not supported")
|
|
389
404
|
|
|
390
405
|
def DecodeValue(self) -> Any:
|
|
391
406
|
"""Decodes a Javascript value."""
|
|
@@ -394,7 +409,7 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
394
409
|
|
|
395
410
|
tag, _ = self._PeekTagAndData()
|
|
396
411
|
if tag == definitions.StructuredDataType.TRANSFER_MAP_HEADER:
|
|
397
|
-
raise errors.ParserError(
|
|
412
|
+
raise errors.ParserError("Transfer Map is not supported.")
|
|
398
413
|
|
|
399
414
|
value = self._StartRead()
|
|
400
415
|
|
|
@@ -437,10 +452,10 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
437
452
|
"""
|
|
438
453
|
tag, _ = self._DecodeTagAndData()
|
|
439
454
|
if tag != definitions.StructuredDataType.HEADER:
|
|
440
|
-
raise errors.ParserError(
|
|
455
|
+
raise errors.ParserError("Header tag not found.")
|
|
441
456
|
return True
|
|
442
457
|
|
|
443
|
-
def _DecodeRegexp(self, flags: int):
|
|
458
|
+
def _DecodeRegexp(self, flags: int) -> types.RegExp:
|
|
444
459
|
"""Decodes a regular expression.
|
|
445
460
|
|
|
446
461
|
Args:
|
|
@@ -451,10 +466,10 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
451
466
|
"""
|
|
452
467
|
tag, string_data = self._DecodeTagAndData()
|
|
453
468
|
if tag != definitions.StructuredDataType.STRING:
|
|
454
|
-
raise errors.ParserError(
|
|
469
|
+
raise errors.ParserError("String tag not found")
|
|
455
470
|
|
|
456
471
|
pattern = self._DecodeString(string_data)
|
|
457
|
-
return types.RegExp(pattern=pattern, flags=flags)
|
|
472
|
+
return types.RegExp(pattern=pattern, flags=str(flags))
|
|
458
473
|
|
|
459
474
|
def _StartRead(self) -> Any:
|
|
460
475
|
"""Reads the start of a serialized value.
|
|
@@ -463,6 +478,7 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
463
478
|
ParserError: when an unsupported StructuredDataType,
|
|
464
479
|
StructuredCloneTag or unknown tag is read.
|
|
465
480
|
"""
|
|
481
|
+
value: Any = None
|
|
466
482
|
tag, data = self._DecodeTagAndData()
|
|
467
483
|
if tag == definitions.StructuredDataType.NULL:
|
|
468
484
|
value = types.Null()
|
|
@@ -491,7 +507,8 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
491
507
|
elif tag == definitions.StructuredDataType.DATE_OBJECT:
|
|
492
508
|
_, timestamp = self.decoder.DecodeDouble()
|
|
493
509
|
value = datetime.datetime.fromtimestamp(
|
|
494
|
-
timestamp/1000, tz=datetime.timezone.utc
|
|
510
|
+
timestamp / 1000, tz=datetime.timezone.utc
|
|
511
|
+
)
|
|
495
512
|
self.all_objects.append(value)
|
|
496
513
|
elif tag == definitions.StructuredDataType.REGEXP_OBJECT:
|
|
497
514
|
value = self._DecodeRegexp(data)
|
|
@@ -532,31 +549,35 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
532
549
|
self._object_stack.append(value)
|
|
533
550
|
self.all_objects.append(value)
|
|
534
551
|
elif tag <= definitions.StructuredDataType.FLOAT_MAX:
|
|
535
|
-
value = struct.unpack(
|
|
536
|
-
elif (
|
|
537
|
-
|
|
552
|
+
value = struct.unpack("<d", struct.pack("<II", data, tag))[0]
|
|
553
|
+
elif (
|
|
554
|
+
definitions.StructuredDataType.TYPED_ARRAY_V1_INT8
|
|
555
|
+
<= tag
|
|
556
|
+
<= definitions.StructuredDataType.TYPED_ARRAY_V1_UINT8_CLAMPED
|
|
557
|
+
):
|
|
538
558
|
value = self._DecodeTypedArray(tag, data)
|
|
539
|
-
elif tag in
|
|
559
|
+
elif tag in definitions.StructuredDataType.__members__.values():
|
|
540
560
|
raise errors.ParserError(
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
elif tag in
|
|
561
|
+
"Unsupported StructuredDataType", definitions.StructuredDataType(tag)
|
|
562
|
+
)
|
|
563
|
+
elif tag in definitions.StructuredCloneTags.__members__.values():
|
|
544
564
|
raise errors.ParserError(
|
|
545
|
-
|
|
546
|
-
|
|
565
|
+
"Unsupported StructuredCloneTag", definitions.StructuredCloneTags(tag)
|
|
566
|
+
)
|
|
547
567
|
else:
|
|
548
|
-
raise errors.ParserError(
|
|
568
|
+
raise errors.ParserError("Unknown tag", hex(tag))
|
|
549
569
|
|
|
550
570
|
# align the stream to an 8 byte boundary
|
|
551
571
|
offset = self.decoder.stream.tell() % 8
|
|
552
572
|
if offset:
|
|
553
|
-
_,
|
|
573
|
+
_, slack_bytes = self.decoder.ReadBytes(8 - offset) # pylint: disable=unused-variable
|
|
554
574
|
|
|
555
575
|
return value
|
|
556
576
|
|
|
557
577
|
@classmethod
|
|
558
578
|
def FromDecoder(
|
|
559
|
-
cls, decoder: utils.StreamDecoder, base_offset: int = 0
|
|
579
|
+
cls, decoder: utils.StreamDecoder, base_offset: int = 0
|
|
580
|
+
) -> Any:
|
|
560
581
|
"""Decodes the parsed JavaScript object from the StreamDecoder.
|
|
561
582
|
|
|
562
583
|
Args:
|
|
@@ -584,14 +605,17 @@ class JSStructuredCloneDecoder(utils.FromDecoderMixin):
|
|
|
584
605
|
while pos < len(raw_data):
|
|
585
606
|
is_uncompressed = raw_data[pos]
|
|
586
607
|
block_size = int.from_bytes(
|
|
587
|
-
raw_data[pos + 1:pos + 4], byteorder="little", signed=False
|
|
588
|
-
|
|
589
|
-
|
|
608
|
+
raw_data[pos + 1 : pos + 4], byteorder="little", signed=False
|
|
609
|
+
)
|
|
610
|
+
masked_checksum = int.from_bytes( # pylint: disable=unused-variable
|
|
611
|
+
raw_data[pos + 4 : pos + 9], byteorder="little", signed=False
|
|
612
|
+
)
|
|
590
613
|
if is_uncompressed:
|
|
591
|
-
uncompressed_data += raw_data[pos + 8: pos + 8 + block_size - 4]
|
|
614
|
+
uncompressed_data += raw_data[pos + 8 : pos + 8 + block_size - 4]
|
|
592
615
|
else:
|
|
593
616
|
uncompressed_data += snappy.decompress(
|
|
594
|
-
raw_data[pos + 8: pos + 8 + block_size - 4]
|
|
617
|
+
raw_data[pos + 8 : pos + 8 + block_size - 4]
|
|
618
|
+
)
|
|
595
619
|
pos += block_size + 4
|
|
596
620
|
else:
|
|
597
621
|
uncompressed_data = snappy.decompress(raw_data)
|
|
@@ -13,10 +13,11 @@
|
|
|
13
13
|
# See the License for the specific language governing permissions and
|
|
14
14
|
# limitations under the License.
|
|
15
15
|
"""Firefox IndexedDB records."""
|
|
16
|
-
|
|
16
|
+
import pathlib
|
|
17
17
|
import sqlite3
|
|
18
18
|
import sys
|
|
19
19
|
import traceback
|
|
20
|
+
from dataclasses import dataclass
|
|
20
21
|
from typing import Any, Generator, Optional
|
|
21
22
|
|
|
22
23
|
from dfindexeddb import errors
|
|
@@ -34,6 +35,7 @@ class FirefoxObjectStoreInfo:
|
|
|
34
35
|
auto_inc: the current auto-increment value.
|
|
35
36
|
database_name: the database name from the database table.
|
|
36
37
|
"""
|
|
38
|
+
|
|
37
39
|
id: int
|
|
38
40
|
name: str
|
|
39
41
|
key_path: str
|
|
@@ -53,6 +55,7 @@ class FirefoxIndexedDBRecord:
|
|
|
53
55
|
object_store_name: the object store name from the object_store table.
|
|
54
56
|
database_name: the IndexedDB database name from the database table.
|
|
55
57
|
"""
|
|
58
|
+
|
|
56
59
|
key: Any
|
|
57
60
|
value: Any
|
|
58
61
|
file_ids: Optional[str]
|
|
@@ -80,10 +83,11 @@ class FileReader:
|
|
|
80
83
|
"""
|
|
81
84
|
self.filename = filename
|
|
82
85
|
|
|
83
|
-
with sqlite3.connect(f
|
|
86
|
+
with sqlite3.connect(f"file:{self.filename}?mode=ro", uri=True) as conn:
|
|
84
87
|
cursor = conn.execute(
|
|
85
|
-
|
|
86
|
-
|
|
88
|
+
"SELECT name, origin, version, last_vacuum_time, last_analyze_time "
|
|
89
|
+
"FROM database"
|
|
90
|
+
)
|
|
87
91
|
result = cursor.fetchone()
|
|
88
92
|
self.database_name = result[0]
|
|
89
93
|
self.origin = result[1]
|
|
@@ -96,7 +100,7 @@ class FileReader:
|
|
|
96
100
|
try:
|
|
97
101
|
return gecko.IDBKey.FromBytes(key)
|
|
98
102
|
except errors.ParserError as e:
|
|
99
|
-
print(
|
|
103
|
+
print("failed to parse", key, file=sys.stderr)
|
|
100
104
|
traceback.print_exception(type(e), e, e.__traceback__)
|
|
101
105
|
return key
|
|
102
106
|
|
|
@@ -105,7 +109,7 @@ class FileReader:
|
|
|
105
109
|
try:
|
|
106
110
|
return gecko.JSStructuredCloneDecoder.FromBytes(value)
|
|
107
111
|
except errors.ParserError as err:
|
|
108
|
-
print(
|
|
112
|
+
print("failed to parse", value, file=sys.stderr)
|
|
109
113
|
traceback.print_exception(type(err), err, err.__traceback__)
|
|
110
114
|
return value
|
|
111
115
|
|
|
@@ -115,9 +119,10 @@ class FileReader:
|
|
|
115
119
|
Yields:
|
|
116
120
|
FirefoxObjectStoreInfo instances.
|
|
117
121
|
"""
|
|
118
|
-
with sqlite3.connect(f
|
|
122
|
+
with sqlite3.connect(f"file:{self.filename}?mode=ro", uri=True) as conn:
|
|
119
123
|
cursor = conn.execute(
|
|
120
|
-
|
|
124
|
+
"SELECT id, auto_increment, name, key_path FROM object_store"
|
|
125
|
+
)
|
|
121
126
|
results = cursor.fetchall()
|
|
122
127
|
for result in results:
|
|
123
128
|
yield FirefoxObjectStoreInfo(
|
|
@@ -125,24 +130,26 @@ class FileReader:
|
|
|
125
130
|
name=result[2],
|
|
126
131
|
key_path=result[3],
|
|
127
132
|
auto_inc=result[1],
|
|
128
|
-
database_name=self.database_name
|
|
133
|
+
database_name=self.database_name,
|
|
134
|
+
)
|
|
129
135
|
|
|
130
136
|
def RecordsByObjectStoreId(
|
|
131
|
-
self,
|
|
132
|
-
object_store_id: int
|
|
137
|
+
self, object_store_id: int
|
|
133
138
|
) -> Generator[FirefoxIndexedDBRecord, None, None]:
|
|
134
139
|
"""Returns FirefoxIndexedDBRecords by a given object store id.
|
|
135
140
|
|
|
136
141
|
Args:
|
|
137
142
|
object_store_id: the object store id.
|
|
138
143
|
"""
|
|
139
|
-
with sqlite3.connect(f
|
|
144
|
+
with sqlite3.connect(f"file:{self.filename}?mode=ro", uri=True) as conn:
|
|
140
145
|
conn.text_factory = bytes
|
|
141
146
|
cursor = conn.execute(
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
147
|
+
"SELECT od.key, od.data, od.object_store_id, od.file_ids, os.name "
|
|
148
|
+
"FROM object_data od "
|
|
149
|
+
"JOIN object_store os ON od.object_store_id == os.id "
|
|
150
|
+
"WHERE os.id = ? ORDER BY od.key",
|
|
151
|
+
(object_store_id,),
|
|
152
|
+
)
|
|
146
153
|
for row in cursor:
|
|
147
154
|
key = self._ParseKey(row[0])
|
|
148
155
|
if row[3]:
|
|
@@ -154,17 +161,19 @@ class FileReader:
|
|
|
154
161
|
value=value,
|
|
155
162
|
object_store_id=row[2],
|
|
156
163
|
file_ids=row[3],
|
|
157
|
-
object_store_name=row[4].decode(
|
|
158
|
-
database_name=self.database_name
|
|
164
|
+
object_store_name=row[4].decode("utf-8"),
|
|
165
|
+
database_name=self.database_name,
|
|
166
|
+
)
|
|
159
167
|
|
|
160
168
|
def Records(self) -> Generator[FirefoxIndexedDBRecord, None, None]:
|
|
161
169
|
"""Returns FirefoxIndexedDBRecords from the database."""
|
|
162
|
-
with sqlite3.connect(f
|
|
170
|
+
with sqlite3.connect(f"file:{self.filename}?mode=ro", uri=True) as conn:
|
|
163
171
|
conn.text_factory = bytes
|
|
164
172
|
cursor = conn.execute(
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
173
|
+
"SELECT od.key, od.data, od.object_store_id, od.file_ids, os.name "
|
|
174
|
+
"FROM object_data od "
|
|
175
|
+
"JOIN object_store os ON od.object_store_id == os.id"
|
|
176
|
+
)
|
|
168
177
|
for row in cursor:
|
|
169
178
|
key = self._ParseKey(row[0])
|
|
170
179
|
if row[3]:
|
|
@@ -176,5 +185,38 @@ class FileReader:
|
|
|
176
185
|
value=value,
|
|
177
186
|
object_store_id=row[2],
|
|
178
187
|
file_ids=row[3],
|
|
179
|
-
object_store_name=row[4].decode(
|
|
180
|
-
database_name=self.database_name
|
|
188
|
+
object_store_name=row[4].decode("utf-8"),
|
|
189
|
+
database_name=self.database_name,
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
class FolderReader:
|
|
194
|
+
"""A reader for a FireFox IndexedDB folder.
|
|
195
|
+
|
|
196
|
+
The path takes a general form of ./<origin>/idb/<filename>.[files|sqlite]
|
|
197
|
+
|
|
198
|
+
"""
|
|
199
|
+
|
|
200
|
+
def __init__(self, folder_name: pathlib.Path):
|
|
201
|
+
"""Initializes the FireFox IndexedDB FolderReader.
|
|
202
|
+
|
|
203
|
+
Args:
|
|
204
|
+
folder_name: the IndexedDB folder name (the origin folder).
|
|
205
|
+
|
|
206
|
+
Raises:
|
|
207
|
+
ValueError: if the folder does not exist or is not a directory.
|
|
208
|
+
"""
|
|
209
|
+
self.folder_name = folder_name
|
|
210
|
+
if not self.folder_name.exists():
|
|
211
|
+
raise ValueError(f"{folder_name} does not exist.")
|
|
212
|
+
if not self.folder_name.is_dir():
|
|
213
|
+
raise ValueError(f"{folder_name} is not a directory.")
|
|
214
|
+
|
|
215
|
+
self.file_names: list[pathlib.Path] = []
|
|
216
|
+
for file_name in self.folder_name.rglob("idb/*.sqlite"):
|
|
217
|
+
self.file_names.append(file_name)
|
|
218
|
+
|
|
219
|
+
def Records(self) -> Generator[FirefoxIndexedDBRecord, None, None]:
|
|
220
|
+
"""Returns FirefoxIndexedDBRecords from the IndexedDB folder."""
|
|
221
|
+
for file_name in self.file_names:
|
|
222
|
+
yield from FileReader(str(file_name)).Records()
|
|
@@ -15,20 +15,20 @@
|
|
|
15
15
|
"""Definitions for Webkit/Safari."""
|
|
16
16
|
from enum import IntEnum
|
|
17
17
|
|
|
18
|
+
CURRENT_VERSION = 0x0000000F # 15
|
|
19
|
+
TERMINATOR_TAG = 0xFFFFFFFF
|
|
20
|
+
STRING_POOL_TAG = 0xFFFFFFFE
|
|
21
|
+
NON_INDEX_PROPERTIES_TAG = 0xFFFFFFFD
|
|
22
|
+
IMAGE_DATA_POOL_TAG = 0xFFFFFFFE
|
|
23
|
+
STRING_DATA_IS_8BIT_FLAG = 0x80000000
|
|
18
24
|
|
|
19
|
-
CurrentVersion = 0x0000000F # 15
|
|
20
|
-
TerminatorTag = 0xFFFFFFFF
|
|
21
|
-
StringPoolTag = 0xFFFFFFFE
|
|
22
|
-
NonIndexPropertiesTag = 0xFFFFFFFD
|
|
23
|
-
ImageDataPoolTag = 0xFFFFFFFE
|
|
24
|
-
StringDataIs8BitFlag = 0x80000000
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
SIDBKeyVersion = 0x00
|
|
26
|
+
SIDB_KEY_VERSION = 0x00
|
|
28
27
|
|
|
29
28
|
|
|
30
29
|
class SIDBKeyType(IntEnum):
|
|
31
30
|
"""SIDBKeyType."""
|
|
31
|
+
|
|
32
32
|
MIN = 0x00
|
|
33
33
|
NUMBER = 0x20
|
|
34
34
|
DATE = 0x40
|
|
@@ -39,10 +39,11 @@ class SIDBKeyType(IntEnum):
|
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
class SerializationTag(IntEnum):
|
|
42
|
-
"""Database Metadata key types.
|
|
43
|
-
|
|
42
|
+
"""Database Metadata key types.
|
|
43
|
+
|
|
44
44
|
All tags are recorded as a single uint8_t.
|
|
45
45
|
"""
|
|
46
|
+
|
|
46
47
|
ARRAY = 1
|
|
47
48
|
OBJECT = 2
|
|
48
49
|
UNDEFINED = 3
|
|
@@ -109,6 +110,7 @@ class SerializationTag(IntEnum):
|
|
|
109
110
|
|
|
110
111
|
class ArrayBufferViewSubtag(IntEnum):
|
|
111
112
|
"""ArrayBufferView sub tags."""
|
|
113
|
+
|
|
112
114
|
DATA_VIEW = 0
|
|
113
115
|
INT8_ARRAY = 1
|
|
114
116
|
UINT8_ARRAY = 2
|