dfindexeddb 20240519__py3-none-any.whl → 20241105__py3-none-any.whl

dfindexeddb/indexeddb/chromium/v8.py

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Parsers for v8 javascript serialized objects."""
+from __future__ import annotations
+
 from dataclasses import dataclass
 from datetime import datetime
 import io
@@ -21,12 +23,13 @@ from typing import Any, BinaryIO, Dict, Optional, Set, Tuple, Union
 
 from dfindexeddb import errors
 from dfindexeddb import utils
+from dfindexeddb.indexeddb import types
 from dfindexeddb.indexeddb.chromium import definitions
 
 
 @dataclass
-class BufferArrayView:
-  """A parsed Javascript BufferArrayView."""
+class ArrayBufferView:
+  """A parsed Javascript ArrayBufferView."""
   buffer: bytes
   tag: definitions.V8ArrayBufferViewTag
   offset: int
@@ -34,48 +37,6 @@ class BufferArrayView:
   flags: int
 
 
-class JSArray(list):
-  """A parsed Javascript array.
-
-  This is a wrapper around a standard Python list to allow assigning arbitrary
-  properties as is possible in the Javascript equivalent.
-  """
-
-  def __repr__(self):
-    array_entries = ", ".join([str(entry) for entry in list(self)])
-    properties = ", ".join(
-        f'{key}: {value}' for key, value in self.properties.items())
-    return f'[{array_entries}, {properties}]'
-
-  @property
-  def properties(self) -> Dict[str, Any]:
-    """Returns the object properties."""
-    return self.__dict__
-
-  def __contains__(self, item):
-    return item in self.__dict__
-
-  def __getitem__(self, name):
-    return self.__dict__[name]
-
-
-@dataclass
-class Null:
-  """A parsed Javascript Null."""
-
-
-@dataclass
-class RegExp:
-  """A parsed Javascript RegExp."""
-  pattern: str
-  flags: int
-
-
-@dataclass(frozen=True)
-class Undefined:
-  """A parsed Javascript undef."""
-
-
 class ValueDeserializer:
   """A class to deserialize v8 Javascript values/objects.
 
@@ -194,6 +155,7 @@ class ValueDeserializer:
     if tag and tag == definitions.V8SerializationTag.ARRAY_BUFFER_VIEW:
       self._ConsumeTag(tag)
       result = self._ReadJSArrayBufferView(result)
+      self.objects[self._GetNextId()] = result
     return result
 
   def _ReadObjectInternal(self) -> Tuple[definitions.V8SerializationTag, Any]:
@@ -207,9 +169,9 @@ class ValueDeserializer:
       _ = self.decoder.DecodeUint32Varint()
       parsed_object = self._ReadObject()
     elif tag == definitions.V8SerializationTag.UNDEFINED:
-      parsed_object = Undefined()
+      parsed_object = types.Undefined()
     elif tag == definitions.V8SerializationTag.NULL:
-      parsed_object = Null()
+      parsed_object = types.Null()
     elif tag == definitions.V8SerializationTag.TRUE:
       parsed_object = True
     elif tag == definitions.V8SerializationTag.FALSE:
@@ -369,7 +331,7 @@ class ValueDeserializer:
 
   def _ReadJSObjectProperties(
       self,
-      js_object: Union[Dict, JSArray],
+      js_object: Union[Dict, types.JSArray],
       end_tag: definitions.V8SerializationTag
   ) -> int:
     """Reads key-value properties and sets them to the given js_object.
@@ -385,7 +347,10 @@ class ValueDeserializer:
     while self._PeekTag() != end_tag:
       key = self._ReadObject()
       value = self._ReadObject()
-      js_object[key] = value
+      if isinstance(js_object, dict):
+        js_object[key] = value
+      else:
+        js_object.properties[key] = value
       num_properties += 1
     self._ConsumeTag(end_tag)
     return num_properties
@@ -396,7 +361,7 @@ class ValueDeserializer:
     self.next_id += 1
     return next_id
 
-  def ReadSparseJSArray(self) -> JSArray:
+  def ReadSparseJSArray(self) -> types.JSArray:
     """Reads a sparse encoded JSArray from the current position.
 
     Raises:
@@ -404,13 +369,13 @@ class ValueDeserializer:
     """
     next_id = self._GetNextId()
 
-    js_array = JSArray()
+    js_array = types.JSArray()
     _, length = self.decoder.DecodeUint32Varint()
     for _ in range(length):
-      js_array.append(Undefined())
+      js_array.values.append(types.Undefined())
 
     num_properties = self._ReadJSObjectProperties(
-        js_array.__dict__, definitions.V8SerializationTag.END_SPARSE_JS_ARRAY)
+        js_array.properties, definitions.V8SerializationTag.END_SPARSE_JS_ARRAY)
     _, expected_num_properties = self.decoder.DecodeUint32Varint()
     _, expected_length = self.decoder.DecodeUint32Varint()
 
@@ -421,7 +386,7 @@ class ValueDeserializer:
     self.objects[next_id] = js_array
     return js_array
 
-  def ReadDenseJSArray(self) -> JSArray:
+  def ReadDenseJSArray(self) -> types.JSArray:
     """Reads a dense encoded JSArray from the current position.
 
     Raises:
@@ -429,7 +394,7 @@ class ValueDeserializer:
     """
     next_id = self._GetNextId()
 
-    js_array = JSArray()
+    js_array = types.JSArray()
     _, length = self.decoder.DecodeUint32Varint()
     for _ in range(length):
       tag = self._PeekTag()
@@ -438,12 +403,12 @@ class ValueDeserializer:
         continue
       array_object = self._ReadObject()
 
-      if self.version < 11 and isinstance(array_object, Undefined):
+      if self.version < 11 and isinstance(array_object, types.Undefined):
         continue
-      js_array.append(array_object)
+      js_array.values.append(array_object)
 
     num_properties = self._ReadJSObjectProperties(
-        js_array.__dict__, definitions.V8SerializationTag.END_DENSE_JS_ARRAY)
+        js_array.properties, definitions.V8SerializationTag.END_DENSE_JS_ARRAY)
     _, expected_num_properties = self.decoder.DecodeUint32Varint()
     _, expected_length = self.decoder.DecodeUint32Varint()
     if num_properties != expected_num_properties:
@@ -496,12 +461,12 @@ class ValueDeserializer:
     self.objects[next_id] = value
     return value
 
-  def _ReadJSRegExp(self) -> RegExp:
+  def _ReadJSRegExp(self) -> types.RegExp:
     """Reads a Javascript regular expression from the current position."""
     next_id = self._GetNextId()
     pattern = self.ReadString()
     _, flags = self.decoder.DecodeUint32Varint()  # TODO: verify flags
-    regexp = RegExp(pattern=pattern, flags=flags)
+    regexp = types.RegExp(pattern=pattern, flags=str(flags))
     self.objects[next_id] = regexp
     return regexp
 
@@ -553,7 +518,10 @@ class ValueDeserializer:
     return js_set
 
   def _ReadJSArrayBuffer(
-      self, is_shared: bool, is_resizable: bool) -> bytes:
+      self,
+      is_shared: bool,
+      is_resizable: bool
+  ) -> bytes:
     """Reads a Javascript ArrayBuffer from the current position.
 
     Args:
@@ -571,6 +539,7 @@ class ValueDeserializer:
     if is_resizable:
       _, max_byte_length = self.decoder.DecodeUint32Varint()
       if byte_length > max_byte_length:
+        self.objects[next_id] = array_buffer
         return array_buffer
     if byte_length:
       _, array_buffer = self.decoder.ReadBytes(byte_length)
@@ -589,7 +558,7 @@ class ValueDeserializer:
     else:
       flags = 0
 
-    return BufferArrayView(
+    return ArrayBufferView(
         buffer=buffer,
         tag=definitions.V8ArrayBufferViewTag(tag[0]),
         offset=byte_offset,

dfindexeddb/indexeddb/cli.py

@@ -22,9 +22,11 @@ import pathlib
 
 from dfindexeddb import utils
 from dfindexeddb import version
+from dfindexeddb.indexeddb import types
 from dfindexeddb.indexeddb.chromium import blink
 from dfindexeddb.indexeddb.chromium import record as chromium_record
-from dfindexeddb.indexeddb.chromium import v8
+from dfindexeddb.indexeddb.firefox import gecko
+from dfindexeddb.indexeddb.firefox import record as firefox_record
 from dfindexeddb.indexeddb.safari import record as safari_record
 
 
@@ -39,7 +41,7 @@ class Encoder(json.JSONEncoder):
     if dataclasses.is_dataclass(o):
       o_dict = utils.asdict(o)
       return o_dict
-    if isinstance(o, bytes):
+    if isinstance(o, (bytes, bytearray)):
       out = []
       for x in o:
         if chr(x) not in _VALID_PRINTABLE_CHARACTERS:
@@ -49,13 +51,15 @@ class Encoder(json.JSONEncoder):
       return ''.join(out)
     if isinstance(o, datetime):
       return o.isoformat()
-    if isinstance(o, v8.Undefined):
-      return "<undefined>"
-    if isinstance(o, v8.Null):
-      return "<null>"
+    if isinstance(o, types.Undefined):
+      return '<undefined>'
+    if isinstance(o, types.JSArray):
+      return o.__dict__
+    if isinstance(o, types.Null):
+      return '<null>'
     if isinstance(o, set):
       return list(o)
-    if isinstance(o, v8.RegExp):
+    if isinstance(o, types.RegExp):
       return str(o)
     if isinstance(o, enum.Enum):
       return o.name
@@ -73,13 +77,21 @@ def _Output(structure, output):
 
 
 def BlinkCommand(args):
-  """The CLI for processing a file as a blink value."""
+  """The CLI for processing a file as a blink-encoded value."""
   with open(args.source, 'rb') as fd:
     buffer = fd.read()
     blink_value = blink.V8ScriptValueDecoder.FromBytes(buffer)
     _Output(blink_value, output=args.output)
 
 
+def GeckoCommand(args):
+  """The CLI for processing a file as a gecko-encoded value."""
+  with open(args.source, 'rb') as fd:
+    buffer = fd.read()
+    blink_value = gecko.JSStructuredCloneDecoder.FromBytes(buffer)
+    _Output(blink_value, output=args.output)
+
+
 def DbCommand(args):
   """The CLI for processing a directory as IndexedDB."""
   if args.format in ('chrome', 'chromium'):
@@ -88,6 +100,9 @@ def DbCommand(args):
             use_manifest=args.use_manifest,
             use_sequence_number=args.use_sequence_number):
       _Output(db_record, output=args.output)
+  elif args.format == 'firefox':
+    for db_record in firefox_record.FileReader(args.source).Records():
+      _Output(db_record, output=args.output)
   elif args.format == 'safari':
     for db_record in safari_record.FileReader(args.source).Records():
       _Output(db_record, output=args.output)
@@ -115,13 +130,13 @@ def App():
   subparsers = parser.add_subparsers()
 
   parser_blink = subparsers.add_parser(
-      'blink', help='Parse a file as a blink value.')
+      'blink', help='Parse a file as a blink-encoded value.')
   parser_blink.add_argument(
-      '-s', '--source',
+      '-s',
+      '--source',
       required=True,
       type=pathlib.Path,
-      help=(
-        'The source file.'))
+      help='The source file.')
   parser_blink.add_argument(
       '-o',
       '--output',
@@ -130,18 +145,38 @@ def App():
           'jsonl',
           'repr'],
       default='json',
-      help='Output format.  Default is json')
+      help='Output format.  Default is json.')
   parser_blink.set_defaults(func=BlinkCommand)
 
+  parser_gecko = subparsers.add_parser(
+      'gecko', help='Parse a file as a gecko-encoded value.')
+  parser_gecko.add_argument(
+      '-s',
+      '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source file.')
+  parser_gecko.add_argument(
+      '-o',
+      '--output',
+      choices=[
+          'json',
+          'jsonl',
+          'repr'],
+      default='json',
+      help='Output format.  Default is json.')
+  parser_gecko.set_defaults(func=GeckoCommand)
+
   parser_db = subparsers.add_parser(
-      'db', help='Parse a directory as IndexedDB.')
+      'db', help='Parse a directory/file as IndexedDB.')
   parser_db.add_argument(
-      '-s', '--source',
+      '-s',
+      '--source',
       required=True,
       type=pathlib.Path,
       help=(
-        'The source IndexedDB folder (for chrome/chromium) '
-        'or file (for safari).'))
+          'The source IndexedDB folder (for chrome/chromium) '
+          'or sqlite3 file (for firefox/safari).'))
   recover_group = parser_db.add_mutually_exclusive_group()
   recover_group.add_argument(
       '--use_manifest',
@@ -156,7 +191,11 @@ def App():
   parser_db.add_argument(
       '--format',
       required=True,
-      choices=['chromium', 'chrome', 'safari'],
+      choices=[
+          'chromium',
+          'chrome',
+          'firefox',
+          'safari'],
       help='The type of IndexedDB to parse.')
   parser_db.add_argument(
       '-o',
@@ -166,14 +205,15 @@ def App():
           'jsonl',
           'repr'],
       default='json',
-      help='Output format.  Default is json')
+      help='Output format.  Default is json.')
   parser_db.set_defaults(func=DbCommand)
 
   parser_ldb = subparsers.add_parser(
       'ldb',
       help='Parse a ldb file as IndexedDB.')
   parser_ldb.add_argument(
-      '-s', '--source',
+      '-s',
+      '--source',
       required=True,
       type=pathlib.Path,
       help='The source .ldb file.')
@@ -185,7 +225,7 @@ def App():
           'jsonl',
           'repr'],
       default='json',
-      help='Output format.  Default is json')
+      help='Output format.  Default is json.')
   parser_ldb.set_defaults(func=LdbCommand)
 
   parser_log = subparsers.add_parser(
@@ -204,7 +244,7 @@ def App():
           'jsonl',
           'repr'],
       default='json',
-      help='Output format.  Default is json')
+      help='Output format.  Default is json.')
   parser_log.set_defaults(func=LogCommand)
 
   args = parser.parse_args()

dfindexeddb/indexeddb/firefox/definitions.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Definitions for Firefox IndexedDB."""
+from enum import IntEnum
+
+
+class IndexedDBKeyType(IntEnum):
+  """IndexedDB Key Types."""
+  TERMINATOR = 0
+  FLOAT = 0x10
+  DATE = 0x20
+  STRING = 0x30
+  BINARY = 0x40
+  ARRAY = 0x50
+
+
+MAX_ARRAY_COLLAPSE = 3
+MAX_RECURSION_DEPTH = 64
+MAX_LENGTH = (1 << 30) - 2
+ONE_BYTE_LIMIT = 0x7E
+TWO_BYTE_LIMIT = 0x3FFF + 0x7F
+ONE_BYTE_ADJUST = 1
+TWO_BYTE_ADJUST = -0x7F
+THREE_BYTE_SHIFT = 6
+
+
+class StructuredDataType(IntEnum):
+  """Structured Data Types."""
+  FLOAT_MAX = 0xFFF00000
+  HEADER = 0xFFF10000
+  NULL = 0xFFFF0000
+  UNDEFINED = 0xFFFF0001
+  BOOLEAN = 0xFFFF0002
+  INT32 = 0xFFFF0003
+  STRING = 0xFFFF0004
+  DATE_OBJECT  = 0xFFFF0005
+  REGEXP_OBJECT = 0xFFFF0006
+  ARRAY_OBJECT = 0xFFFF0007
+  OBJECT_OBJECT = 0xFFFF0008
+  ARRAY_BUFFER_OBJECT_V2 = 0xFFFF0009
+  BOOLEAN_OBJECT = 0xFFFF000A
+  STRING_OBJECT = 0xFFFF000B
+  NUMBER_OBJECT = 0xFFFF000C
+  BACK_REFERENCE_OBJECT  = 0xFFFF000D
+  DO_NOT_USE_1  = 0xFFFF000E
+  DO_NOT_USE_2 = 0xFFFF000F
+  TYPED_ARRAY_OBJECT_V2 = 0xFFFF0010
+  MAP_OBJECT = 0xFFFF0011
+  SET_OBJECT = 0xFFFF0012
+  END_OF_KEYS = 0xFFFF0013
+  DO_NOT_USE_3 = 0xFFFF0014
+  DATA_VIEW_OBJECT_V2 = 0xFFFF0015
+  SAVED_FRAME_OBJECT = 0xFFFF0016
+  JSPRINCIPALS = 0xFFFF0017
+  NULL_JSPRINCIPALS = 0xFFFF0018
+  RECONSTRUCTED_SAVED_FRAME_PRINCIPALS_IS_SYSTEM = 0xFFFF0019
+  RECONSTRUCTED_SAVED_FRAME_PRINCIPALS_IS_NOT_SYSTEM = 0xFFFF001A
+  SHARED_ARRAY_BUFFER_OBJECT = 0xFFFF001B
+  SHARED_WASM_MEMORY_OBJECT = 0xFFFF001C
+  BIGINT = 0xFFFF001D
+  BIGINT_OBJECT = 0xFFFF001E
+  ARRAY_BUFFER_OBJECT = 0xFFFF001F
+  TYPED_ARRAY_OBJECT = 0xFFFF0020
+  DATA_VIEW_OBJECT = 0xFFFF0021
+  ERROR_OBJECT = 0xFFFF0022
+  RESIZABLE_ARRAY_BUFFER_OBJECT = 0xFFFF0023
+  GROWABLE_SHARED_ARRAY_BUFFER_OBJECT = 0xFFFF0024
+  TYPED_ARRAY_V1_INT8 = 0xFFFF0100
+  TYPED_ARRAY_V1_UINT8 = 0xFFFF0101
+  TYPED_ARRAY_V1_INT16 = 0xFFFF0102
+  TYPED_ARRAY_V1_UINT16 = 0xFFFF0103
+  TYPED_ARRAY_V1_INT32 = 0xFFFF0104
+  TYPED_ARRAY_V1_UINT32 = 0xFFFF0105
+  TYPED_ARRAY_V1_FLOAT32 = 0xFFFF0106
+  TYPED_ARRAY_V1_FLOAT64 = 0xFFFF0107
+  TYPED_ARRAY_V1_UINT8_CLAMPED = 0xFFFF0108
+  TRANSFER_MAP_HEADER = 0xFFFF0200
+  TRANSFER_MAP_PENDING_ENTRY = 0xFFFF0201
+  TRANSFER_MAP_ARRAY_BUFFER = 0xFFFF0202
+  TRANSFER_MAP_STORED_ARRAY_BUFFER = 0xFFFF0203
+  TRANSFER_MAP_END_OF_BUILTIN_TYPES = 0xFFFF0204
+
+
+class StructuredCloneTags(IntEnum):
+  """Structured Clone Tags."""
+  BLOB = 0xFFFF8001
+  FILE_WITHOUT_LASTMODIFIEDDATE = 0xFFFF8002
+  FILELIST = 0xFFFF8003
+  MUTABLEFILE = 0xFFFF8004
+  FILE = 0xFFFF8005
+  WASM_MODULE = 0xFFFF8006
+  IMAGEDATA = 0xFFFF8007
+  DOMPOINT = 0xFFFF8008
+  DOMPOINTREADONLY = 0xFFFF8009
+  CRYPTOKEY = 0xFFFF800A
+  NULL_PRINCIPAL = 0xFFFF800B
+  SYSTEM_PRINCIPAL = 0xFFFF800C
+  CONTENT_PRINCIPAL = 0xFFFF800D
+  DOMQUAD = 0xFFFF800E
+  RTCCERTIFICATE = 0xFFFF800F
+  DOMRECT = 0xFFFF8010
+  DOMRECTREADONLY = 0xFFFF8011
+  EXPANDED_PRINCIPAL = 0xFFFF8012
+  DOMMATRIX = 0xFFFF8013
+  URLSEARCHPARAMS = 0xFFFF8014
+  DOMMATRIXREADONLY = 0xFFFF8015
+  DOMEXCEPTION = 0xFFFF80016
+  EMPTY_SLOT_9 = 0xFFFF8017
+  STRUCTUREDCLONETESTER = 0xFFFF8018
+  FILESYSTEMHANDLE = 0xFFFF8019
+  FILESYSTEMFILEHANDLE = 0xFFFF801A
+  FILESYSTEMDIRECTORYHANDLE = 0xFFFF801B
+  IMAGEBITMAP = 0xFFFF801C
+  MAP_MESSAGEPORT = 0xFFFF801D
+  FORMDATA = 0xFFFF801E
+  CANVAS = 0xFFFF801F  # This tag is for OffscreenCanvas.
+  DIRECTORY = 0xFFFF8020
+  INPUTSTREAM = 0xFFFF8021
+  STRUCTURED_CLONE_HOLDER = 0xFFFF8022
+  BROWSING_CONTEXT = 0xFFFF8023
+  CLONED_ERROR_OBJECT = 0xFFFF8024
+  READABLESTREAM = 0xFFFF8025
+  WRITABLESTREAM = 0xFFFF8026
+  TRANSFORMSTREAM = 0xFFFF8027
+  VIDEOFRAME = 0xFFFF8028
+  ENCODEDVIDEOCHUNK = 0xFFFF8029
+  AUDIODATA = 0xFFFF8030
+  ENCODEDAUDIOCHUNK = 0xFFFF8031
+
+
+FRAME_HEADER = b'\xff\x06\x00\x00sNaPpY'