dfindexeddb 20240519__tar.gz → 20241031__tar.gz

{dfindexeddb-20240519/dfindexeddb.egg-info → dfindexeddb-20241031}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dfindexeddb
-Version: 20240519
+Version: 20241031
 Summary: dfindexeddb is an experimental Python tool for performing digital forensic analysis of IndexedDB and leveldb files.
 Author-email: Syd Pleno <sydp@google.com>
 Maintainer-email: dfIndexeddb Developers <dfindexeddb-dev@googlegroups.com>
@@ -230,8 +230,7 @@ analysis of IndexedDB and LevelDB files.
 
 It parses LevelDB, IndexedDB and JavaScript structures from these files without
 requiring native libraries.  (Note: only a subset of IndexedDB key types and
-JavaScript types for Safari and Chromium-based browsers are currently supported.
-Firefox is under development).
+JavaScript types for Firefox, Safari and Chromium-based browsers are currently supported).
 
 The content of IndexedDB files is dependent on what a web application stores
 locally/offline using the web browser's
@@ -313,6 +312,13 @@ options:
 
 #### Examples:
 
+To parse IndexedDB records from an sqlite file for Firefox and output the
+results as JSON, use the following command:
+
+```
+dfindexeddb db -s SOURCE --format firefox -o json
+```
+
 To parse IndexedDB records from an sqlite file for Safari and output the
 results as JSON-L, use the following command:
 
@@ -376,7 +382,7 @@ To parse records from a LevelDB folder, use the following command:
 dfleveldb db -s SOURCE
 ```
 
-To parse records from a LevelDB folder, and use the sequence number to 
+To parse records from a LevelDB folder, and use the sequence number to
 determine recovered records and output as JSON, use the
 following command:
 
@@ -409,8 +415,8 @@ $ dfleveldb descriptor -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_rec
 
 #### Plugins
 
-To apply a plugin parser for a leveldb file/folder, add the 
-`--plugin [Plugin Name]` argument.  Currently, there is support for the 
+To apply a plugin parser for a leveldb file/folder, add the
+`--plugin [Plugin Name]` argument.  Currently, there is support for the
 following artifacts:
 
 | Plugin Name | Artifact Name |

{dfindexeddb-20240519 → dfindexeddb-20241031}/README.md

@@ -5,8 +5,7 @@ analysis of IndexedDB and LevelDB files.
 
 It parses LevelDB, IndexedDB and JavaScript structures from these files without
 requiring native libraries.  (Note: only a subset of IndexedDB key types and
-JavaScript types for Safari and Chromium-based browsers are currently supported.
-Firefox is under development).
+JavaScript types for Firefox, Safari and Chromium-based browsers are currently supported).
 
 The content of IndexedDB files is dependent on what a web application stores
 locally/offline using the web browser's
@@ -88,6 +87,13 @@ options:
 
 #### Examples:
 
+To parse IndexedDB records from an sqlite file for Firefox and output the
+results as JSON, use the following command:
+
+```
+dfindexeddb db -s SOURCE --format firefox -o json
+```
+
 To parse IndexedDB records from an sqlite file for Safari and output the
 results as JSON-L, use the following command:
 
@@ -151,7 +157,7 @@ To parse records from a LevelDB folder, use the following command:
 dfleveldb db -s SOURCE
 ```
 
-To parse records from a LevelDB folder, and use the sequence number to 
+To parse records from a LevelDB folder, and use the sequence number to
 determine recovered records and output as JSON, use the
 following command:
 
@@ -184,8 +190,8 @@ $ dfleveldb descriptor -s SOURCE [-o {json,jsonl,repr}] [-t {blocks,physical_rec
 
 #### Plugins
 
-To apply a plugin parser for a leveldb file/folder, add the 
-`--plugin [Plugin Name]` argument.  Currently, there is support for the 
+To apply a plugin parser for a leveldb file/folder, add the
+`--plugin [Plugin Name]` argument.  Currently, there is support for the
 following artifacts:
 
 | Plugin Name | Artifact Name |

{dfindexeddb-20240519 → dfindexeddb-20241031}/dfindexeddb/indexeddb/chromium/v8.py

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Parsers for v8 javascript serialized objects."""
+from __future__ import annotations
+
 from dataclasses import dataclass
 from datetime import datetime
 import io
@@ -25,8 +27,8 @@ from dfindexeddb.indexeddb.chromium import definitions
 
 
 @dataclass
-class BufferArrayView:
-  """A parsed Javascript BufferArrayView."""
+class ArrayBufferView:
+  """A parsed Javascript ArrayBufferView."""
   buffer: bytes
   tag: definitions.V8ArrayBufferViewTag
   offset: int
@@ -34,15 +36,23 @@ class BufferArrayView:
   flags: int
 
 
-class JSArray(list):
+@dataclass
+class JSArray:
   """A parsed Javascript array.
 
-  This is a wrapper around a standard Python list to allow assigning arbitrary
-  properties as is possible in the Javascript equivalent.
+  A Javascript array behaves like a Python list but allows assigning arbitrary
+  properties.  The array is stored in the attribute __array__.
   """
+  def __init__(self):
+    self.__array__ = []
+
+  def Append(self, element: Any):
+    """Appends a new element to the array."""
+    self.__array__.append(element)
 
   def __repr__(self):
-    array_entries = ", ".join([str(entry) for entry in list(self)])
+    array_entries = ", ".join(
+        [str(entry) for entry in list(self.__array__)])
     properties = ", ".join(
         f'{key}: {value}' for key, value in self.properties.items())
     return f'[{array_entries}, {properties}]'
@@ -52,6 +62,11 @@ class JSArray(list):
     """Returns the object properties."""
     return self.__dict__
 
+  def __eq__(self, other: JSArray):
+    return (
+        self.__array__ == other.__array__
+        and self.properties == other.properties)
+
   def __contains__(self, item):
     return item in self.__dict__
 
@@ -194,6 +209,7 @@ class ValueDeserializer:
     if tag and tag == definitions.V8SerializationTag.ARRAY_BUFFER_VIEW:
       self._ConsumeTag(tag)
       result = self._ReadJSArrayBufferView(result)
+      self.objects[self._GetNextId()] = result
     return result
 
   def _ReadObjectInternal(self) -> Tuple[definitions.V8SerializationTag, Any]:
@@ -385,7 +401,10 @@ class ValueDeserializer:
     while self._PeekTag() != end_tag:
       key = self._ReadObject()
       value = self._ReadObject()
-      js_object[key] = value
+      if isinstance(js_object, dict):
+        js_object[key] = value
+      else:
+        js_object.properties[key] = value
       num_properties += 1
     self._ConsumeTag(end_tag)
     return num_properties
@@ -407,7 +426,7 @@ class ValueDeserializer:
     js_array = JSArray()
     _, length = self.decoder.DecodeUint32Varint()
     for _ in range(length):
-      js_array.append(Undefined())
+      js_array.Append(Undefined())
 
     num_properties = self._ReadJSObjectProperties(
         js_array.__dict__, definitions.V8SerializationTag.END_SPARSE_JS_ARRAY)
@@ -440,7 +459,7 @@ class ValueDeserializer:
 
       if self.version < 11 and isinstance(array_object, Undefined):
         continue
-      js_array.append(array_object)
+      js_array.Append(array_object)
 
     num_properties = self._ReadJSObjectProperties(
         js_array.__dict__, definitions.V8SerializationTag.END_DENSE_JS_ARRAY)
@@ -571,6 +590,7 @@ class ValueDeserializer:
     if is_resizable:
       _, max_byte_length = self.decoder.DecodeUint32Varint()
       if byte_length > max_byte_length:
+        self.objects[next_id] = array_buffer
         return array_buffer
     if byte_length:
       _, array_buffer = self.decoder.ReadBytes(byte_length)
@@ -589,7 +609,7 @@ class ValueDeserializer:
     else:
       flags = 0
 
-    return BufferArrayView(
+    return ArrayBufferView(
         buffer=buffer,
         tag=definitions.V8ArrayBufferViewTag(tag[0]),
         offset=byte_offset,

{dfindexeddb-20240519 → dfindexeddb-20241031}/dfindexeddb/indexeddb/cli.py

@@ -25,6 +25,8 @@ from dfindexeddb import version
 from dfindexeddb.indexeddb.chromium import blink
 from dfindexeddb.indexeddb.chromium import record as chromium_record
 from dfindexeddb.indexeddb.chromium import v8
+from dfindexeddb.indexeddb.firefox import gecko
+from dfindexeddb.indexeddb.firefox import record as firefox_record
 from dfindexeddb.indexeddb.safari import record as safari_record
 
 
@@ -39,7 +41,7 @@ class Encoder(json.JSONEncoder):
     if dataclasses.is_dataclass(o):
       o_dict = utils.asdict(o)
       return o_dict
-    if isinstance(o, bytes):
+    if isinstance(o, (bytes, bytearray)):
       out = []
       for x in o:
         if chr(x) not in _VALID_PRINTABLE_CHARACTERS:
@@ -51,6 +53,8 @@ class Encoder(json.JSONEncoder):
       return o.isoformat()
     if isinstance(o, v8.Undefined):
       return "<undefined>"
+    if isinstance(o, v8.JSArray):
+      return o.__dict__
     if isinstance(o, v8.Null):
       return "<null>"
     if isinstance(o, set):
@@ -73,13 +77,21 @@ def _Output(structure, output):
 
 
 def BlinkCommand(args):
-  """The CLI for processing a file as a blink value."""
+  """The CLI for processing a file as a blink-encoded value."""
   with open(args.source, 'rb') as fd:
     buffer = fd.read()
     blink_value = blink.V8ScriptValueDecoder.FromBytes(buffer)
     _Output(blink_value, output=args.output)
 
 
+def GeckoCommand(args):
+  """The CLI for processing a file as a gecko-encoded value."""
+  with open(args.source, 'rb') as fd:
+    buffer = fd.read()
+    blink_value = gecko.JSStructuredCloneDecoder.FromBytes(buffer)
+    _Output(blink_value, output=args.output)
+
+
 def DbCommand(args):
   """The CLI for processing a directory as IndexedDB."""
   if args.format in ('chrome', 'chromium'):
@@ -88,6 +100,9 @@ def DbCommand(args):
             use_manifest=args.use_manifest,
             use_sequence_number=args.use_sequence_number):
       _Output(db_record, output=args.output)
+  elif args.format == 'firefox':
+    for db_record in firefox_record.FileReader(args.source).Records():
+      _Output(db_record, output=args.output)
   elif args.format == 'safari':
     for db_record in safari_record.FileReader(args.source).Records():
       _Output(db_record, output=args.output)
@@ -133,6 +148,24 @@ def App():
       help='Output format.  Default is json')
   parser_blink.set_defaults(func=BlinkCommand)
 
+  parser_gecko = subparsers.add_parser(
+      'gecko', help='Parse a file as a gecko-encoded value.')
+  parser_gecko.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source file.')
+  parser_gecko.add_argument(
+      '-o',
+      '--output',
+      choices=[
+          'json',
+          'jsonl',
+          'repr'],
+      default='json',
+      help='Output format.  Default is json')
+  parser_gecko.set_defaults(func=GeckoCommand)
+
   parser_db = subparsers.add_parser(
       'db', help='Parse a directory as IndexedDB.')
   parser_db.add_argument(
@@ -156,7 +189,7 @@ def App():
   parser_db.add_argument(
       '--format',
       required=True,
-      choices=['chromium', 'chrome', 'safari'],
+      choices=['chromium', 'chrome', 'firefox', 'safari'],
       help='The type of IndexedDB to parse.')
   parser_db.add_argument(
       '-o',

dfindexeddb-20241031/dfindexeddb/indexeddb/firefox/definitions.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Definitions for Firefox IndexedDB."""
+from enum import IntEnum
+
+
+class IndexedDBKeyType(IntEnum):
+  """IndexedDB Key Types."""
+  TERMINATOR = 0
+  FLOAT = 0x10
+  DATE = 0x20
+  STRING = 0x30
+  BINARY = 0x40
+  ARRAY = 0x50
+
+
+MAX_ARRAY_COLLAPSE = 3
+MAX_RECURSION_DEPTH = 64
+MAX_LENGTH = (1 << 30) - 2
+ONE_BYTE_LIMIT = 0x7E
+TWO_BYTE_LIMIT = 0x3FFF + 0x7F
+ONE_BYTE_ADJUST = 1
+TWO_BYTE_ADJUST = -0x7F
+THREE_BYTE_SHIFT = 6
+
+
+class StructuredDataType(IntEnum):
+  """Structured Data Types."""
+  FLOAT_MAX = 0xFFF00000
+  HEADER = 0xFFF10000
+  NULL = 0xFFFF0000
+  UNDEFINED = 0xFFFF0001
+  BOOLEAN = 0xFFFF0002
+  INT32 = 0xFFFF0003
+  STRING = 0xFFFF0004
+  DATE_OBJECT  = 0xFFFF0005
+  REGEXP_OBJECT = 0xFFFF0006
+  ARRAY_OBJECT = 0xFFFF0007
+  OBJECT_OBJECT = 0xFFFF0008
+  ARRAY_BUFFER_OBJECT_V2 = 0xFFFF0009
+  BOOLEAN_OBJECT = 0xFFFF000A
+  STRING_OBJECT = 0xFFFF000B
+  NUMBER_OBJECT = 0xFFFF000C
+  BACK_REFERENCE_OBJECT  = 0xFFFF000D
+  DO_NOT_USE_1  = 0xFFFF000E
+  DO_NOT_USE_2 = 0xFFFF000F
+  TYPED_ARRAY_OBJECT_V2 = 0xFFFF0010
+  MAP_OBJECT = 0xFFFF0011
+  SET_OBJECT = 0xFFFF0012
+  END_OF_KEYS = 0xFFFF0013
+  DO_NOT_USE_3 = 0xFFFF0014
+  DATA_VIEW_OBJECT_V2 = 0xFFFF0015
+  SAVED_FRAME_OBJECT = 0xFFFF0016
+  JSPRINCIPALS = 0xFFFF0017
+  NULL_JSPRINCIPALS = 0xFFFF0018
+  RECONSTRUCTED_SAVED_FRAME_PRINCIPALS_IS_SYSTEM = 0xFFFF0019
+  RECONSTRUCTED_SAVED_FRAME_PRINCIPALS_IS_NOT_SYSTEM = 0xFFFF001A
+  SHARED_ARRAY_BUFFER_OBJECT = 0xFFFF001B
+  SHARED_WASM_MEMORY_OBJECT = 0xFFFF001C
+  BIGINT = 0xFFFF001D
+  BIGINT_OBJECT = 0xFFFF001E
+  ARRAY_BUFFER_OBJECT = 0xFFFF001F
+  TYPED_ARRAY_OBJECT = 0xFFFF0020
+  DATA_VIEW_OBJECT = 0xFFFF0021
+  ERROR_OBJECT = 0xFFFF0022
+  RESIZABLE_ARRAY_BUFFER_OBJECT = 0xFFFF0023
+  GROWABLE_SHARED_ARRAY_BUFFER_OBJECT = 0xFFFF0024
+  TYPED_ARRAY_V1_INT8 = 0xFFFF0100
+  TYPED_ARRAY_V1_UINT8 = 0xFFFF0101
+  TYPED_ARRAY_V1_INT16 = 0xFFFF0102
+  TYPED_ARRAY_V1_UINT16 = 0xFFFF0103
+  TYPED_ARRAY_V1_INT32 = 0xFFFF0104
+  TYPED_ARRAY_V1_UINT32 = 0xFFFF0105
+  TYPED_ARRAY_V1_FLOAT32 = 0xFFFF0106
+  TYPED_ARRAY_V1_FLOAT64 = 0xFFFF0107
+  TYPED_ARRAY_V1_UINT8_CLAMPED = 0xFFFF0108
+  TRANSFER_MAP_HEADER = 0xFFFF0200
+  TRANSFER_MAP_PENDING_ENTRY = 0xFFFF0201
+  TRANSFER_MAP_ARRAY_BUFFER = 0xFFFF0202
+  TRANSFER_MAP_STORED_ARRAY_BUFFER = 0xFFFF0203
+  TRANSFER_MAP_END_OF_BUILTIN_TYPES = 0xFFFF0204
+
+
+class StructuredCloneTags(IntEnum):
+  """Structured Clone Tags."""
+  BLOB = 0xFFFF8001
+  FILE_WITHOUT_LASTMODIFIEDDATE = 0xFFFF8002
+  FILELIST = 0xFFFF8003
+  MUTABLEFILE = 0xFFFF8004
+  FILE = 0xFFFF8005
+  WASM_MODULE = 0xFFFF8006
+  IMAGEDATA = 0xFFFF8007
+  DOMPOINT = 0xFFFF8008
+  DOMPOINTREADONLY = 0xFFFF8009
+  CRYPTOKEY = 0xFFFF800A
+  NULL_PRINCIPAL = 0xFFFF800B
+  SYSTEM_PRINCIPAL = 0xFFFF800C
+  CONTENT_PRINCIPAL = 0xFFFF800D
+  DOMQUAD = 0xFFFF800E
+  RTCCERTIFICATE = 0xFFFF800F
+  DOMRECT = 0xFFFF8010
+  DOMRECTREADONLY = 0xFFFF8011
+  EXPANDED_PRINCIPAL = 0xFFFF8012
+  DOMMATRIX = 0xFFFF8013
+  URLSEARCHPARAMS = 0xFFFF8014
+  DOMMATRIXREADONLY = 0xFFFF8015
+  DOMEXCEPTION = 0xFFFF80016
+  EMPTY_SLOT_9 = 0xFFFF8017
+  STRUCTUREDCLONETESTER = 0xFFFF8018
+  FILESYSTEMHANDLE = 0xFFFF8019
+  FILESYSTEMFILEHANDLE = 0xFFFF801A
+  FILESYSTEMDIRECTORYHANDLE = 0xFFFF801B
+  IMAGEBITMAP = 0xFFFF801C
+  MAP_MESSAGEPORT = 0xFFFF801D
+  FORMDATA = 0xFFFF801E
+  CANVAS = 0xFFFF801F  # This tag is for OffscreenCanvas.
+  DIRECTORY = 0xFFFF8020
+  INPUTSTREAM = 0xFFFF8021
+  STRUCTURED_CLONE_HOLDER = 0xFFFF8022
+  BROWSING_CONTEXT = 0xFFFF8023
+  CLONED_ERROR_OBJECT = 0xFFFF8024
+  READABLESTREAM = 0xFFFF8025
+  WRITABLESTREAM = 0xFFFF8026
+  TRANSFORMSTREAM = 0xFFFF8027
+  VIDEOFRAME = 0xFFFF8028
+  ENCODEDVIDEOCHUNK = 0xFFFF8029
+  AUDIODATA = 0xFFFF8030
+  ENCODEDAUDIOCHUNK = 0xFFFF8031
+
+
+FRAME_HEADER = b'\xff\x06\x00\x00sNaPpY'