dfindexeddb 20240519__py3-none-any.whl → 20241105__py3-none-any.whl

dfindexeddb/indexeddb/firefox/record.py

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Firefox IndexedDB records."""
+from dataclasses import dataclass
+import sqlite3
+import sys
+import traceback
+from typing import Any, Generator, Optional
+
+from dfindexeddb import errors
+from dfindexeddb.indexeddb.firefox import gecko
+
+
+@dataclass
+class FirefoxObjectStoreInfo:
+  """A FireFox ObjectStoreInfo.
+
+  Attributes:
+    id: the object store ID.
+    name: the object store name.
+    key_path: the object store key path.
+    auto_inc: the current auto-increment value.
+    database_name: the database name from the database table.
+  """
+  id: int
+  name: str
+  key_path: str
+  auto_inc: int
+  database_name: str
+
+
+@dataclass
+class FirefoxIndexedDBRecord:
+  """A Firefox IndexedDBRecord.
+
+  Attributes:
+    key: the parsed key.
+    value: the parsed value.
+    file_ids: the file identifiers.
+    object_store_id: the object store id.
+    object_store_name: the object store name from the object_store table.
+    database_name: the IndexedDB database name from the database table.
+  """
+  key: Any
+  value: Any
+  file_ids: Optional[str]
+  object_store_id: int
+  object_store_name: str
+  database_name: str
+
+
+class FileReader:
+  """A reader for Firefox IndexedDB sqlite3 files.
+
+  Attributes:
+    database_name: the database name.
+    origin: the database origin.
+    metadata_version: the metadata version.
+    last_vacuum_time: the last vacuum time.
+    last_analyze_time: the last analyze time.
+  """
+
+  def __init__(self, filename: str):
+    """Initializes the FileReader.
+
+    Args:
+      filename: the IndexedDB filename.
+    """
+    self.filename = filename
+
+    with sqlite3.connect(f'file:{self.filename}?mode=ro', uri=True) as conn:
+      cursor = conn.execute(
+          'SELECT name, origin, version, last_vacuum_time, last_analyze_time '
+          'FROM database')
+      result = cursor.fetchone()
+      self.database_name = result[0]
+      self.origin = result[1]
+      self.metadata_version = result[2]
+      self.last_vacuum_time = result[3]
+      self.last_analyze_time = result[4]
+
+  def _ParseKey(self, key: bytes) -> Any:
+    """Parses a key."""
+    try:
+      return gecko.IDBKey.FromBytes(key)
+    except errors.ParserError as e:
+      print('failed to parse', key, file=sys.stderr)
+      traceback.print_exception(type(e), e, e.__traceback__)
+      return key
+
+  def _ParseValue(self, value: bytes) -> Any:
+    """Parses a value."""
+    try:
+      return gecko.JSStructuredCloneDecoder.FromBytes(value)
+    except errors.ParserError as err:
+      print('failed to parse', value, file=sys.stderr)
+      traceback.print_exception(type(err), err, err.__traceback__)
+      return value
+
+  def ObjectStores(self) -> Generator[FirefoxObjectStoreInfo, None, None]:
+    """Returns the Object Store information from the IndexedDB database.
+
+    Yields:
+      FirefoxObjectStoreInfo instances.
+    """
+    with sqlite3.connect(f'file:{self.filename}?mode=ro', uri=True) as conn:
+      cursor = conn.execute(
+          'SELECT id, auto_increment, name, key_path FROM object_store')
+      results = cursor.fetchall()
+      for result in results:
+        yield FirefoxObjectStoreInfo(
+            id=result[0],
+            name=result[2],
+            key_path=result[3],
+            auto_inc=result[1],
+            database_name=self.database_name)
+
+  def RecordsByObjectStoreId(
+      self,
+      object_store_id: int
+  ) -> Generator[FirefoxIndexedDBRecord, None, None]:
+    """Returns FirefoxIndexedDBRecords by a given object store id.
+
+    Args:
+      object_store_id: the object store id.
+    """
+    with sqlite3.connect(f'file:{self.filename}?mode=ro', uri=True) as conn:
+      conn.text_factory = bytes
+      cursor = conn.execute(
+          'SELECT od.key, od.data, od.object_store_id, od.file_ids, os.name '
+          'FROM object_data od '
+          'JOIN object_store os ON od.object_store_id == os.id '
+          'WHERE os.id = ? ORDER BY od.key', (object_store_id, ))
+      for row in cursor:
+        key = self._ParseKey(row[0])
+        if row[3]:
+          value = row[1]
+        else:
+          value = self._ParseValue(row[1])
+        yield FirefoxIndexedDBRecord(
+            key=key,
+            value=value,
+            object_store_id=row[2],
+            file_ids=row[3],
+            object_store_name=row[4].decode('utf-8'),
+            database_name=self.database_name)
+
+  def Records(self) -> Generator[FirefoxIndexedDBRecord, None, None]:
+    """Returns FirefoxIndexedDBRecords from the database."""
+    with sqlite3.connect(f'file:{self.filename}?mode=ro', uri=True) as conn:
+      conn.text_factory = bytes
+      cursor = conn.execute(
+          'SELECT od.key, od.data, od.object_store_id, od.file_ids, os.name '
+          'FROM object_data od '
+          'JOIN object_store os ON od.object_store_id == os.id')
+      for row in cursor:
+        key = self._ParseKey(row[0])
+        if row[3]:
+          value = row[1]
+        else:
+          value = self._ParseValue(row[1])
+        yield FirefoxIndexedDBRecord(
+            key=key,
+            value=value,
+            object_store_id=row[2],
+            file_ids=row[3],
+            object_store_name=row[4].decode('utf-8'),
+            database_name=self.database_name)

dfindexeddb/indexeddb/safari/definitions.py

@@ -16,15 +16,15 @@
 from enum import IntEnum
 
 
-CurrentVersion = 0x0000000F  # 15
-TerminatorTag = 0xFFFFFFFF
-StringPoolTag = 0xFFFFFFFE
-NonIndexPropertiesTag = 0xFFFFFFFD
-ImageDataPoolTag = 0xFFFFFFFE
-StringDataIs8BitFlag = 0x80000000
+CURRENT_VERSION = 0x0000000F  # 15
+TERMINATOR_TAG = 0xFFFFFFFF
+STRING_POOL_TAG = 0xFFFFFFFE
+NON_INDEX_PROPERTIES_TAG = 0xFFFFFFFD
+IMAGE_DATA_POOL_TAG = 0xFFFFFFFE
+STRING_DATA_IS_8BIT_FLAG = 0x80000000
 
 
-SIDBKeyVersion = 0x00
+SIDB_KEY_VERSION = 0x00
 
 
 class SIDBKeyType(IntEnum):

dfindexeddb/indexeddb/safari/webkit.py

@@ -23,6 +23,7 @@ from typing import Any, Dict, List, Tuple, Union
 
 from dfindexeddb import errors
 from dfindexeddb import utils
+from dfindexeddb.indexeddb import types
 from dfindexeddb.indexeddb.safari import definitions
 
 
@@ -82,78 +83,6 @@ class FileList:
   files: List[FileData]
 
 
-class JSArray(list):
-  """A parsed JavaScript array.
-
-  This is a wrapper around a standard Python list to allow assigning arbitrary
-  properties as is possible in the JavaScript equivalent.
-  """
-
-  def __repr__(self):
-    array_entries = ", ".join([str(entry) for entry in list(self)])
-    properties = ", ".join(
-        f'{key}: {value}' for key, value in self.properties.items())
-    return f'[{array_entries}, {properties}]'
-
-  @property
-  def properties(self) -> Dict[str, Any]:
-    """Returns the object properties."""
-    return self.__dict__
-
-  def __contains__(self, item):
-    return item in self.__dict__
-
-  def __getitem__(self, name):
-    return self.__dict__[name]
-
-
-class JSSet(set):
-  """A parsed JavaScript set.
-
-  This is a wrapper around a standard Python set to allow assigning arbitrary
-  properties as is possible in the JavaScript equivalent.
-  """
-
-  def __repr__(self):
-    set_entries = ", ".join([str(entry) for entry in list(self)])
-    properties = ", ".join(
-        f'{key}: {value}' for key, value in self.properties.items())
-    return f'[{set_entries}, {properties}]'
-
-  @property
-  def properties(self) -> Dict[str, Any]:
-    """Returns the object properties."""
-    return self.__dict__
-
-  def __contains__(self, item):
-    return item in self.__dict__
-
-  def __getitem__(self, name):
-    return self.__dict__[name]
-
-
-@dataclass
-class Null:
-  """A parsed JavaScript Null."""
-
-
-@dataclass
-class RegExp:
-  """A parsed JavaScript RegExp.
-
-  Attributes:
-    pattern: the pattern.
-    flags: the flags.
-  """
-  pattern: str
-  flags: str
-
-
-@dataclass(frozen=True)
-class Undefined:
-  """A parsed JavaScript undef."""
-
-
 @dataclass
 class IDBKeyData(utils.FromDecoderMixin):
   """An IDBKeyData.
@@ -169,7 +98,10 @@ class IDBKeyData(utils.FromDecoderMixin):
 
   @classmethod
   def FromDecoder(
-      cls, decoder: utils.StreamDecoder, base_offset: int = 0) -> IDBKeyData:
+      cls,
+      decoder: utils.StreamDecoder,
+      base_offset: int = 0
+  ) -> IDBKeyData:
     """Decodes an IDBKeyData from the current position of decoder.
 
     Refer to IDBSerialization.cpp for the encoding scheme.
@@ -211,7 +143,7 @@ class IDBKeyData(utils.FromDecoderMixin):
       return data
 
     offset, version_header = decoder.DecodeUint8()
-    if version_header != definitions.SIDBKeyVersion:
+    if version_header != definitions.SIDB_KEY_VERSION:
       raise errors.ParserError('SIDBKeyVersion not found.')
 
     _, raw_key_type = decoder.DecodeUint8()
@@ -279,7 +211,7 @@ class SerializedScriptValueDecoder():
       raise errors.ParserError(
           f'Invalid terminal {terminal_byte} at offset {offset}') from error
 
-  def DecodeArray(self) -> JSArray:
+  def DecodeArray(self) -> types.JSArray:
     """Decodes an Array value.
 
     Returns:
@@ -289,25 +221,25 @@ class SerializedScriptValueDecoder():
       ParserError if an invalid Terminator tag was found.
     """
     _, length = self.decoder.DecodeUint32()
-    array = JSArray()
+    array = types.JSArray()
     self.object_pool.append(array)
     for _ in range(length):
       _, _ = self.decoder.DecodeUint32()
       _, value = self.DecodeValue()
-      array.append(value)
+      array.values.append(value)
 
     offset, terminator_tag = self.decoder.DecodeUint32()
-    if terminator_tag != definitions.TerminatorTag:
+    if terminator_tag != definitions.TERMINATOR_TAG:
       raise errors.ParserError(f'Terminator tag not found at offset {offset}.')
 
     offset, tag = self.decoder.DecodeUint32()
-    if tag == definitions.NonIndexPropertiesTag:
-      while tag != definitions.TerminatorTag:
+    if tag == definitions.NON_INDEX_PROPERTIES_TAG:
+      while tag != definitions.TERMINATOR_TAG:
         name = self.DecodeStringData()
         _, value = self.DecodeValue()
         _, tag = self.decoder.DecodeUint32()
         array.properties[name] = value
-    elif tag != definitions.TerminatorTag:
+    elif tag != definitions.TERMINATOR_TAG:
       raise errors.ParserError(f'Terminator tag not found at offset {offset}.')
     return array
 
@@ -316,7 +248,7 @@ class SerializedScriptValueDecoder():
     tag = self.PeekTag()
     js_object = {}
     self.object_pool.append(js_object)
-    while tag != definitions.TerminatorTag:
+    while tag != definitions.TERMINATOR_TAG:
       name = self.DecodeStringData()
       _, value = self.DecodeValue()
       js_object[name] = value
@@ -338,10 +270,10 @@ class SerializedScriptValueDecoder():
           * unable to to decode a buffer as utf-16-le.
     """
     peeked_tag = self.PeekTag()
-    if peeked_tag == definitions.TerminatorTag:
+    if peeked_tag == definitions.TERMINATOR_TAG:
       raise errors.ParserError('Unexpected TerminatorTag found')
 
-    if peeked_tag == definitions.StringPoolTag:
+    if peeked_tag == definitions.STRING_POOL_TAG:
       _ = self.decoder.DecodeUint32()
       if len(self.constant_pool) <= 0xff:
         _, cp_index = self.decoder.DecodeUint8()
@@ -354,11 +286,11 @@ class SerializedScriptValueDecoder():
       return self.constant_pool[cp_index]
 
     _, length_with_8bit_flag = self.decoder.DecodeUint32()
-    if length_with_8bit_flag == definitions.TerminatorTag:
+    if length_with_8bit_flag == definitions.TERMINATOR_TAG:
       raise errors.ParserError('Disallowed string length found.')
 
     length = length_with_8bit_flag & 0x7FFFFFFF
-    is_8bit = length_with_8bit_flag & definitions.StringDataIs8BitFlag
+    is_8bit = length_with_8bit_flag & definitions.STRING_DATA_IS_8BIT_FLAG
 
     if is_8bit:
       _, characters = self.decoder.ReadBytes(length)
@@ -367,9 +299,10 @@ class SerializedScriptValueDecoder():
       _, characters = self.decoder.ReadBytes(2*length)
       try:
         value = characters.decode('utf-16-le')
-      except UnicodeDecodeError:
+      except UnicodeDecodeError as exc:
         raise errors.ParserError(
-            f'Unable to decode {len(characters)} characters as utf-16-le')
+            f'Unable to decode {len(characters)} characters as utf-16-le'
+        ) from exc
     self.constant_pool.append(value)
     return value
 
@@ -441,11 +374,11 @@ class SerializedScriptValueDecoder():
       'memory_cost': memory_cost
     }
 
-  def DecodeRegExp(self) -> RegExp:
+  def DecodeRegExp(self) -> types.RegExp:
     """Decodes a RegExp value."""
     pattern = self.DecodeStringData()
     flags = self.DecodeStringData()
-    return RegExp(pattern=pattern, flags=flags)
+    return types.RegExp(pattern=pattern, flags=flags)
 
   def DecodeMapData(self) -> dict:
     """Decodes a Map value."""
@@ -463,7 +396,7 @@ class SerializedScriptValueDecoder():
     _, tag = self.DecodeSerializationTag()
 
     pool_tag = self.PeekTag()
-    while pool_tag != definitions.TerminatorTag:
+    while pool_tag != definitions.TERMINATOR_TAG:
       name = self.DecodeStringData()
       value = self.DecodeValue()
       js_map[name] = value
@@ -472,22 +405,22 @@ class SerializedScriptValueDecoder():
     _, tag = self.decoder.DecodeUint32()
     return js_map
 
-  def DecodeSetData(self) -> JSSet:
+  def DecodeSetData(self) -> types.JSSet:
     """Decodes a SetData value."""
     tag = self.PeekSerializationTag()
-    js_set = JSSet()
+    js_set = types.JSSet()
     self.object_pool.append(js_set)
 
     while tag != definitions.SerializationTag.NON_SET_PROPERTIES:
       _, key = self.DecodeValue()
-      js_set.add(key)
+      js_set.values.add(key)
       tag = self.PeekSerializationTag()
 
     # consume the NonSetPropertiesTag
     _, tag = self.DecodeSerializationTag()
 
     pool_tag = self.PeekTag()
-    while pool_tag != definitions.TerminatorTag:
+    while pool_tag != definitions.TERMINATOR_TAG:
       name = self.DecodeStringData()
       value = self.DecodeValue()
       js_set.properties[name] = value
@@ -590,7 +523,7 @@ class SerializedScriptValueDecoder():
       ParserError when CurrentVersion is not found.
     """
     _, current_version = self.decoder.DecodeUint32()
-    if current_version != definitions.CurrentVersion:
+    if current_version != definitions.CURRENT_VERSION:
       raise errors.ParserError(
           f'{current_version} is not the expected CurrentVersion')
     _, value = self.DecodeValue()
@@ -611,9 +544,9 @@ class SerializedScriptValueDecoder():
     elif tag == definitions.SerializationTag.OBJECT:
       value = self.DecodeObject()
     elif tag == definitions.SerializationTag.UNDEFINED:
-      value = Undefined()
+      value = types.Undefined()
     elif tag == definitions.SerializationTag.NULL:
-      value = Null()
+      value = types.Null()
     elif tag == definitions.SerializationTag.INT:
       _, value = self.decoder.DecodeInt32()
     elif tag == definitions.SerializationTag.ZERO:

dfindexeddb/indexeddb/types.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Types for indexeddb."""
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Dict, List, Set
+
+
+@dataclasses.dataclass
+class JSArray:
+  """A parsed Javascript array.
+
+  A JavaScript array behaves like a Python list but allows assigning arbitrary
+  properties.  The array is stored in the attribute __array__.
+
+  Attributes:
+    values: the array values.
+    properties: the array properties.
+  """
+  values: List[Any] = dataclasses.field(default_factory=list)
+  properties: Dict[Any, Any] = dataclasses.field(default_factory=dict)
+
+
+@dataclasses.dataclass
+class JSSet:
+  """A parsed JavaScript set.
+
+  A JavaScript set behaves like a Python set but allows assigning arbitrary
+  properties.  The array is stored in the attribute __set__.
+
+  Attributes:
+    values: the set values.
+    properties: the set properties.
+  """
+  values: Set[Any] = dataclasses.field(default_factory=set)
+  properties: Dict[Any, Any] = dataclasses.field(default_factory=dict)
+
+
+@dataclasses.dataclass
+class Null:
+  """A parsed JavaScript Null."""
+
+
+@dataclasses.dataclass
+class RegExp:
+  """A parsed JavaScript RegExp.
+  
+  Attributes:
+    pattern: the pattern.
+    flags: the flags.
+  """
+  pattern: str
+  flags: str
+
+
+@dataclasses.dataclass
+class Undefined:
+  """A JavaScript undef."""

dfindexeddb/leveldb/cli.py

@@ -203,10 +203,11 @@ def App():
   parser_db = subparsers.add_parser(
       'db', help='Parse a directory as leveldb.')
   parser_db.add_argument(
-      '-s', '--source',
+      '-s',
+      '--source',
       required=True,
       type=pathlib.Path,
-      help='The source leveldb directory')
+      help='The source leveldb directory.')
   recover_group = parser_db.add_mutually_exclusive_group()
   recover_group.add_argument(
       '--use_manifest',
@@ -226,7 +227,7 @@ def App():
           'jsonl',
           'repr'],
       default='json',
-      help='Output format.  Default is json')
+      help='Output format.  Default is json.')
   parser_db.add_argument(
       '--plugin',
       help='Use plugin to parse records.')
@@ -235,10 +236,11 @@ def App():
   parser_log = subparsers.add_parser(
       'log', help='Parse a leveldb log file.')
   parser_log.add_argument(
-      '-s', '--source',
+      '-s',
+      '--source',
       required=True,
       type=pathlib.Path,
-      help='The source leveldb file')
+      help='The source leveldb file.')
   parser_log.add_argument(
       '-o',
       '--output',
@@ -247,7 +249,7 @@ def App():
           'jsonl',
           'repr'],
       default='json',
-      help='Output format.  Default is json')
+      help='Output format.  Default is json.')
   parser_log.add_argument(
       '--plugin',
       help='Use plugin to parse records.')
@@ -265,7 +267,8 @@ def App():
   parser_ldb = subparsers.add_parser(
       'ldb', help='Parse a leveldb table (.ldb) file.')
   parser_ldb.add_argument(
-      '-s', '--source',
+      '-s',
+      '--source',
       required=True,
       type=pathlib.Path,
       help='The source leveldb file')
@@ -277,7 +280,7 @@ def App():
           'jsonl',
           'repr'],
       default='json',
-      help='Output format.  Default is json')
+      help='Output format.  Default is json.')
   parser_ldb.add_argument(
       '--plugin',
       help='Use plugin to parse records.')
@@ -293,7 +296,8 @@ def App():
   parser_descriptor = subparsers.add_parser(
       'descriptor', help='Parse a leveldb descriptor (MANIFEST) file.')
   parser_descriptor.add_argument(
-      '-s', '--source',
+      '-s',
+      '--source',
       required=True,
       type=pathlib.Path,
       help='The source leveldb file')
@@ -305,13 +309,16 @@ def App():
           'jsonl',
           'repr'],
       default='json',
-      help='Output format.  Default is json')
+      help='Output format.  Default is json.')
   db_group = parser_descriptor.add_mutually_exclusive_group()
   db_group.add_argument(
       '-t',
       '--structure_type',
       choices=[
-          'blocks', 'physical_records', 'versionedit'],
+          'blocks',
+          'physical_records',
+          'versionedit'
+      ],
       help='Parses the specified structure.  Default is versionedit.')
   db_group.add_argument(
       '-v',