dfindexeddb 20240501__py3-none-any.whl → 20241031__py3-none-any.whl

dfindexeddb/indexeddb/firefox/record.py

@@ -0,0 +1,180 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Firefox IndexedDB records."""
+from dataclasses import dataclass
+import sqlite3
+import sys
+import traceback
+from typing import Any, Generator, Optional
+
+from dfindexeddb import errors
+from dfindexeddb.indexeddb.firefox import gecko
+
+
+@dataclass
+class FirefoxObjectStoreInfo:
+  """A FireFox ObjectStoreInfo.
+
+  Attributes:
+    id: the object store ID.
+    name: the object store name.
+    key_path: the object store key path.
+    auto_inc: the current auto-increment value.
+    database_name: the database name from the database table.
+  """
+  id: int
+  name: str
+  key_path: str
+  auto_inc: int
+  database_name: str
+
+
+@dataclass
+class FirefoxIndexedDBRecord:
+  """A Firefox IndexedDBRecord.
+
+  Attributes:
+    key: the parsed key.
+    value: the parsed value.
+    file_ids: the file identifiers.
+    object_store_id: the object store id.
+    object_store_name: the object store name from the object_store table.
+    database_name: the IndexedDB database name from the database table.
+  """
+  key: Any
+  value: Any
+  file_ids: Optional[str]
+  object_store_id: int
+  object_store_name: str
+  database_name: str
+
+
+class FileReader:
+  """A reader for Firefox IndexedDB sqlite3 files.
+
+  Attributes:
+    database_name: the database name.
+    origin: the database origin.
+    metadata_version: the metadata version.
+    last_vacuum_time: the last vacuum time.
+    last_analyze_time: the last analyze time.
+  """
+
+  def __init__(self, filename: str):
+    """Initializes the FileReader.
+
+    Args:
+      filename: the IndexedDB filename.
+    """
+    self.filename = filename
+
+    with sqlite3.connect(f'file:{self.filename}?mode=ro', uri=True) as conn:
+      cursor = conn.execute(
+          'SELECT name, origin, version, last_vacuum_time, last_analyze_time '
+          'FROM database')
+      result = cursor.fetchone()
+      self.database_name = result[0]
+      self.origin = result[1]
+      self.metadata_version = result[2]
+      self.last_vacuum_time = result[3]
+      self.last_analyze_time = result[4]
+
+  def _ParseKey(self, key: bytes) -> Any:
+    """Parses a key."""
+    try:
+      return gecko.IDBKey.FromBytes(key)
+    except errors.ParserError as e:
+      print('failed to parse', key, file=sys.stderr)
+      traceback.print_exception(type(e), e, e.__traceback__)
+      return key
+
+  def _ParseValue(self, value: bytes) -> Any:
+    """Parses a value."""
+    try:
+      return gecko.JSStructuredCloneDecoder.FromBytes(value)
+    except errors.ParserError as err:
+      print('failed to parse', value, file=sys.stderr)
+      traceback.print_exception(type(err), err, err.__traceback__)
+      return value
+
+  def ObjectStores(self) -> Generator[FirefoxObjectStoreInfo, None, None]:
+    """Returns the Object Store information from the IndexedDB database.
+
+    Yields:
+      FirefoxObjectStoreInfo instances.
+    """
+    with sqlite3.connect(f'file:{self.filename}?mode=ro', uri=True) as conn:
+      cursor = conn.execute(
+          'SELECT id, auto_increment, name, key_path FROM object_store')
+      results = cursor.fetchall()
+      for result in results:
+        yield FirefoxObjectStoreInfo(
+            id=result[0],
+            name=result[2],
+            key_path=result[3],
+            auto_inc=result[1],
+            database_name=self.database_name)
+
+  def RecordsByObjectStoreId(
+      self,
+      object_store_id: int
+  ) -> Generator[FirefoxIndexedDBRecord, None, None]:
+    """Returns FirefoxIndexedDBRecords by a given object store id.
+
+    Args:
+      object_store_id: the object store id.
+    """
+    with sqlite3.connect(f'file:{self.filename}?mode=ro', uri=True) as conn:
+      conn.text_factory = bytes
+      cursor = conn.execute(
+          'SELECT od.key, od.data, od.object_store_id, od.file_ids, os.name '
+          'FROM object_data od '
+          'JOIN object_store os ON od.object_store_id == os.id '
+          'WHERE os.id = ? ORDER BY od.key', (object_store_id, ))
+      for row in cursor:
+        key = self._ParseKey(row[0])
+        if row[3]:
+          value = row[1]
+        else:
+          value = self._ParseValue(row[1])
+        yield FirefoxIndexedDBRecord(
+            key=key,
+            value=value,
+            object_store_id=row[2],
+            file_ids=row[3],
+            object_store_name=row[4].decode('utf-8'),
+            database_name=self.database_name)
+
+  def Records(self) -> Generator[FirefoxIndexedDBRecord, None, None]:
+    """Returns FirefoxIndexedDBRecords from the database."""
+    with sqlite3.connect(f'file:{self.filename}?mode=ro', uri=True) as conn:
+      conn.text_factory = bytes
+      cursor = conn.execute(
+          'SELECT od.key, od.data, od.object_store_id, od.file_ids, os.name '
+          'FROM object_data od '
+          'JOIN object_store os ON od.object_store_id == os.id')
+      for row in cursor:
+        key = self._ParseKey(row[0])
+        if row[3]:
+          value = row[1]
+        else:
+          value = self._ParseValue(row[1])
+        yield FirefoxIndexedDBRecord(
+            key=key,
+            value=value,
+            object_store_id=row[2],
+            file_ids=row[3],
+            object_store_name=row[4].decode('utf-8'),
+            database_name=self.database_name)

dfindexeddb/indexeddb/safari/webkit.py

@@ -82,15 +82,22 @@ class FileList:
   files: List[FileData]
 
 
-class JSArray(list):
-  """A parsed JavaScript array.
+class JSArray:
+  """A parsed Javascript array.
 
-  This is a wrapper around a standard Python list to allow assigning arbitrary
-  properties as is possible in the JavaScript equivalent.
+  A Javascript array behaves like a Python list but allows assigning arbitrary
+  properties.  The array is stored in the attribute __array__.
   """
+  def __init__(self):
+    self.__array__ = []
+
+  def Append(self, element: Any):
+    """Appends a new element to the array."""
+    self.__array__.append(element)
 
   def __repr__(self):
-    array_entries = ", ".join([str(entry) for entry in list(self)])
+    array_entries = ", ".join(
+        [str(entry) for entry in list(self.__array__)])
     properties = ", ".join(
         f'{key}: {value}' for key, value in self.properties.items())
     return f'[{array_entries}, {properties}]'
@@ -100,6 +107,11 @@ class JSArray(list):
     """Returns the object properties."""
     return self.__dict__
 
+  def __eq__(self, other: JSArray):
+    return (
+        self.__array__ == other.__array__
+        and self.properties == other.properties)
+
   def __contains__(self, item):
     return item in self.__dict__
 
@@ -107,24 +119,36 @@ class JSArray(list):
     return self.__dict__[name]
 
 
-class JSSet(set):
-  """A parsed JavaScript set.
+class JSSet:
+  """A parsed Javascript set.
 
-  This is a wrapper around a standard Python set to allow assigning arbitrary
-  properties as is possible in the JavaScript equivalent.
+  A Javascript set behaves like a Python set but allows assigning arbitrary
+  properties.  The array is stored in the attribute __set__.
   """
+  def __init__(self):
+    self.__set__ = set()
+
+  def Add(self, element: Any):
+    """Adds a new element to the set."""
+    self.__set__.add(element)
 
   def __repr__(self):
-    set_entries = ", ".join([str(entry) for entry in list(self)])
+    array_entries = ", ".join(
+        [str(entry) for entry in list(self.__set__)])
     properties = ", ".join(
         f'{key}: {value}' for key, value in self.properties.items())
-    return f'[{set_entries}, {properties}]'
+    return f'[{array_entries}, {properties}]'
 
   @property
   def properties(self) -> Dict[str, Any]:
     """Returns the object properties."""
     return self.__dict__
 
+  def __eq__(self, other: JSSet):
+    return (
+        self.__set__ == other.__set__
+        and self.properties == other.properties)
+
   def __contains__(self, item):
     return item in self.__dict__
 
@@ -290,10 +314,11 @@ class SerializedScriptValueDecoder():
     """
     _, length = self.decoder.DecodeUint32()
     array = JSArray()
+    self.object_pool.append(array)
     for _ in range(length):
       _, _ = self.decoder.DecodeUint32()
       _, value = self.DecodeValue()
-      array.append(value)
+      array.Append(value)
 
     offset, terminator_tag = self.decoder.DecodeUint32()
     if terminator_tag != definitions.TerminatorTag:
@@ -314,13 +339,13 @@ class SerializedScriptValueDecoder():
     """Decodes an Object value."""
     tag = self.PeekTag()
     js_object = {}
+    self.object_pool.append(js_object)
     while tag != definitions.TerminatorTag:
       name = self.DecodeStringData()
       _, value = self.DecodeValue()
       js_object[name] = value
       tag = self.PeekTag()
     _ = self.decoder.DecodeUint32()
-    self.object_pool.append(js_object)
     return js_object
 
   def DecodeStringData(self) -> str:
@@ -342,11 +367,11 @@ class SerializedScriptValueDecoder():
 
     if peeked_tag == definitions.StringPoolTag:
       _ = self.decoder.DecodeUint32()
-      if len(self.constant_pool) < 0xff:
+      if len(self.constant_pool) <= 0xff:
         _, cp_index = self.decoder.DecodeUint8()
-      elif len(self.constant_pool) < 0xffff:
+      elif len(self.constant_pool) <= 0xffff:
         _, cp_index = self.decoder.DecodeUint16()
-      elif len(self.constant_pool) < 0xffffffff:
+      elif len(self.constant_pool) <= 0xffffffff:
         _, cp_index = self.decoder.DecodeUint32()
       else:
         raise errors.ParserError('Unexpected constant pool size value.')
@@ -450,6 +475,7 @@ class SerializedScriptValueDecoder():
     """Decodes a Map value."""
     tag = self.PeekSerializationTag()
     js_map = {}   # TODO: make this into a JSMap (like JSArray/JSSet)
+    self.object_pool.append(js_map)
 
     while tag != definitions.SerializationTag.NON_MAP_PROPERTIES:
       _, key = self.DecodeValue()
@@ -468,17 +494,17 @@ class SerializedScriptValueDecoder():
       pool_tag = self.PeekTag()
 
     _, tag = self.decoder.DecodeUint32()
-
     return js_map
 
   def DecodeSetData(self) -> JSSet:
     """Decodes a SetData value."""
     tag = self.PeekSerializationTag()
     js_set = JSSet()
+    self.object_pool.append(js_set)
 
     while tag != definitions.SerializationTag.NON_SET_PROPERTIES:
       _, key = self.DecodeValue()
-      js_set.add(key)
+      js_set.Add(key)
       tag = self.PeekSerializationTag()
 
     # consume the NonSetPropertiesTag
@@ -540,8 +566,13 @@ class SerializedScriptValueDecoder():
 
   def DecodeObjectReference(self) -> Any:
     """Decodes an ObjectReference value."""
-    _, object_ref = self.decoder.DecodeUint8()
-    return self.object_pool[object_ref - 1]
+    if len(self.object_pool) < 0xFF:
+      _, object_ref = self.decoder.DecodeUint8()
+    elif len(self.object_pool) < 0xFFFF:
+      _, object_ref = self.decoder.DecodeUint16()
+    else:  # if len(self.object_pool) < 0xFFFFFFFF:
+      _, object_ref = self.decoder.DecodeUint32()
+    return self.object_pool[object_ref]
 
   def DecodeArrayBufferView(self) -> ArrayBufferView:
     """Decodes an ArrayBufferView value.
@@ -641,6 +672,7 @@ class SerializedScriptValueDecoder():
       value = self.DecodeArrayBuffer()
     elif tag == definitions.SerializationTag.ARRAY_BUFFER_VIEW:
       value = self.DecodeArrayBufferView()
+      self.object_pool.append(value)
     elif tag == definitions.SerializationTag.ARRAY_BUFFER_TRANSFER:
       value = self.DecodeArrayBufferTransfer()
     elif tag == definitions.SerializationTag.TRUE_OBJECT:

dfindexeddb/indexeddb/types.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Types for indexeddb."""
+from __future__ import annotations
+
+import dataclasses
+from typing import Any, Dict, List, Set
+
+
+@dataclasses.dataclass
+class JSArray:
+  """A parsed Javascript array.
+
+  A JavaScript array behaves like a Python list but allows assigning arbitrary
+  properties.  The array is stored in the attribute __array__.
+
+  Attributes:
+    values: the array values.
+    properties: the array properties.
+  """
+  values: List[Any] = dataclasses.field(default_factory=list)
+  properties: Dict[Any, Any] = dataclasses.field(default_factory=dict)
+
+
+@dataclasses.dataclass
+class JSSet:
+  """A parsed JavaScript set.
+
+  A JavaScript set behaves like a Python set but allows assigning arbitrary
+  properties.  The array is stored in the attribute __set__.
+
+  Attributes:
+    values: the set values.
+    properties: the set properties.
+  """
+  values: Set[Any] = dataclasses.field(default_factory=set)
+  properties: Dict[Any, Any] = dataclasses.field(default_factory=dict)
+
+
+@dataclasses.dataclass
+class Null:
+  """A parsed JavaScript Null."""
+
+
+@dataclasses.dataclass
+class RegExp:
+  """A parsed JavaScript RegExp.
+  
+  Attributes:
+    pattern: the pattern.
+    flags: the flags.
+  """
+  pattern: str
+  flags: int
+
+
+@dataclasses.dataclass
+class Undefined:
+  """A JavaScript undef."""

dfindexeddb/leveldb/cli.py

@@ -19,11 +19,13 @@ from datetime import datetime
 import json
 import pathlib
 
+from dfindexeddb import utils
 from dfindexeddb import version
 from dfindexeddb.leveldb import descriptor
 from dfindexeddb.leveldb import ldb
 from dfindexeddb.leveldb import log
 from dfindexeddb.leveldb import record
+from dfindexeddb.leveldb.plugins import manager
 
 
 _VALID_PRINTABLE_CHARACTERS = (
@@ -37,7 +39,7 @@ class Encoder(json.JSONEncoder):
   def default(self, o):
     """Returns a serializable object for o."""
     if dataclasses.is_dataclass(o):
-      o_dict = dataclasses.asdict(o)
+      o_dict = utils.asdict(o)
       return o_dict
     if isinstance(o, bytes):
       out = []
@@ -66,13 +68,39 @@ def _Output(structure, output):
 
 def DbCommand(args):
   """The CLI for processing leveldb folders."""
+  if args.plugin and args.plugin == 'list':
+    for plugin, _ in manager.LeveldbPluginManager.GetPlugins():
+      print(plugin)
+    return
+
+  if args.plugin:
+    plugin_class = manager.LeveldbPluginManager.GetPlugin(args.plugin)
+  else:
+    plugin_class = None
+
   for leveldb_record in record.FolderReader(
-      args.source).GetRecords(use_manifest=args.use_manifest):
-    _Output(leveldb_record, output=args.output)
+      args.source).GetRecords(
+          use_manifest=args.use_manifest,
+          use_sequence_number=args.use_sequence_number):
+    if plugin_class:
+      plugin_record = plugin_class.FromLevelDBRecord(leveldb_record)
+      _Output(plugin_record, output=args.output)
+    else:
+      _Output(leveldb_record, output=args.output)
 
 
 def LdbCommand(args):
   """The CLI for processing ldb files."""
+  if args.plugin and args.plugin == 'list':
+    for plugin, _ in manager.LeveldbPluginManager.GetPlugins():
+      print(plugin)
+    return
+
+  if args.plugin:
+    plugin_class = manager.LeveldbPluginManager.GetPlugin(args.plugin)
+  else:
+    plugin_class = None
+
   ldb_file = ldb.FileReader(args.source)
 
   if args.structure_type == 'blocks':
@@ -83,7 +111,11 @@ def LdbCommand(args):
   elif args.structure_type == 'records' or not args.structure_type:
     # Prints key value record information.
     for key_value_record in ldb_file.GetKeyValueRecords():
-      _Output(key_value_record, output=args.output)
+      if plugin_class:
+        plugin_record = plugin_class.FromKeyValueRecord(key_value_record)
+        _Output(plugin_record, output=args.output)
+      else:
+        _Output(key_value_record, output=args.output)
 
   else:
     print(f'{args.structure_type} is not supported for ldb files.')
@@ -91,6 +123,16 @@ def LdbCommand(args):
 
 def LogCommand(args):
   """The CLI for processing log files."""
+  if args.plugin and args.plugin == 'list':
+    for plugin, _ in manager.LeveldbPluginManager.GetPlugins():
+      print(plugin)
+    return
+
+  if args.plugin:
+    plugin_class = manager.LeveldbPluginManager.GetPlugin(args.plugin)
+  else:
+    plugin_class = None
+
   log_file = log.FileReader(args.source)
 
   if args.structure_type == 'blocks':
@@ -112,7 +154,11 @@ def LogCommand(args):
         or not args.structure_type):
     # Prints key value record information.
     for internal_key_record in log_file.GetParsedInternalKeys():
-      _Output(internal_key_record, output=args.output)
+      if plugin_class:
+        plugin_record = plugin_class.FromKeyValueRecord(internal_key_record)
+        _Output(plugin_record, output=args.output)
+      else:
+        _Output(internal_key_record, output=args.output)
 
   else:
     print(f'{args.structure_type} is not supported for log files.')
@@ -144,6 +190,7 @@ def DescriptorCommand(args):
   else:
     print(f'{args.structure_type} is not supported for descriptor files.')
 
+
 def App():
   """The CLI app entrypoint for parsing leveldb files."""
   parser = argparse.ArgumentParser(
@@ -160,10 +207,17 @@ def App():
       required=True,
       type=pathlib.Path,
       help='The source leveldb directory')
-  parser_db.add_argument(
+  recover_group = parser_db.add_mutually_exclusive_group()
+  recover_group.add_argument(
       '--use_manifest',
       action='store_true',
       help='Use manifest file to determine active/deleted records.')
+  recover_group.add_argument(
+      '--use_sequence_number',
+      action='store_true',
+      help=(
+          'Use sequence number and file offset to determine active/deleted '
+          'records.'))
   parser_db.add_argument(
       '-o',
       '--output',
@@ -173,6 +227,9 @@ def App():
           'repr'],
       default='json',
       help='Output format.  Default is json')
+  parser_db.add_argument(
+      '--plugin',
+      help='Use plugin to parse records.')
   parser_db.set_defaults(func=DbCommand)
 
   parser_log = subparsers.add_parser(
@@ -191,6 +248,9 @@ def App():
           'repr'],
       default='json',
       help='Output format.  Default is json')
+  parser_log.add_argument(
+      '--plugin',
+      help='Use plugin to parse records.')
   parser_log.add_argument(
       '-t',
       '--structure_type',
@@ -218,6 +278,9 @@ def App():
           'repr'],
       default='json',
       help='Output format.  Default is json')
+  parser_ldb.add_argument(
+      '--plugin',
+      help='Use plugin to parse records.')
   parser_ldb.add_argument(
       '-t',
       '--structure_type',

dfindexeddb/leveldb/descriptor.py

@@ -384,5 +384,6 @@ class FileReader:
 
   def GetLatestVersion(self) -> LevelDBVersion:
     """Returns the latest LevelDBVersion instance."""
-    *_, latest = self.GetVersions()
+    for version in self.GetVersions():
+      latest = version
     return latest

dfindexeddb/leveldb/log.py

@@ -152,7 +152,7 @@ class PhysicalRecord(utils.FromDecoderMixin):
   @classmethod
   def FromDecoder(
       cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
-  ) -> PhysicalRecord:
+  ) -> Optional[PhysicalRecord]:
     """Decodes a PhysicalRecord from the current position of a LevelDBDecoder.
 
     Args:
@@ -161,11 +161,13 @@ class PhysicalRecord(utils.FromDecoderMixin):
           read from.
 
     Returns:
-      A PhysicalRecord.
+      A PhysicalRecord or None if the parsed header is 0.
     """
     offset, checksum = decoder.DecodeUint32()
     _, length = decoder.DecodeUint16()
     _, record_type_byte = decoder.DecodeUint8()
+    if checksum == 0 or length == 0 or record_type_byte == 0:
+      return None
     try:
       record_type = definitions.LogFilePhysicalRecordType(record_type_byte)
     except ValueError as error:
@@ -206,7 +208,11 @@ class Block:
     buffer_length = len(self.data)
 
     while buffer.tell() + PhysicalRecord.PHYSICAL_HEADER_LENGTH < buffer_length:
-      yield PhysicalRecord.FromStream(buffer, base_offset=self.offset)
+      record = PhysicalRecord.FromStream(buffer, base_offset=self.offset)
+      if record:
+        yield record
+      else:
+        return
 
   @classmethod
   def FromStream(cls, stream: BinaryIO) -> Optional[Block]:

dfindexeddb/leveldb/plugins/__init__.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Leveldb Plugin module."""
+
+from dfindexeddb.leveldb.plugins import chrome_notifications