dfindexeddb 20240402__py3-none-any.whl → 20240501__py3-none-any.whl

dfindexeddb/indexeddb/chromium/definitions.py

@@ -141,10 +141,12 @@ class BlinkSerializationTag(IntEnum):
   ENCODED_AUDIO_CHUNK = ord('y')
   ENCODED_VIDEO_CHUNK = ord('z')
   CROP_TARGET = ord('c')
+  RESTRICTION_TARGET = ord('D')
   MEDIA_SOURCE_HANDLE = ord('S')
   DEPRECATED_DETECTED_BARCODE = ord('B')
   DEPRECATED_DETECTED_FACE = ord('F')
   DEPRECATED_DETECTED_TEXT = ord('t')
+  FENCED_FRAME_CONFIG = ord('C')
   DOM_EXCEPTION = ord('x')
   TRAILER_OFFSET = 0xFE
   VERSION = 0xFF
@@ -304,3 +306,67 @@ class V8ErrorTag(IntEnum):
   CAUSE = ord('c')
   STACK = ord('s')
   END = ord('.')
+
+
+class ImageSerializationTag(IntEnum):
+  """Image Serialization tags."""
+  END = 0
+  PREDEFINED_COLOR_SPACE = 1
+  CANVAS_PIXEL_FORMAT = 2
+  IMAGE_DATA_STORAGE_FORMAT = 3
+  ORIGIN_CLEAN = 4
+  IS_PREMULTIPLIED = 5
+  CANVAS_OPACITY_MODE = 6
+  PARAMETRIC_COLOR_SPACE = 7
+  IMAGE_ORIENTATION = 8
+  LAST = IMAGE_ORIENTATION
+
+
+class SerializedPredefinedColorSpace(IntEnum):
+  """Serialized Predefined Color Space enumeration."""
+  LEGACY_OBSOLETE = 0
+  SRGB = 1
+  REC2020 = 2
+  P3 = 3
+  REC2100HLG = 4
+  REC2100PQ = 5
+  SRGB_LINEAR = 6
+  LAST = SRGB_LINEAR
+
+
+class SerializedPixelFormat(IntEnum):
+  """Serialized Pixel Format enumeration."""
+  NATIVE8_LEGACY_OBSOLETE = 0
+  F16 = 1
+  RGBA8 = 2
+  BGRA8 = 3
+  RGBX8 = 4
+  LAST = RGBX8
+
+
+class SerializedImageDataStorageFormat(IntEnum):
+  """The Serialized Image Data Storage Format."""
+  UINT8CLAMPED = 0
+  UINT16 = 1
+  FLOAT32 = 2
+  LAST = FLOAT32
+
+
+class SerializedOpacityMode(IntEnum):
+  """The Serialized Opacity Mode."""
+  KNONOPAQUE = 0
+  KOPAQUE = 1
+  KLAST = KOPAQUE
+
+
+class SerializedImageOrientation(IntEnum):
+  """The Serialized Image Orientation."""
+  TOP_LEFT = 0
+  TOP_RIGHT = 1
+  BOTTOM_RIGHT = 2
+  BOTTOM_LEFT = 3
+  LEFT_TOP = 4
+  RIGHT_TOP = 5
+  RIGHT_BOTTOM = 6
+  LEFT_BOTTOM = 7
+  LAST = LEFT_BOTTOM

dfindexeddb/indexeddb/chromium/record.py

@@ -17,13 +17,16 @@ from __future__ import annotations
 from dataclasses import dataclass, field
 from datetime import datetime
 import io
-from typing import Any, BinaryIO, Optional, Tuple, Type, TypeVar, Union
+import pathlib
+import sys
+import traceback
+from typing import Any, BinaryIO, Generator, Optional, Tuple, Type, TypeVar, \
+    Union
 
 from dfindexeddb import errors
 from dfindexeddb.indexeddb.chromium import blink
 from dfindexeddb.indexeddb.chromium import definitions
-from dfindexeddb.leveldb import ldb
-from dfindexeddb.leveldb import log
+from dfindexeddb.leveldb import record
 from dfindexeddb.leveldb import utils
 
 
@@ -457,7 +460,7 @@ class MaxDatabaseIdKey(BaseIndexedDBKey):
       cls, decoder: utils.LevelDBDecoder, key_prefix: KeyPrefix,
       base_offset: int = 0
   ) -> MaxDatabaseIdKey:
-    """Decodes the maximum databse key."""
+    """Decodes the maximum database key."""
     offset, key_type = decoder.DecodeUint8()
     if key_type != definitions.GlobalMetadataKeyType.MAX_DATABASE_ID:
       raise errors.ParserError('Not a MaxDatabaseIdKey')
@@ -546,7 +549,7 @@ class EarliestSweepKey(BaseIndexedDBKey):
 
 
 @dataclass
-class EarlistCompactionTimeKey(BaseIndexedDBKey):
+class EarliestCompactionTimeKey(BaseIndexedDBKey):
   """An earliest compaction time IndexedDB key."""
 
   def DecodeValue(self, decoder: utils.LevelDBDecoder) -> int:
@@ -558,11 +561,11 @@ class EarlistCompactionTimeKey(BaseIndexedDBKey):
   def FromDecoder(
       cls, decoder: utils.LevelDBDecoder, key_prefix: KeyPrefix,
       base_offset: int = 0
-  ) -> EarlistCompactionTimeKey:
+  ) -> EarliestCompactionTimeKey:
     """Decodes the earliest compaction time key."""
     offset, key_type = decoder.DecodeUint8()
     if key_type != definitions.GlobalMetadataKeyType.EARLIEST_COMPACTION_TIME:
-      raise errors.ParserError('Not a EarlistCompactionTimeKey')
+      raise errors.ParserError('Not a EarliestCompactionTimeKey')
     return cls(offset=base_offset + offset, key_prefix=key_prefix)
 
 
@@ -668,7 +671,7 @@ class GlobalMetaDataKey(BaseIndexedDBKey):
       definitions.GlobalMetadataKeyType
           .EARLIEST_SWEEP: EarliestSweepKey,
       definitions.GlobalMetadataKeyType
-          .EARLIEST_COMPACTION_TIME: EarlistCompactionTimeKey,
+          .EARLIEST_COMPACTION_TIME: EarliestCompactionTimeKey,
       definitions.GlobalMetadataKeyType
           .SCOPES_PREFIX: ScopesPrefixKey,
       definitions.GlobalMetadataKeyType
@@ -692,7 +695,7 @@ class GlobalMetaDataKey(BaseIndexedDBKey):
              Type[DatabaseFreeListKey],
              Type[DatabaseNameKey],
              Type[EarliestSweepKey],
-             Type[EarlistCompactionTimeKey],
+             Type[EarliestCompactionTimeKey],
              Type[MaxDatabaseIdKey],
              Type[RecoveryBlobJournalKey],
              Type[SchemaVersionKey],
@@ -972,7 +975,7 @@ class ObjectStoreDataValue:
     blob_offset: the blob offset, only valid if wrapped.
     value: the blink serialized value, only valid if not wrapped.
   """
-  unkown: int
+  unknown: int
   is_wrapped: bool
   blob_size: Optional[int]
   blob_offset: Optional[int]
@@ -1003,7 +1006,7 @@ class ObjectStoreDataKey(BaseIndexedDBKey):
       _, blob_size = decoder.DecodeVarint()
       _, blob_offset = decoder.DecodeVarint()
       return ObjectStoreDataValue(
-          unkown=unknown_integer,
+          unknown=unknown_integer,
           is_wrapped=True,
           blob_size=blob_size,
           blob_offset=blob_offset,
@@ -1011,7 +1014,7 @@ class ObjectStoreDataKey(BaseIndexedDBKey):
     _, blink_bytes = decoder.ReadBytes()
     blink_value = blink.V8ScriptValueDecoder.FromBytes(blink_bytes)
     return ObjectStoreDataValue(
-        unkown=unknown_integer,
+        unknown=unknown_integer,
         is_wrapped=False,
         blob_size=None,
         blob_offset=None,
@@ -1332,29 +1335,112 @@ class IndexedDBRecord:
   """An IndexedDB Record.
 
   Attributes:
+    path: the source file path
     offset: the offset of the record.
     key: the key of the record.
     value: the value of the record.
     sequence_number: if available, the sequence number of the record.
     type: the type of the record.
+    level: the leveldb level, if applicable, None can indicate the record
+        originated from a log file or the level could not be determined.
+    recovered: True if the record is a recovered record.
   """
+  path: str
   offset: int
   key: Any
   value: Any
-  sequence_number: int
+  sequence_number: Optional[int]
   type: int
+  level: Optional[int]
+  recovered: Optional[bool]
 
   @classmethod
   def FromLevelDBRecord(
-      cls, record: Union[ldb.KeyValueRecord, log.ParsedInternalKey]
+      cls,
+      db_record: record.LevelDBRecord
   ) -> IndexedDBRecord:
     """Returns an IndexedDBRecord from a ParsedInternalKey."""
-    idb_key = IndexedDbKey.FromBytes(record.key, base_offset=record.offset)
-    idb_value = idb_key.ParseValue(record.value)
+    idb_key = IndexedDbKey.FromBytes(
+        db_record.record.key, base_offset=db_record.record.offset)
+    idb_value = idb_key.ParseValue(db_record.record.value)
     return cls(
-      offset=record.offset,
-      key=idb_key,
-      value=idb_value,
-      sequence_number=record.sequence_number if hasattr(
-          record, 'sequence_number') else None,
-      type=record.record_type)
+        path=db_record.path,
+        offset=db_record.record.offset,
+        key=idb_key,
+        value=idb_value,
+        sequence_number=db_record.record.sequence_number if hasattr(
+            db_record.record, 'sequence_number') else None,
+        type=db_record.record.record_type,
+        level=db_record.level,
+        recovered=db_record.recovered)
+
+  @classmethod
+  def FromFile(
+      cls,
+      file_path: pathlib.Path
+  ) -> Generator[IndexedDBRecord, None, None]:
+    """Yields IndexedDBRecords from a file."""
+    for db_record in record.LevelDBRecord.FromFile(file_path):
+      try:
+        yield cls.FromLevelDBRecord(db_record)
+      except(
+          errors.ParserError,
+          errors.DecoderError,
+          NotImplementedError) as err:
+        print((
+            'Error parsing Indexeddb record: '
+            f'{err} at offset {db_record.record.offset} in '
+            f'{db_record.path}'),
+            file=sys.stderr)
+        print(f'Traceback: {traceback.format_exc()}', file=sys.stderr)
+
+
+class FolderReader:
+  """A IndexedDB folder reader for Chrome/Chromium.
+
+  Attributes:
+    foldername (str): the source LevelDB folder.
+  """
+
+  def __init__(self, foldername: pathlib.Path):
+    """Initializes the FileReader.
+
+    Args:
+      foldername: the source IndexedDB folder.
+
+    Raises:
+      ValueError: if foldername is None or not a directory.
+    """
+    if not foldername or not foldername.is_dir():
+      raise ValueError(f'{foldername} is None or not a directory')
+    self.foldername = foldername
+
+  def GetRecords(
+      self,
+      use_manifest: bool = False
+  ) -> Generator[IndexedDBRecord, None, None]:
+    """Yield LevelDBRecords.
+
+    Args:
+      use_manifest: True to use the current manifest in the folder as a means to
+          find the active file set.
+
+    Yields:
+      IndexedDBRecord.
+    """
+    leveldb_folder_reader = record.FolderReader(self.foldername)
+    for leveldb_record in leveldb_folder_reader.GetRecords(
+        use_manifest=use_manifest):
+      try:
+        yield IndexedDBRecord.FromLevelDBRecord(
+            leveldb_record)
+      except(
+          errors.ParserError,
+          errors.DecoderError,
+          NotImplementedError) as err:
+        print((
+            'Error parsing Indexeddb record: '
+            f'{err} at offset {leveldb_record.record.offset} in '
+            f'{leveldb_record.path}'),
+            file=sys.stderr)
+        print(f'Traceback: {traceback.format_exc()}', file=sys.stderr)

dfindexeddb/indexeddb/chromium/v8.py

@@ -152,7 +152,12 @@ class ValueDeserializer:
       _, tag_value = self.decoder.PeekBytes(1)
     except errors.DecoderError:
       return None
-    return definitions.V8SerializationTag(tag_value[0])
+    try:
+      return definitions.V8SerializationTag(tag_value[0])
+    except ValueError as error:
+      raise errors.ParserError(
+          f'Invalid v8 tag value {tag_value} at offset'
+          f' {self.decoder.stream.tell()}') from error
 
   def _ReadTag(self) -> definitions.V8SerializationTag:
     """Returns the next non-padding serialization tag.
@@ -269,7 +274,7 @@ class ValueDeserializer:
             self.version >= 15):
       parsed_object = self.ReadSharedObject()
     elif self.version < 13:
-      self.decoder.stream.seek(-1)
+      self.decoder.stream.seek(-1, os.SEEK_CUR)
       parsed_object = self.ReadHostObject()
     else:
       parsed_object = None
@@ -492,7 +497,7 @@ class ValueDeserializer:
     return value
 
   def _ReadJSRegExp(self) -> RegExp:
-    """Reads a Javscript regular expression from the current position."""
+    """Reads a Javascript regular expression from the current position."""
     next_id = self._GetNextId()
     pattern = self.ReadString()
     _, flags = self.decoder.DecodeUint32Varint()  # TODO: verify flags

dfindexeddb/indexeddb/cli.py

@@ -15,17 +15,16 @@
 """A CLI tool for dfindexeddb."""
 import argparse
 import dataclasses
+import enum
 from datetime import datetime
 import json
 import pathlib
-import sys
-import traceback
 
-from dfindexeddb import errors
 from dfindexeddb import version
-from dfindexeddb.leveldb import record as leveldb_record
+from dfindexeddb.indexeddb.chromium import blink
 from dfindexeddb.indexeddb.chromium import record as chromium_record
 from dfindexeddb.indexeddb.chromium import v8
+from dfindexeddb.indexeddb.safari import record as safari_record
 
 
 _VALID_PRINTABLE_CHARACTERS = (
@@ -57,6 +56,8 @@ class Encoder(json.JSONEncoder):
       return list(o)
     if isinstance(o, v8.RegExp):
       return str(o)
+    if isinstance(o, enum.Enum):
+      return o.name
     return json.JSONEncoder.default(self, o)
 
 
@@ -70,21 +71,34 @@ def _Output(structure, output):
     print(structure)
 
 
-def IndexeddbCommand(args):
-  """The CLI for processing a log/ldb file as indexeddb."""
-  for db_record in leveldb_record.LevelDBRecord.FromDir(args.source):
-    record = db_record.record
-    try:
-      db_record.record = chromium_record.IndexedDBRecord.FromLevelDBRecord(
-          record)
-    except(
-        errors.ParserError,
-        errors.DecoderError,
-        NotImplementedError) as err:
-      print(
-          (f'Error parsing blink value: {err} for {record.__class__.__name__} '
-           f'at offset {record.offset} in {db_record.path}'), file=sys.stderr)
-      print(f'Traceback: {traceback.format_exc()}', file=sys.stderr)
+def BlinkCommand(args):
+  """The CLI for processing a file as a blink value."""
+  with open(args.source, 'rb') as fd:
+    buffer = fd.read()
+    blink_value = blink.V8ScriptValueDecoder.FromBytes(buffer)
+    _Output(blink_value, output=args.output)
+
+
+def DbCommand(args):
+  """The CLI for processing a directory as IndexedDB."""
+  if args.format in ('chrome', 'chromium'):
+    for db_record in chromium_record.FolderReader(
+        args.source).GetRecords(use_manifest=args.use_manifest):
+      _Output(db_record, output=args.output)
+  elif args.format == 'safari':
+    for db_record in safari_record.FileReader(args.source).Records():
+      _Output(db_record, output=args.output)
+
+
+def LdbCommand(args):
+  """The CLI for processing a LevelDB table (.ldb) file as IndexedDB."""
+  for db_record in chromium_record.IndexedDBRecord.FromFile(args.source):
+    _Output(db_record, output=args.output)
+
+
+def LogCommand(args):
+  """The CLI for processing a LevelDB log file as IndexedDB."""
+  for db_record in chromium_record.IndexedDBRecord.FromFile(args.source):
     _Output(db_record, output=args.output)
 
 
@@ -92,12 +106,87 @@ def App():
   """The CLI app entrypoint for dfindexeddb."""
   parser = argparse.ArgumentParser(
       prog='dfindexeddb',
-      description='A cli tool for parsing indexeddb files',
+      description='A cli tool for parsing IndexedDB files',
       epilog=f'Version {version.GetVersion()}')
-  parser.add_argument(
-      '-s', '--source', required=True, type=pathlib.Path,
-      help='The source leveldb folder')
-  parser.add_argument(
+
+  subparsers = parser.add_subparsers()
+
+  parser_blink = subparsers.add_parser(
+      'blink', help='Parse a file as a blink value.')
+  parser_blink.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help=(
+        'The source file.'))
+  parser_blink.add_argument(
+      '-o',
+      '--output',
+      choices=[
+          'json',
+          'jsonl',
+          'repr'],
+      default='json',
+      help='Output format.  Default is json')
+  parser_blink.set_defaults(func=BlinkCommand)
+
+  parser_db = subparsers.add_parser(
+      'db', help='Parse a directory as IndexedDB.')
+  parser_db.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help=(
+        'The source IndexedDB folder (for chrome/chromium) '
+        'or file (for safari).'))
+  parser_db.add_argument(
+      '--format',
+      required=True,
+      choices=['chromium', 'chrome', 'safari'],
+      help='The type of IndexedDB to parse.')
+  parser_db.add_argument(
+      '--use_manifest',
+      action='store_true',
+      help='Use manifest file to determine active/deleted records.')
+  parser_db.add_argument(
+      '-o',
+      '--output',
+      choices=[
+          'json',
+          'jsonl',
+          'repr'],
+      default='json',
+      help='Output format.  Default is json')
+  parser_db.set_defaults(func=DbCommand)
+
+  parser_ldb = subparsers.add_parser(
+      'ldb',
+      help='Parse a ldb file as IndexedDB.')
+  parser_ldb.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source .ldb file.')
+  parser_ldb.add_argument(
+      '-o',
+      '--output',
+      choices=[
+          'json',
+          'jsonl',
+          'repr'],
+      default='json',
+      help='Output format.  Default is json')
+  parser_ldb.set_defaults(func=LdbCommand)
+
+  parser_log = subparsers.add_parser(
+      'log',
+      help='Parse a log file as IndexedDB.')
+  parser_log.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source .log file.')
+  parser_log.add_argument(
       '-o',
       '--output',
       choices=[
@@ -106,7 +195,10 @@ def App():
           'repr'],
       default='json',
       help='Output format.  Default is json')
-  parser.set_defaults(func=IndexeddbCommand)
+  parser_log.set_defaults(func=LogCommand)
 
   args = parser.parse_args()
-  args.func(args)
+  if hasattr(args, 'func'):
+    args.func(args)
+  else:
+    parser.print_help()

dfindexeddb/indexeddb/safari/definitions.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Definitions for Webkit/Safari."""
+from enum import IntEnum
+
+
+CurrentVersion = 0x0000000F  # 15
+TerminatorTag = 0xFFFFFFFF
+StringPoolTag = 0xFFFFFFFE
+NonIndexPropertiesTag = 0xFFFFFFFD
+ImageDataPoolTag = 0xFFFFFFFE
+StringDataIs8BitFlag = 0x80000000
+
+
+SIDBKeyVersion = 0x00
+
+
+class SIDBKeyType(IntEnum):
+  """SIDBKeyType."""
+  MIN = 0x00
+  NUMBER = 0x20
+  DATE = 0x40
+  STRING = 0x60
+  BINARY = 0x80
+  ARRAY = 0xA0
+  MAX = 0xFF
+
+
+class SerializationTag(IntEnum):
+  """Database Metadata key types.  
+  
+  All tags are recorded as a single uint8_t.
+  """
+  ARRAY = 1
+  OBJECT = 2
+  UNDEFINED = 3
+  NULL = 4
+  INT = 5
+  ZERO = 6
+  ONE = 7
+  FALSE = 8
+  TRUE = 9
+  DOUBLE = 10
+  DATE = 11
+  FILE = 12
+  FILE_LIST = 13
+  IMAGE_DATA = 14
+  BLOB = 15
+  STRING = 16
+  EMPTY_STRING = 17
+  REG_EXP = 18
+  OBJECT_REFERENCE = 19
+  MESSAGE_PORT_REFERENCE = 20
+  ARRAY_BUFFER = 21
+  ARRAY_BUFFER_VIEW = 22
+  ARRAY_BUFFER_TRANSFER = 23
+  TRUE_OBJECT = 24
+  FALSE_OBJECT = 25
+  STRING_OBJECT = 26
+  EMPTY_STRING_OBJECT = 27
+  NUMBER_OBJECT = 28
+  SET_OBJECT = 29
+  MAP_OBJECT = 30
+  NON_MAP_PROPERTIES = 31
+  NON_SET_PROPERTIES = 32
+  CRYPTO_KEY = 33
+  SHARED_ARRAY_BUFFER = 34
+  WASM_MODULE = 35
+  DOM_POINT_READONLY = 36
+  DOM_POINT = 37
+  DOM_RECT_READONLY = 38
+  DOM_RECT = 39
+  DOM_MATRIX_READONLY = 40
+  DOM_MATRIX = 41
+  DOM_QUAD = 42
+  IMAGE_BITMAP_TRANSFER = 43
+  RTC_CERTIFICATE = 44
+  IMAGE_BITMAP = 45
+  OFF_SCREEN_CANVAS_TRANSFER = 46
+  BIGINT = 47
+  BIGINT_OBJECT = 48
+  WASM_MEMORY = 49
+  RTC_DATA_CHANNEL_TRANSFER = 50
+  DOM_EXCEPTION = 51
+  WEB_CODECS_ENCODED_VIDEO_CHUNK = 52
+  WEB_CODECS_VIDEO_FRAME = 53
+  RESIZABLE_ARRAY_BUFFER = 54
+  ERROR_INSTANCE = 55
+  IN_MEMORY_OFFSCREEN_CANVAS = 56
+  IN_MEMORY_MESSAGE_PORT = 57
+  WEB_CODECS_ENCODED_AUDIO_CHUNK = 58
+  WEB_CODECS_AUDIO_DATA = 59
+  MEDIA_STREAM_TRACK = 60
+  MEDIA_SOURCE_HANDLE_TRANSFER = 61
+  ERROR = 255
+
+
+class ArrayBufferViewSubtag(IntEnum):
+  """ArrayBufferView sub tags."""
+  DATA_VIEW = 0
+  INT8_ARRAY = 1
+  UINT8_ARRAY = 2
+  UINT8_CLAMPED_ARRAY = 3
+  INT16_ARRAY = 4
+  UINT16_ARRAY = 5
+  INT32_ARRAY = 6
+  UINT32_ARRAY = 7
+  FLOAT32_ARRAY = 8
+  FLOAT64_ARRAY = 9
+  BIG_INT64_ARRAY = 10
+  BIG_UINT64_ARRAY = 11