dfindexeddb 20251109__py3-none-any.whl → 20260205__py3-none-any.whl

dfindexeddb/indexeddb/chromium/definitions.py

@@ -14,11 +14,14 @@
 # limitations under the License.
 """Definitions for IndexedDB."""
 from enum import Enum, IntEnum, IntFlag
+import textwrap
 
 REQUIRES_PROCESSING_SSV_PSEUDO_VERSION = 0x11
 REPLACE_WITH_BLOB = 0x01
 COMPRESSED_WITH_SNAPPY = 0x02
 
+SENTINEL = 0x00
+
 
 class DatabaseMetaDataKeyType(IntEnum):
   """Database Metadata key types."""
@@ -72,6 +75,16 @@ class IDBKeyType(IntEnum):
   BINARY = 6
 
 
+class OrderedIDBKeyType(IntEnum):
+  """Ordered IndexedDB key types."""
+
+  NUMBER = 0x10
+  DATE = 0x20
+  STRING = 0x30
+  BINARY = 0x40
+  ARRAY = 0x50
+
+
 class IndexMetaDataKeyType(IntEnum):
   """IndexedDB metadata key types."""
 
@@ -398,3 +411,82 @@ class SerializedImageOrientation(IntEnum):
   RIGHT_BOTTOM = 6
   LEFT_BOTTOM = 7
   LAST = LEFT_BOTTOM
+
+
+class DatabaseCompressionType(IntEnum):
+  """Database Compression Types."""
+
+  UNCOMPRESSED = 0
+  ZSTD = 1
+  SNAPPY = 2
+
+
+SQL_RECORDS_QUERY_BASE = textwrap.dedent(
+    """
+    SELECT
+        row_id,
+        object_store_id,
+        compression_type,
+        key,
+        value,
+        EXISTS (
+            SELECT 1
+            FROM blob_references
+            WHERE record_row_id = records.row_id
+        ) AS has_blobs
+    FROM records"""
+).strip()
+
+SQL_RECORDS_QUERY = SQL_RECORDS_QUERY_BASE
+
+SQL_RECORDS_BY_ID_QUERY = f"{SQL_RECORDS_QUERY_BASE} WHERE object_store_id = ?"
+
+SQL_RECORDS_BY_NAME_QUERY = textwrap.dedent(
+    f"""
+    {SQL_RECORDS_QUERY_BASE}
+    JOIN object_stores ON records.object_store_id = object_stores.id
+    WHERE object_stores.name = ?"""
+).strip()
+
+SQL_OBJECT_STORES_QUERY = textwrap.dedent(
+    """
+    SELECT
+        id,
+        name,
+        key_path,
+        auto_increment,
+        key_generator_current_number
+    FROM object_stores"""
+).strip()
+
+SQL_BLOB_DATA_QUERY = textwrap.dedent(
+    """
+    SELECT
+        b.row_id,
+        b.object_type,
+        b.mime_type,
+        b.size_bytes,
+        b.file_name,
+        0 AS chunk_index,
+        b.bytes
+    FROM blobs b
+    JOIN blob_references r ON b.row_id = r.blob_row_id
+    WHERE r.record_row_id = ?
+
+    UNION ALL
+
+    SELECT
+        c.blob_row_id AS row_id,
+        b.object_type,
+        b.mime_type,
+        b.size_bytes,
+        b.file_name,
+        c.chunk_index,
+        c.bytes
+    FROM overflow_blob_chunks c
+    JOIN blobs b ON c.blob_row_id = b.row_id
+    JOIN blob_references r ON b.row_id = r.blob_row_id
+    WHERE r.record_row_id = ?
+
+    ORDER BY row_id, chunk_index"""
+).strip()

dfindexeddb/indexeddb/chromium/record.py

@@ -24,6 +24,7 @@ from datetime import datetime
 from typing import (
     Any,
     BinaryIO,
+    ClassVar,
     Generator,
     Optional,
     Tuple,
@@ -36,7 +37,7 @@ from dfindexeddb import errors
 from dfindexeddb.indexeddb.chromium import blink, definitions
 from dfindexeddb.leveldb import record, utils
 
-T = TypeVar("T")
+T = TypeVar("T", bound="BaseIndexedDBKey")
 
 
 @dataclass(frozen=True)
@@ -74,10 +75,9 @@ class KeyPrefix(utils.FromDecoderMixin):
     """
     offset, raw_prefix = decoder.ReadBytes(1)
 
-    database_id_length = (raw_prefix[0] & 0xE0 >> 5) + 1
-    object_store_id_length = (raw_prefix[0] & 0x1C >> 2) + 1
+    database_id_length = ((raw_prefix[0] & 0xE0) >> 5) + 1
+    object_store_id_length = ((raw_prefix[0] & 0x1C) >> 2) + 1
     index_id_length = (raw_prefix[0] & 0x03) + 1
-
     if database_id_length < 1 or database_id_length > 8:
       raise errors.ParserError("Invalid database ID length")
 
@@ -90,7 +90,6 @@ class KeyPrefix(utils.FromDecoderMixin):
     _, database_id = decoder.DecodeInt(database_id_length, signed=False)
     _, object_store_id = decoder.DecodeInt(object_store_id_length, signed=False)
     _, index_id = decoder.DecodeInt(index_id_length, signed=False)
-
     return cls(
         offset=base_offset + offset,
         database_id=database_id,
@@ -217,6 +216,95 @@ class IDBKey(utils.FromDecoderMixin):
     return cls(base_offset + offset, key_type, value)
 
 
+@dataclass(frozen=True)
+class SortableIDBKey(utils.FromDecoderMixin):
+  """A sortable IDBKey.
+
+  Attributes:
+    offset: the offset of the IDBKey.
+    type: the type of the IDBKey.
+    value: the value of the IDBKey.
+  """
+
+  offset: int = field(compare=False)
+  type: definitions.IDBKeyType
+  value: Union[list[Any], bytes, str, float, datetime, None]
+
+  _MAXIMUM_DEPTH = 2000
+
+  @classmethod
+  def FromDecoder(
+      cls,
+      decoder: utils.LevelDBDecoder,
+      base_offset: int = 0,
+  ) -> SortableIDBKey:
+    """Decodes a sortable IDBKey from the current position of a LevelDBDecoder.
+
+    Args:
+      decoder: the LevelDBDecoder.
+      base_offset: the base offset.
+
+    Returns:
+      The decoded SortableIDBKey.
+
+    Raises:
+      ParserError: on invalid key type or truncated data.
+      RecursionError: if maximum depth encountered.
+    """
+
+    def RecursiveParse(depth: int) -> Tuple[int, definitions.IDBKeyType, Any]:
+      """Recursively parses sortable IDBKeys.
+
+      Args:
+        depth: the current recursion depth.
+
+      Returns:
+        A tuple of the offset, the key type and the key value (where the value
+          can be bytes, str, float, datetime or a list of these types).
+
+      Raises:
+        ParserError: on invalid IDBKeyType or invalid array length during
+          parsing.
+        RecursionError: if maximum depth encountered during parsing.
+      """
+      if depth == cls._MAXIMUM_DEPTH:
+        raise RecursionError("Maximum recursion depth encountered")
+
+      value: Any = None
+      offset, ordered_type = decoder.DecodeUint8()
+      if ordered_type == definitions.OrderedIDBKeyType.NUMBER:
+        _, value = decoder.DecodeSortableDouble()
+        return offset, definitions.IDBKeyType.NUMBER, value
+      if ordered_type == definitions.OrderedIDBKeyType.DATE:
+        _, raw_date = decoder.DecodeSortableDouble()
+        return (
+            offset,
+            definitions.IDBKeyType.DATE,
+            datetime.utcfromtimestamp(raw_date / 1000.0),
+        )
+      if ordered_type == definitions.OrderedIDBKeyType.STRING:
+        _, value = decoder.DecodeSortableString()
+        return offset, definitions.IDBKeyType.STRING, value
+      if ordered_type == definitions.OrderedIDBKeyType.BINARY:
+        _, value = decoder.DecodeSortableBinary()
+        return offset, definitions.IDBKeyType.BINARY, value
+      if ordered_type == definitions.OrderedIDBKeyType.ARRAY:
+        value = []
+        while True:
+          _, next_byte = decoder.PeekBytes(1)
+          if next_byte[0] == definitions.SENTINEL:
+            decoder.ReadBytes(1)
+            break
+          _, _, item = RecursiveParse(depth + 1)
+          value.append(item)
+        return offset, definitions.IDBKeyType.ARRAY, value
+
+      raise errors.ParserError(f"Unknown ordered key type {ordered_type}")
+
+    offset, key_type, value = RecursiveParse(0)
+    return cls(base_offset + offset, key_type, value)
+
+
 @dataclass
 class IDBKeyPath(utils.FromDecoderMixin):
   """An IDBKeyPath.
@@ -383,6 +471,9 @@ class BaseIndexedDBKey:
     Args:
       decoder: the stream decoder
 
+    Returns:
+      The decoded value.
+
     Raises:
       NotImplementedError.
     """
@@ -392,7 +483,7 @@ class BaseIndexedDBKey:
     """Parses the value from raw bytes.
 
     Args:
-      value_data: the raw value bytes.
+      value_data: the raw value data.
 
     Returns:
       The parsed value.
@@ -408,13 +499,15 @@ class BaseIndexedDBKey:
       decoder: utils.LevelDBDecoder,
       key_prefix: KeyPrefix,
       base_offset: int = 0,
-  ) -> T:  # pylint: disable=unused-variable
-    """Decodes the remaining key data from the current decoder position.
+  ) -> T:
+    """Parses the key from the current position of the LevelDBDecoder.
+
+    To be implemented by subclasses.
 
     Args:
       decoder: the stream decoder.
-      key_prefix: the decoded key_prefix.
-      base_offset: the base offset.
+      key_prefix: the key prefix.
+      base_offset: the base offset of the key.
 
     Returns:
       The decoded key.
@@ -437,7 +530,7 @@ class BaseIndexedDBKey:
     """
     decoder = utils.LevelDBDecoder(stream)
     key_prefix = KeyPrefix.FromDecoder(decoder, base_offset=base_offset)
-    return cls.FromDecoder(  # type: ignore[no-any-return,attr-defined]
+    return cls.FromDecoder(
         decoder=decoder, key_prefix=key_prefix, base_offset=base_offset
     )
 
@@ -453,9 +546,7 @@ class BaseIndexedDBKey:
       The decoded key.
     """
     stream = io.BytesIO(raw_data)
-    return cls.FromStream(  # type: ignore[no-any-return,attr-defined]
-        stream=stream, base_offset=base_offset
-    )
+    return cls.FromStream(stream=stream, base_offset=base_offset)
 
 
 @dataclass
@@ -716,7 +807,11 @@ class GlobalMetaDataKey(BaseIndexedDBKey):
   """A GlobalMetaDataKey parser."""
 
   # pylint: disable=line-too-long
-  METADATA_TYPE_TO_CLASS = {
+  METADATA_TYPE_TO_CLASS: ClassVar[
+      dict[  # pylint: disable=invalid-name
+          definitions.GlobalMetadataKeyType, type[BaseIndexedDBKey]
+      ]
+  ] = {
       definitions.GlobalMetadataKeyType.ACTIVE_BLOB_JOURNAL: ActiveBlobJournalKey,
       definitions.GlobalMetadataKeyType.DATA_VERSION: DataVersionKey,
       definitions.GlobalMetadataKeyType.DATABASE_FREE_LIST: DatabaseFreeListKey,
@@ -743,18 +838,7 @@ class GlobalMetaDataKey(BaseIndexedDBKey):
       decoder: utils.LevelDBDecoder,
       key_prefix: KeyPrefix,
       base_offset: int = 0,
-  ) -> Union[
-      ActiveBlobJournalKey,
-      DataVersionKey,
-      DatabaseFreeListKey,
-      DatabaseNameKey,
-      EarliestSweepKey,
-      EarliestCompactionTimeKey,
-      MaxDatabaseIdKey,
-      RecoveryBlobJournalKey,
-      SchemaVersionKey,
-      ScopesPrefixKey,
-  ]:
+  ) -> BaseIndexedDBKey:
     """Decodes the global metadata key.
 
     Raises:
@@ -766,9 +850,7 @@ class GlobalMetaDataKey(BaseIndexedDBKey):
     key_class = cls.METADATA_TYPE_TO_CLASS.get(metadata_type)
     if not key_class:
       raise errors.ParserError("Unknown metadata key type")
-    return key_class.FromDecoder(  # type: ignore[attr-defined,no-any-return]
-        decoder, key_prefix, base_offset
-    )
+    return key_class.FromDecoder(decoder, key_prefix, base_offset)
 
 
 @dataclass
@@ -1284,7 +1366,11 @@ class IndexedDbKey(BaseIndexedDBKey):
   A factory class for parsing IndexedDB keys.
   """
 
-  METADATA_TYPE_TO_CLASS = {
+  METADATA_TYPE_TO_CLASS: ClassVar[
+      dict[  # pylint: disable=invalid-name
+          definitions.KeyPrefixType, Optional[type[BaseIndexedDBKey]]
+      ]
+  ] = {
       definitions.KeyPrefixType.BLOB_ENTRY: BlobEntryKey,
       definitions.KeyPrefixType.DATABASE_METADATA: DatabaseMetaDataKey,
       definitions.KeyPrefixType.EXISTS_ENTRY: ExistsEntryKey,
@@ -1307,14 +1393,7 @@ class IndexedDbKey(BaseIndexedDBKey):
       decoder: utils.LevelDBDecoder,
       key_prefix: KeyPrefix,
       base_offset: int = 0,
-  ) -> Union[
-      BlobEntryKey,
-      DatabaseMetaDataKey,
-      ExistsEntryKey,
-      GlobalMetaDataKey,
-      IndexDataKey,
-      ObjectStoreDataKey,
-  ]:
+  ) -> BaseIndexedDBKey:
     """Decodes the IndexedDB key."""
     key_type = key_prefix.GetKeyPrefixType()
     key_class = cls.METADATA_TYPE_TO_CLASS.get(key_type)
@@ -1322,7 +1401,7 @@ class IndexedDbKey(BaseIndexedDBKey):
       raise errors.ParserError("Unknown KeyPrefixType")
     return key_class.FromDecoder(
         decoder=decoder,
-        key_prefix=key_prefix,  # type: ignore[return-value]
+        key_prefix=key_prefix,
         base_offset=base_offset,
     )
 
@@ -1481,8 +1560,8 @@ class IndexedDBExternalObject(utils.FromDecoderMixin):
 
 
 @dataclass
-class IndexedDBRecord:
-  """An IndexedDB Record.
+class ChromiumIndexedDBRecord:
+  """An IndexedDB Record parsed from LevelDB.
 
   Attributes:
     path: the source file path
@@ -1498,7 +1577,7 @@ class IndexedDBRecord:
     object_store_id: the object store ID.
     database_name: the name of the database, if available.
     object_store_name: the name of the object store, if available.
-    blob: the blob contents, if available.
+    blobs: the list of blob paths and contents or error message, if available.
     raw_key: the raw key, if available.
     raw_value: the raw value, if available.
   """
@@ -1515,15 +1594,19 @@ class IndexedDBRecord:
   object_store_id: int
   database_name: Optional[str] = None
   object_store_name: Optional[str] = None
-  blob: Optional[bytes] = None
+  blobs: Optional[list[tuple[str, Optional[Any]]]] = None
   raw_key: Optional[bytes] = None
   raw_value: Optional[bytes] = None
 
   @classmethod
   def FromLevelDBRecord(
-      cls, db_record: record.LevelDBRecord, parse_value: bool = True
-  ) -> IndexedDBRecord:
-    """Returns an IndexedDBRecord from a ParsedInternalKey."""
+      cls,
+      db_record: record.LevelDBRecord,
+      parse_value: bool = True,
+      include_raw_data: bool = False,
+      blob_folder_reader: Optional[BlobFolderReader] = None,
+  ) -> ChromiumIndexedDBRecord:
+    """Returns an ChromiumIndexedDBRecord from a ParsedInternalKey."""
     idb_key = IndexedDbKey.FromBytes(
         db_record.record.key, base_offset=db_record.record.offset
     )
@@ -1533,14 +1616,30 @@ class IndexedDBRecord:
     else:
       idb_value = None
 
+    blobs = []
+    if isinstance(idb_value, IndexedDBExternalObject) and blob_folder_reader:
+      for (
+          blob_path_or_error,
+          blob_data,
+      ) in blob_folder_reader.ReadBlobsFromExternalObjectEntries(
+          idb_key.key_prefix.database_id, idb_value.entries
+      ):
+        if blob_data:
+          blob = blink.V8ScriptValueDecoder.FromBytes(blob_data)
+        else:
+          blob = None
+        blobs.append((blob_path_or_error, blob))
+
     return cls(
         path=db_record.path,
         offset=db_record.record.offset,
         key=idb_key,
         value=idb_value,
-        sequence_number=db_record.record.sequence_number
-        if hasattr(db_record.record, "sequence_number")
-        else None,
+        sequence_number=(
+            db_record.record.sequence_number
+            if hasattr(db_record.record, "sequence_number")
+            else None
+        ),
         type=db_record.record.record_type,
         level=db_record.level,
         recovered=db_record.recovered,
@@ -1548,19 +1647,28 @@ class IndexedDBRecord:
         object_store_id=idb_key.key_prefix.object_store_id,
         database_name=None,
         object_store_name=None,
-        blob=None,
-        raw_key=db_record.record.key,
-        raw_value=db_record.record.value,
+        blobs=blobs,
+        raw_key=db_record.record.key if include_raw_data else None,
+        raw_value=db_record.record.value if include_raw_data else None,
     )
 
   @classmethod
   def FromFile(
-      cls, file_path: pathlib.Path, parse_value: bool = True
-  ) -> Generator[IndexedDBRecord, None, None]:
-    """Yields IndexedDBRecords from a file."""
+      cls,
+      file_path: pathlib.Path,
+      parse_value: bool = True,
+      include_raw_data: bool = False,
+      blob_folder_reader: Optional[BlobFolderReader] = None,
+  ) -> Generator[ChromiumIndexedDBRecord, None, None]:
+    """Yields ChromiumIndexedDBRecord from a file."""
     for db_record in record.LevelDBRecord.FromFile(file_path):
       try:
-        yield cls.FromLevelDBRecord(db_record, parse_value=parse_value)
+        yield cls.FromLevelDBRecord(
+            db_record,
+            parse_value=parse_value,
+            include_raw_data=include_raw_data,
+            blob_folder_reader=blob_folder_reader,
+        )
       except (
           errors.ParserError,
           errors.DecoderError,
@@ -1577,6 +1685,88 @@ class IndexedDBRecord:
         print(f"Traceback: {traceback.format_exc()}", file=sys.stderr)
 
 
+class BlobFolderReader:
+  """A blob folder reader for Chrome/Chromium.
+
+  Attributes:
+    folder_name (str): the source blob folder.
+  """
+
+  def __init__(self, folder_name: pathlib.Path):
+    """Initializes the BlobFolderReader.
+
+    Args:
+      folder_name: the source blob folder.
+
+    Raises:
+      ValueError: if folder_name is None or not a directory.
+    """
+    if not folder_name or not folder_name.is_dir():
+      raise ValueError(f"{folder_name} is None or not a directory")
+    self.folder_name = folder_name.absolute()
+
+  def ReadBlob(self, database_id: int, blob_id: int) -> tuple[str, bytes]:
+    """Reads a blob from the blob folder.
+
+    Args:
+      database_id: the database id of the blob to read.
+      blob_id: the blob id to read.
+
+    Returns:
+      A tuple of the blob path and contents.
+
+    Raises:
+      FileNotFoundError: if the database directory or blob folder or blob not
+          found.
+    """
+    directory_path = self.folder_name / f"{database_id:x}"
+    if not directory_path.exists():
+      raise FileNotFoundError(f"Database directory not found: {directory_path}")
+
+    blob_folder = directory_path / f"{(blob_id & 0xff00) >> 8:02x}"
+    if not blob_folder.exists():
+      raise FileNotFoundError(f"Blob folder not found: {blob_folder}")
+
+    blob_path = blob_folder / f"{blob_id:x}"
+    if not blob_path.exists():
+      raise FileNotFoundError(f"Blob ({blob_id}) not found: {blob_path}")
+
+    with open(blob_path, "rb") as f:
+      return str(blob_path), f.read()
+
+  def ReadBlobsFromExternalObjectEntries(
+      self, database_id: int, entries: list[ExternalObjectEntry]
+  ) -> Generator[tuple[str, Optional[bytes]], None, None]:
+    """Reads blobs from the blob folder.
+
+    Args:
+      database_id: the database id.
+      entries: the external object entries.
+
+    Yields:
+      A tuple of blob path and contents or if the blob is not found, an error
+      message and None.
+    """
+    for entry in entries:
+      if (
+          entry.object_type
+          in (
+              definitions.ExternalObjectType.BLOB,
+              definitions.ExternalObjectType.FILE,
+          )
+          and entry.blob_number is not None
+      ):
+        try:
+          yield self.ReadBlob(database_id, entry.blob_number)
+        except FileNotFoundError as err:
+          error_message = (
+              f"Blob not found for ExternalObjectEntry at offset {entry.offset}"
+              f": {err}"
+          )
+          print(error_message, file=sys.stderr)
+          yield error_message, None
+
+
 class FolderReader:
   """A IndexedDB folder reader for Chrome/Chromium.
 
@@ -1595,15 +1785,25 @@ class FolderReader:
     """
     if not folder_name or not folder_name.is_dir():
       raise ValueError(f"{folder_name} is None or not a directory")
-    self.folder_name = folder_name
+    self.folder_name = folder_name.absolute()
+
+    # Locate the correponding blob folder. The folder_name should be
+    # <origin>.leveldb and the blob folder should be <origin>.blob
+    if str(self.folder_name).endswith(".leveldb"):
+      self.blob_folder_reader = BlobFolderReader(
+          pathlib.Path(str(self.folder_name).replace(".leveldb", ".blob"))
+      )
+    else:
+      self.blob_folder_reader = None  # type: ignore[assignment]
 
   def GetRecords(
       self,
       use_manifest: bool = False,
       use_sequence_number: bool = False,
       parse_value: bool = True,
-  ) -> Generator[IndexedDBRecord, None, None]:
-    """Yield LevelDBRecords.
+      include_raw_data: bool = False,
+  ) -> Generator[ChromiumIndexedDBRecord, None, None]:
+    """Yields ChromiumIndexedDBRecord.
 
     Args:
       use_manifest: True to use the current manifest in the folder as a means to
@@ -1613,15 +1813,18 @@ class FolderReader:
       parse_value: True to parse values.
 
     Yields:
-      IndexedDBRecord.
+      ChromiumIndexedDBRecord.
     """
     leveldb_folder_reader = record.FolderReader(self.folder_name)
     for leveldb_record in leveldb_folder_reader.GetRecords(
         use_manifest=use_manifest, use_sequence_number=use_sequence_number
     ):
       try:
-        yield IndexedDBRecord.FromLevelDBRecord(
-            leveldb_record, parse_value=parse_value
+        yield ChromiumIndexedDBRecord.FromLevelDBRecord(
+            leveldb_record,
+            parse_value=parse_value,
+            include_raw_data=include_raw_data,
+            blob_folder_reader=self.blob_folder_reader,
         )
       except (
           errors.ParserError,