linear-mcp-fast 0.1.0__py3-none-any.whl

ccl_chromium_reader/ccl_chromium_indexeddb.py

@@ -0,0 +1,1060 @@
+"""
+Copyright 2020-2024, CCL Forensics
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+import sys
+import struct
+import os
+import pathlib
+import io
+import enum
+import datetime
+import dataclasses
+import types
+import typing
+
+from .storage_formats import ccl_leveldb
+from .serialization_formats import ccl_blink_value_deserializer, ccl_v8_value_deserializer
+
+__version__ = "0.19"
+__description__ = "Module for reading Chromium IndexedDB LevelDB databases."
+__contact__ = "Alex Caithness"
+
+
+# TODO: need to go through and ensure that we have endianness right in all cases
+#  (it should sit behind a switch for integers, fixed for most other stuff)
+
+
+def _read_le_varint(stream: typing.BinaryIO, *, is_google_32bit=False) -> typing.Optional[tuple[int, bytes]]:
+    # this only outputs unsigned
+    i = 0
+    result = 0
+    underlying_bytes = []
+    limit = 5 if is_google_32bit else 10
+    while i < limit:
+        raw = stream.read(1)
+        if len(raw) < 1:
+            return None
+        tmp, = raw
+        underlying_bytes.append(tmp)
+        result |= ((tmp & 0x7f) << (i * 7))
+
+        if (tmp & 0x80) == 0:
+            break
+        i += 1
+    return result, bytes(underlying_bytes)
+
+
+def read_le_varint(stream: typing.BinaryIO, *, is_google_32bit=False) -> typing.Optional[int]:
+    x = _read_le_varint(stream, is_google_32bit=is_google_32bit)
+    if x is None:
+        return None
+    else:
+        return x[0]
+
+
+def _le_varint_from_bytes(data: bytes) -> typing.Optional[tuple[int, bytes]]:
+    with io.BytesIO(data) as buff:
+        return _read_le_varint(buff)
+
+
+def le_varint_from_bytes(data: bytes) -> typing.Optional[int]:
+    with io.BytesIO(data) as buff:
+        return read_le_varint(buff)
+
+
+def decode_truncated_int(data: bytes) -> int:
+    # See: /content/browser/indexed_db/indexed_db_leveldb_coding.h EncodeInt()
+    # "// Unlike EncodeVarInt, this is a 'dumb' implementation of a variable int
+    # // encoder. It writes, little-endian', until there are no more '1' bits in the
+    # // number. The Decoder must know how to calculate the size of the encoded int,
+    # // typically by having this reside at the end of the value or key."
+    if len(data) == 0:
+        raise ValueError("No data to decode")
+    result = 0
+    for i, b in enumerate(data):
+        result |= (b << (i * 8))
+    return result
+
+
+class IdbKeyType(enum.IntEnum):
+    Null = 0
+    String = 1
+    Date = 2
+    Number = 3
+    Array = 4
+    MinKey = 5
+    Binary = 6
+
+
+class IdbKey:
+    # See: https://github.com/chromium/chromium/blob/master/content/browser/indexed_db/indexed_db_leveldb_coding.cc
+    def __init__(self, buffer: bytes):
+        self.raw_key = buffer
+        self.key_type = IdbKeyType(buffer[0])
+        raw_key = buffer[1:]
+
+        if self.key_type == IdbKeyType.Null:
+            self.value = None
+            self._raw_length = 1
+        elif self.key_type == IdbKeyType.String:
+            str_len, varint_raw = _le_varint_from_bytes(raw_key)
+            self.value = raw_key[len(varint_raw):len(varint_raw) + str_len * 2].decode("utf-16-be")
+            self._raw_length = 1 + len(varint_raw) + str_len * 2
+        elif self.key_type == IdbKeyType.Date:
+            ts, = struct.unpack("<d", raw_key[0:8])
+            self.value = datetime.datetime(1970, 1, 1) + datetime.timedelta(milliseconds=ts)
+            self._raw_length = 9
+        elif self.key_type == IdbKeyType.Number:
+            self.value = struct.unpack("<d", raw_key[0:8])[0]
+            self._raw_length = 9
+        elif self.key_type == IdbKeyType.Array:
+            array_count, varint_raw = _le_varint_from_bytes(raw_key)
+            raw_key = raw_key[len(varint_raw):]
+            self.value = []
+            self._raw_length = 1 + len(varint_raw)
+            for i in range(array_count):
+                key = IdbKey(raw_key)
+                raw_key = raw_key[key._raw_length:]
+                self._raw_length += key._raw_length
+                self.value.append(key)
+            self.value = tuple(self.value)
+        elif self.key_type == IdbKeyType.MinKey:
+            # TODO: not sure what this actually implies, the code doesn't store a value
+            self.value = None
+            self._raw_length = 1
+            raise NotImplementedError()
+        elif self.key_type == IdbKeyType.Binary:
+            bin_len, varint_raw = _le_varint_from_bytes(raw_key)
+            self.value = raw_key[len(varint_raw):len(varint_raw) + bin_len]
+            self._raw_length = 1 + len(varint_raw) + bin_len
+        else:
+            raise ValueError()  # Shouldn't happen
+
+        # trim the raw_key in case this is an inner key:
+        self.raw_key = self.raw_key[0: self._raw_length]
+
+    def __repr__(self):
+        return f"<IdbKey {self.value}>"
+
+    def __str__(self):
+        return self.__repr__()
+
+    def __eq__(self, other):
+        if not isinstance(other, IdbKey):
+            raise NotImplementedError()
+        return self.raw_key == other.raw_key
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __hash__(self):
+        return self.raw_key.__hash__()
+
+
+class IndexedDBExternalObjectType(enum.IntEnum):
+    # see: https://github.com/chromium/chromium/blob/master/content/browser/indexed_db/indexed_db_external_object.h
+    Blob = 0
+    File = 1
+    NativeFileSystemHandle = 2
+
+
+class IndexedDBExternalObject:
+    # see: https://github.com/chromium/chromium/blob/master/content/browser/indexed_db/indexed_db_backing_store.cc
+    # for encoding.
+
+    def __init__(self, object_type: IndexedDBExternalObjectType, blob_number: typing.Optional[int],
+                 mime_type: typing.Optional[str], size: typing.Optional[int],
+                 file_name: typing.Optional[str], last_modified: typing.Optional[datetime.datetime],
+                 native_file_token: typing.Optional):
+        self.object_type = object_type
+        self.blob_number = blob_number
+        self.mime_type = mime_type
+        self.size = size
+        self.file_name = file_name
+        self.last_modified = last_modified
+        self.native_file_token = native_file_token
+
+    @classmethod
+    def from_stream(cls, stream: typing.BinaryIO):
+        blob_type = IndexedDBExternalObjectType(stream.read(1)[0])
+        if blob_type in (IndexedDBExternalObjectType.Blob, IndexedDBExternalObjectType.File):
+            blob_number = read_le_varint(stream)
+            mime_type_length = read_le_varint(stream)
+            mime_type = stream.read(mime_type_length * 2).decode("utf-16-be")
+            data_size = read_le_varint(stream)
+
+            if blob_type == IndexedDBExternalObjectType.File:
+                file_name_length = read_le_varint(stream)
+                file_name = stream.read(file_name_length * 2).decode("utf-16-be")
+                x, x_raw = _read_le_varint(stream)
+                last_modified_td = datetime.timedelta(microseconds=x)
+                last_modified = datetime.datetime(1601, 1, 1) + last_modified_td
+                return cls(blob_type, blob_number, mime_type, data_size, file_name,
+                           last_modified, None)
+            else:
+                return cls(blob_type, blob_number, mime_type, data_size, None, None, None)
+        else:
+            raise NotImplementedError()
+
+
+@dataclasses.dataclass(frozen=True)
+class DatabaseId:
+    dbid_no: int
+    origin: str
+    name: str
+
+
+class GlobalMetadata:
+    def __init__(self, raw_meta_dict: dict):
+        # TODO: more of these meta types if required
+        self.backing_store_schema_version = None
+        if raw_schema_version := raw_meta_dict.get("\x00\x00\x00\x00\x00"):
+            self.backing_store_schema_version = le_varint_from_bytes(raw_schema_version)
+
+        self.max_allocated_db_id = None
+        if raw_max_db_id := raw_meta_dict.get("\x00\x00\x00\x00\x01"):
+            self.max_allocated_db_id = le_varint_from_bytes(raw_max_db_id)
+
+        database_ids_raw = (raw_meta_dict[x] for x in raw_meta_dict
+                            if x.startswith(b"\x00\x00\x00\x00\xc9"))
+
+        dbids = []
+        for dbid_rec in database_ids_raw:
+            with io.BytesIO(dbid_rec.key[5:]) as buff:
+                origin_length = read_le_varint(buff)
+                origin = buff.read(origin_length * 2).decode("utf-16-be")
+                db_name_length = read_le_varint(buff)
+                db_name = buff.read(db_name_length * 2).decode("utf-16-be")
+
+            db_id_no = decode_truncated_int(dbid_rec.value)
+
+            dbids.append(DatabaseId(db_id_no, origin, db_name))
+
+        self._db_ids = tuple(dbids)
+        self._db_ids_lookup = types.MappingProxyType({x.dbid_no: x for x in self._db_ids})
+
+    @property
+    def db_ids(self) -> tuple[DatabaseId, ...]:
+        return self._db_ids
+
+    @property
+    def db_ids_lookup(self) -> dict[int: DatabaseId]:
+        return self._db_ids_lookup
+
+
+class DatabaseMetadataType(enum.IntEnum):
+    OriginName = 0  # String
+    DatabaseName = 1  # String
+    IdbVersionString = 2  # String (and obsolete)
+    MaximumObjectStoreId = 3  # Int
+    IdbVersion = 4  # Varint
+    BlobNumberGeneratorCurrentNumber = 5  # Varint
+
+
+class DatabaseMetadata:
+    def __init__(self, raw_meta: dict):
+        self._metas = types.MappingProxyType(raw_meta)
+
+    def get_meta(self, db_id: int, meta_type: DatabaseMetadataType) -> typing.Optional[typing.Union[str, int]]:
+        record = self._metas.get((db_id, meta_type))
+        if not record:
+            return None
+
+        if meta_type == DatabaseMetadataType.MaximumObjectStoreId:
+            return decode_truncated_int(record.value)
+
+        # TODO
+        raise NotImplementedError()
+
+
+class ObjectStoreMetadataType(enum.IntEnum):
+    StoreName = 0  # String
+    KeyPath = 1  # IDBKeyPath
+    AutoIncrementFlag = 2  # Bool
+    IsEvictable = 3  # Bool (and obsolete apparently)
+    LastVersionNumber = 4  # Int
+    MaximumAllocatedIndexId = 5  # Int
+    HasKeyPathFlag = 6  # Bool (and obsolete apparently)
+    KeygeneratorCurrentNumber = 7  # Int
+
+
+class ObjectStoreMetadata:
+    # All metadata fields are prefaced by a 0x00 byte
+    def __init__(self, raw_meta: dict):
+        self._metas = types.MappingProxyType(raw_meta)
+
+    def get_meta(self, db_id: int, obj_store_id: int, meta_type: ObjectStoreMetadataType):
+        record = self._metas.get((db_id, obj_store_id, meta_type))
+        if not record:
+            return None
+
+        if meta_type == ObjectStoreMetadataType.StoreName:
+            return record.value.decode("utf-16-be")
+
+        # TODO
+        raise NotImplementedError()
+
+
+@dataclasses.dataclass(frozen=True)
+class BlinkTrailer:
+    # third_party/blink/renderer/bindings/core/v8/serialization/trailer_reader.h
+    offset: int
+    length: int
+
+    TRAILER_SIZE: typing.ClassVar[int] = 13
+    MIN_WIRE_FORMAT_VERSION_FOR_TRAILER: typing.ClassVar[int] = 21
+
+    @classmethod
+    def from_buffer(cls, buffer, trailer_offset: int):
+        tag, offset, length = struct.unpack(">cQI", buffer[trailer_offset: trailer_offset + BlinkTrailer.TRAILER_SIZE])
+        if tag != ccl_blink_value_deserializer.Constants.tag_kTrailerOffsetTag:
+            raise ValueError(
+                f"Trailer doesn't start with kTrailerOffsetTag "
+                f"(expected: 0x{ccl_blink_value_deserializer.Constants.tag_kTrailerOffsetTag.hex()}; "
+                f"got: 0x{tag.hex()}")
+
+        return BlinkTrailer(offset, length)
+
+
+@dataclasses.dataclass(frozen=True)
+class IndexedDbRecord:
+    owner: "IndexedDb"
+    db_id: int
+    obj_store_id: int
+    key: IdbKey
+    value: typing.Any
+    is_live: bool
+    ldb_seq_no: int
+    origin_file: os.PathLike
+    external_value_path: typing.Optional[str] = None
+
+    def resolve_blob_index(self, blob_index: ccl_blink_value_deserializer.BlobIndex) -> IndexedDBExternalObject:
+        """Resolve a ccl_blink_value_deserializer.BlobIndex to its IndexedDBExternalObject
+         to get metadata (file name, timestamps, etc)"""
+        return self.owner.get_blob_info(self.db_id, self.obj_store_id, self.key.raw_key, blob_index.index_id)
+
+    def get_blob_stream(self, blob_index: ccl_blink_value_deserializer.BlobIndex) -> typing.BinaryIO:
+        """Resolve a ccl_blink_value_deserializer.BlobIndex to a stream of its content"""
+        return self.owner.get_blob(self.db_id, self.obj_store_id, self.key.raw_key, blob_index.index_id)
+
+    @property
+    def database_name(self):
+        return self.owner.global_metadata.db_ids_lookup[self.db_id].name
+
+    @property
+    def database_origin(self):
+        return self.owner.global_metadata.db_ids_lookup[self.db_id].origin
+
+    @property
+    def object_store_name(self):
+        return self.owner.get_object_store_metadata(self.db_id, self.obj_store_id, ObjectStoreMetadataType.StoreName)
+
+    @property
+    def record_location(self) -> str:
+        return f"File: {pathlib.Path(*pathlib.Path(self.origin_file).parts[-2:])} Seq: {self.ldb_seq_no}"
+
+
+class IndexedDb:
+    # This will be informative for a lot of the data below:
+    # https://github.com/chromium/chromium/blob/master/content/browser/indexed_db/docs/leveldb_coding_scheme.md
+
+    # Of note, the first byte of the key defines the length of the db_id, obj_store_id and index_id in bytes:
+    # 0b xxxyyyzz (x = db_id size - 1, y = obj_store size - 1, z = index_id - 1)
+
+    def __init__(self, leveldb_dir: os.PathLike, leveldb_blob_dir: os.PathLike = None):
+        self._db = ccl_leveldb.RawLevelDb(leveldb_dir)
+        self._blob_dir = leveldb_blob_dir
+        self.global_metadata: typing.Optional[GlobalMetadata] = None
+        self.database_metadata: typing.Optional[DatabaseMetadata] = None
+        self.object_store_meta: typing.Optional[ObjectStoreMetadata] = None
+        self._cache_records()
+        self._fetch_meta_data()
+        self._blob_lookup_cache = {}
+
+    def _cache_records(self):
+        self._fetched_records = []
+        # Fetch the records only once
+        for record in self._db.iterate_records_raw():
+            self._fetched_records.append(record)
+
+    def _fetch_meta_data(self):
+        global_metadata_raw = self._get_raw_global_metadata()
+        self.global_metadata = GlobalMetadata(global_metadata_raw)
+        database_metadata_raw = self._get_raw_database_metadata()
+        self.database_metadata = DatabaseMetadata(database_metadata_raw)
+        objectstore_metadata_raw = self._get_raw_object_store_metadata()
+        self.object_store_meta = ObjectStoreMetadata(objectstore_metadata_raw)
+
+    @staticmethod
+    def make_prefix(
+            db_id: int, obj_store_id: int, index_id: int, end: typing.Optional[typing.Sequence[int]]=None) -> bytes:
+        if end is None:
+            end = []
+
+        def count_bytes(val):
+            if val == 0:
+                return 1
+            i = 0
+            while val > 0:
+                i += 1
+                val = val >> 8
+            return i
+
+        def yield_le_bytes(val):
+            if val == 0:
+                yield 0
+            if val < 0:
+                raise ValueError
+            while val > 0:
+                yield val & 0xff
+                val = val >> 8
+
+        db_id_size = count_bytes(db_id)
+        obj_store_id_size = count_bytes(obj_store_id)
+        index_id_size = count_bytes(index_id)
+
+        if db_id_size > 8 or obj_store_id_size > 8 or index_id_size > 4:
+            raise ValueError("id sizes are too big")
+
+        byte_one = ((db_id_size - 1) << 5) | ((obj_store_id_size - 1) << 2) | (index_id_size - 1)
+        # print([byte_one, *yield_le_bytes(db_id), *yield_le_bytes(obj_store_id), *yield_le_bytes(index_id), *end])
+        return bytes([byte_one, *yield_le_bytes(db_id), *yield_le_bytes(obj_store_id), *yield_le_bytes(index_id), *end])
+
+    @staticmethod
+    def read_prefix(stream: typing.BinaryIO) -> tuple[int, int, int, int]:
+        """
+        :param stream: file-like to read the prefix from
+        :return: a tuple of db_id, object_store_id, index_id, length of the prefix
+        """
+        lengths_bytes = stream.read(1)
+        if not lengths_bytes:
+            raise ValueError("Couldn't get enough data when reading prefix length")
+        lengths = lengths_bytes[0]
+        db_id_size = ((lengths >> 5) & 0x07) + 1
+        object_store_size = ((lengths >> 2) & 0x07) + 1
+        index_size = (lengths & 0x03) + 1
+
+        db_id_raw = stream.read(db_id_size)
+        object_store_raw = stream.read(object_store_size)
+        index_raw = stream.read(index_size)
+
+        if (len(db_id_raw) != db_id_size or
+                len(object_store_raw) != object_store_size or
+                len(index_raw) != index_size):
+            raise ValueError("Couldn't read enough bytes for the prefix")
+
+        db_id = int.from_bytes(db_id_raw, "little")
+        object_store_id = int.from_bytes(object_store_raw, "little")
+        index_id = int.from_bytes(index_raw, "little")
+
+        return db_id, object_store_id, index_id, (db_id_size + object_store_size + index_size + 1)
+
+    def get_database_metadata(self, db_id: int, meta_type: DatabaseMetadataType):
+        return self.database_metadata.get_meta(db_id, meta_type)
+
+    def get_object_store_metadata(self, db_id: int, obj_store_id: int, meta_type: ObjectStoreMetadataType):
+        return self.object_store_meta.get_meta(db_id, obj_store_id, meta_type)
+
+    def _get_raw_global_metadata(self, live_only=True) -> typing.Dict[bytes, ccl_leveldb.Record]:
+        # Global metadata always has the prefix 0 0 0 0
+        if not live_only:
+            raise NotImplementedError("Deleted metadata not implemented yet")
+        meta = {}
+        for record in reversed(self._fetched_records):
+            if record.key.startswith(b"\x00\x00\x00\x00") and record.state == ccl_leveldb.KeyState.Live:
+                # we only want live keys and the newest version thereof (highest seq)
+                if record.key not in meta or meta[record.key].seq < record.seq:
+                    meta[record.key] = record
+
+        return meta
+
+    def _get_raw_database_metadata(self, live_only=True):
+        if not live_only:
+            raise NotImplementedError("Deleted metadata not implemented yet")
+
+        db_meta = {}
+
+        for db_id in self.global_metadata.db_ids:
+
+            prefix = IndexedDb.make_prefix(db_id.dbid_no, 0, 0)
+            for record in reversed(self._fetched_records):
+                if record.key.startswith(prefix) and record.state == ccl_leveldb.KeyState.Live:
+                    # we only want live keys and the newest version thereof (highest seq)
+                    meta_type = record.key[len(prefix)]
+                    old_version = db_meta.get((db_id.dbid_no, meta_type))
+                    if old_version is None or old_version.seq < record.seq:
+                        db_meta[(db_id.dbid_no, meta_type)] = record
+
+        return db_meta
+
+    def _get_raw_object_store_metadata(self, live_only=True):
+        if not live_only:
+            raise NotImplementedError("Deleted metadata not implemented yet")
+
+        os_meta = {}
+
+        for db_id in self.global_metadata.db_ids:
+
+            prefix = IndexedDb.make_prefix(db_id.dbid_no, 0, 0, [50])
+
+            for record in reversed(self._fetched_records):
+                if record.key.startswith(prefix) and record.state == ccl_leveldb.KeyState.Live:
+                    # we only want live keys and the newest version thereof (highest seq)
+                    objstore_id, varint_raw = _le_varint_from_bytes(record.key[len(prefix):])
+                    meta_type = record.key[len(prefix) + len(varint_raw)]
+
+                    old_version = os_meta.get((db_id.dbid_no, objstore_id, meta_type))
+
+                    if old_version is None or old_version.seq < record.seq:
+                        os_meta[(db_id.dbid_no, objstore_id, meta_type)] = record
+
+        return os_meta
+
+    def read_record_precursor(
+            self, key: IdbKey, db_id: int, store_id: int, buffer: bytes,
+            bad_deserializer_data_handler: typing.Callable[[IdbKey, bytes], typing.Any],
+            external_data_path: typing.Optional[str] = None):
+        val_idx = 0
+        trailer = None
+        blink_type_tag = buffer[val_idx]
+        if blink_type_tag != 0xff:
+            # TODO: probably don't want to fail hard here long term...
+            if bad_deserializer_data_handler is not None:
+                bad_deserializer_data_handler(key, buffer)
+                return None
+            else:
+                raise ValueError("Blink type tag not present")
+
+        val_idx += 1
+
+        blink_version, varint_raw = _le_varint_from_bytes(buffer[val_idx:])
+
+        val_idx += len(varint_raw)
+
+        # Peek the next byte to work out if the data is held externally:
+        # third_party/blink/renderer/modules/indexeddb/idb_value_wrapping.cc
+        if buffer[val_idx] == 0x01:  # kReplaceWithBlob
+            val_idx += 1
+            externally_serialized_blob_size, varint_raw = _le_varint_from_bytes(buffer[val_idx:])
+            val_idx += len(varint_raw)
+            externally_serialized_blob_index, varint_raw = _le_varint_from_bytes(buffer[val_idx:])
+            val_idx += len(varint_raw)
+
+            try:
+                info = self.get_blob_info(db_id, store_id, key.raw_key, externally_serialized_blob_index)
+            except KeyError:
+                info = None
+
+            if info is not None:
+                data_path = pathlib.Path(str(db_id), f"{info.blob_number >> 8:02x}", f"{info.blob_number:x}")
+                try:
+                    blob = self.get_blob(db_id, store_id, key.raw_key, externally_serialized_blob_index).read()
+                except FileNotFoundError:
+                    if bad_deserializer_data_handler is not None:
+                        bad_deserializer_data_handler(key, buffer)
+                        return None
+                    raise
+
+                return self.read_record_precursor(
+                    key, db_id, store_id,
+                    blob,
+                    bad_deserializer_data_handler, str(data_path))
+            else:
+                return None
+        else:
+            if blink_version >= BlinkTrailer.MIN_WIRE_FORMAT_VERSION_FOR_TRAILER:
+                trailer = BlinkTrailer.from_buffer(buffer, val_idx)  # TODO: do something with the trailer
+                val_idx += BlinkTrailer.TRAILER_SIZE
+
+            obj_raw = io.BytesIO(buffer[val_idx:])
+
+        return blink_version, obj_raw, trailer, external_data_path
+
+    def iterate_records(
+            self, db_id: int, store_id: int, *,
+            live_only=False, bad_deserializer_data_handler: typing.Callable[[IdbKey, bytes], typing.Any] = None):
+        blink_deserializer = ccl_blink_value_deserializer.BlinkV8Deserializer()
+
+        # goodness me this is a slow way of doing things
+        prefix = IndexedDb.make_prefix(db_id, store_id, 1)
+
+        for record in self._fetched_records:
+            if record.key.startswith(prefix):
+                key = IdbKey(record.key[len(prefix):])
+                if not record.value:
+                    # empty values will obviously fail, returning None is probably better than dying.
+                    yield IndexedDbRecord(self, db_id, store_id, key, None,
+                                          record.state == ccl_leveldb.KeyState.Live, record.seq, record.origin_file)
+                    continue
+                value_version, varint_raw = _le_varint_from_bytes(record.value)
+                val_idx = len(varint_raw)
+                # read the blink envelope
+                precursor = self.read_record_precursor(
+                    key, db_id, store_id, record.value[val_idx:], bad_deserializer_data_handler)
+                if precursor is None:
+                    continue  # only returns None on error, handled in the function if bad_deserializer_data_handler can
+
+                blink_version, obj_raw, trailer, external_path = precursor
+
+                try:
+                    deserializer = ccl_v8_value_deserializer.Deserializer(
+                        obj_raw, host_object_delegate=blink_deserializer.read)
+                    value = deserializer.read()
+                except Exception:
+                    if bad_deserializer_data_handler is not None:
+                        bad_deserializer_data_handler(key, record.value)
+                        continue
+                    raise
+                yield IndexedDbRecord(self, db_id, store_id, key, value,
+                                      record.state == ccl_leveldb.KeyState.Live,
+                                      record.seq, record.origin_file, external_path)
+
+    def get_blob_info(self, db_id: int, store_id: int, raw_key: bytes, file_index: int) -> IndexedDBExternalObject:
+        # if db_id > 0x7f or store_id > 0x7f:
+        #     raise NotImplementedError("there could be this many dbs, but I don't support it yet")
+
+        if result := self._blob_lookup_cache.get((db_id, store_id, raw_key, file_index)):
+            return result
+
+        # goodness me this is a slow way of doing things,
+        # TODO: we should at least cache along the way to our record
+        # prefix = bytes([0, db_id, store_id, 3])
+        prefix = IndexedDb.make_prefix(db_id, store_id, 3)
+        for record in self._fetched_records:
+            if record.user_key.startswith(prefix):
+                this_raw_key = record.user_key[len(prefix):]
+                buff = io.BytesIO(record.value)
+                idx = 0
+                while buff.tell() < len(record.value):
+                    blob_info = IndexedDBExternalObject.from_stream(buff)
+                    self._blob_lookup_cache[(db_id, store_id, this_raw_key, idx)] = blob_info
+                    idx += 1
+                # if this_raw_key == raw_key:
+                #     break
+
+        if result := self._blob_lookup_cache.get((db_id, store_id, raw_key, file_index)):
+            return result
+        else:
+            raise KeyError((db_id, store_id, raw_key, file_index))
+
+    def get_blob(self, db_id: int, store_id: int, raw_key: bytes, file_index: int) -> typing.BinaryIO:
+        # Some detail here: https://github.com/chromium/chromium/blob/master/content/browser/indexed_db/docs/README.md
+        if self._blob_dir is None:
+            raise ValueError("Can't resolve blob if blob dir is not set")
+        info = self.get_blob_info(db_id, store_id, raw_key, file_index)
+
+        # path will be: origin.blob/database id/top 16 bits of blob number with two digits/blob number
+        # TODO: check if this is still the case on non-windows systems
+        path = pathlib.Path(self._blob_dir, f"{db_id:x}", f"{info.blob_number >> 8:02x}", f"{info.blob_number:x}")
+
+        if path.exists():
+            return path.open("rb")
+
+        raise FileNotFoundError(path)
+
+    def get_undo_task_scopes(self):
+        # https://github.com/chromium/chromium/blob/master/components/services/storage/indexed_db/scopes/leveldb_scopes_coding.cc
+
+        # Prefix will be 00 00 00 00 32 (00|01|02) (varint of scope number) 00
+        # 00 00 00 00  =  Global metadata
+        # 32           =  kScopesPrefixByte from indexed_db_leveldb_coding.cc
+        # (00|01|02)   =  one of: kGlobalMetadataByte, kScopesMetadataByte or kLogByte from leveldb_scopes_coding.h
+        # (varint of scope)
+        # 00           =  kUndoTasksByte from leveldb_scopes_coding.h
+
+        # This is a slow way of doing this:
+        prefix = bytes.fromhex("00 00 00 00 32")
+        for record in self._fetched_records:
+            if record.state != ccl_leveldb.KeyState.Live:
+                continue
+            if record.user_key.startswith(prefix):
+                # process the key first as they define what we'll do later
+                o = len(prefix)
+                metadata_byte = record.user_key[o]
+                assert metadata_byte in (0, 1, 2)  # TODO: replace with real exception
+
+                o += 1
+
+                if metadata_byte == 0:  # global meta
+                    # print(f"Global metadata:\t{record.user_key.hex(' ')}")
+                    continue  # Don't currently think I need this to do the work
+                elif metadata_byte == 1:  # scope meta
+                    # print(f"Scope metadata:\t{record.user_key.hex(' ')}")
+                    # scope_number, varint_bytes = _le_varint_from_bytes(record.user_key)
+                    # o += len(varint_bytes)
+                    continue  # Don't currently think I need this to do the work
+                elif metadata_byte == 2:  # log
+                    scope_number, varint_bytes = _le_varint_from_bytes(record.user_key)
+                    o += len(varint_bytes)
+                    undo_byte = record.key[o]
+                    if undo_byte != 0:
+                        continue
+                    o += 1
+                    # print(f"Log\t{record.user_key.hex(' ')}")
+                    undo_sequence_number, = struct.unpack(">q", record.user_key[o:o + 8])
+
+                    # Value should be a LevelDBScopesUndoTask protobuf
+                    # (indexed_db_components\indexed_db\scopes\scopes_metadata.proto).
+                    # We're looking for a "Put" protobuf (first and only tag should be a Message numbered 1, with two
+                    # bytes values numbered 1 and 2 which are the original key and value respectively.
+                    # To reduce the need for dependencies, as they are so simple, the protobuf can be decoded "manually"
+                    with io.BytesIO(record.value) as value_stream:
+                        root_tag_raw = read_le_varint(value_stream)
+                        root_number = root_tag_raw >> 3
+                        if root_tag_raw & 0x07 != 2 or root_number != 1:
+                            assert root_number in (2, 3)  # TODO: remove?
+                            continue  # I don't think I need to raise an exception here?
+                        data_length = read_le_varint(value_stream)
+                        inner_value_bytes = value_stream.read(data_length)
+                        if len(inner_value_bytes) != data_length:
+                            raise ValueError("Couldn't get all data when reading the LevelDBScopesUndoTask")
+
+                    record_key_raw = None
+                    record_value_raw = None
+                    with io.BytesIO(inner_value_bytes) as inner_value_stream:
+                        while inner_value_stream.tell() < len(inner_value_bytes) and (
+                                record_key_raw is None or record_value_raw is None):
+                            tag_raw = read_le_varint(inner_value_stream)
+                            assert tag_raw & 0x07 == 2
+                            tag_number = tag_raw >> 3
+                            data_length = read_le_varint(inner_value_stream)
+                            data = inner_value_stream.read(data_length)
+                            if len(data) != data_length:
+                                raise ValueError("Couldn't get enough from the protobuf in LevelDBScopesUndoTask")
+                            if tag_number == 1:
+                                record_key_raw = data
+                            elif tag_number == 2:
+                                record_value_raw = data
+                            else:
+                                raise ValueError("Unexpected message in LevelDBScopesUndoTask")
+
+                    if not record_value_raw:
+                        continue  # I don't think we need to go further here
+
+                    with io.BytesIO(record_key_raw) as record_key_stream:
+                        db_id, object_store, index_id, length = IndexedDb.read_prefix(record_key_stream)
+                        if db_id < 1 or object_store < 1 or index_id < 1:
+                            continue  # only work with indexeddb record records
+
+                        key = IdbKey(record_key_stream.read())
+
+                        yield key, record_value_raw
+
+    def close(self):
+        self._db.close()
+
+    @property
+    def database_path(self):
+        return self._db.in_dir_path
+
+    def __enter__(self) -> "IndexedDb":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+
+class WrappedObjectStore:
+    """
+    A wrapper class around a "raw" IndexedDb which simplifies accessing records related to an object store. Usually only
+    created by a WrappedDatabase.
+    """
+    def __init__(self, raw_db: IndexedDb,  dbid_no: int, obj_store_id: int):
+        self._raw_db = raw_db
+        self._dbid_no = dbid_no
+        self._obj_store_id = obj_store_id
+
+    @property
+    def object_store_id(self) -> int:
+        return self._obj_store_id
+
+    @property
+    def name(self) -> str:
+        return self._raw_db.get_object_store_metadata(
+            self._dbid_no, self._obj_store_id, ObjectStoreMetadataType.StoreName)
+
+    @staticmethod
+    def _log_error(key: IdbKey, data: bytes):
+        sys.stderr.write(f"ERROR decoding key: {key}\n")
+
+    def get_blob(self, raw_key: bytes, file_index: int) -> typing.BinaryIO:
+        """
+        Deprecated: use IndexedDbRecord.get_blob_stream
+
+        :param raw_key: raw key of the record from which the blob originates
+        :param file_index: the file/blob index from a ccl_blink_value_deserializer.BlobIndex
+        :return: a file-like object of the blob
+        """
+
+        return self._raw_db.get_blob(self._dbid_no, self.object_store_id, raw_key, file_index)
+
+    # def __iter__(self):
+    #     yield from self._raw_db.iterate_records(self._dbid_no, self._obj_store_id)
+
+    def iterate_records(
+            self, *, live_only=False, errors_to_stdout=False,
+            bad_deserializer_data_handler: typing.Callable[[IdbKey, bytes], typing.Any] = None):
+
+        def _handler(key, record):
+            if bad_deserializer_data_handler is not None:
+                bad_deserializer_data_handler(key, record)
+            if errors_to_stdout:
+                WrappedObjectStore._log_error(key, record)
+
+        handler = _handler if errors_to_stdout or bad_deserializer_data_handler is not None else None
+
+        yield from self._raw_db.iterate_records(
+            self._dbid_no, self._obj_store_id, live_only=live_only,
+            bad_deserializer_data_handler=handler)
+
+    def __repr__(self):
+        return f"<WrappedObjectStore: object_store_id={self.object_store_id}; name={self.name}>"
+
+
+class WrappedDatabase:
+    """
+    A wrapper class around the raw "IndexedDb" class which simplifies access to a Database in the IndexedDb. Usually
+    only created by WrappedIndexedDb.
+    """
+    def __init__(self, raw_db: IndexedDb,  dbid: DatabaseId):
+        self._raw_db = raw_db
+        self._dbid = dbid
+
+        names = []
+        for obj_store_id in range(1, self.object_store_count + 1):
+            names.append(self._raw_db.get_object_store_metadata(
+                self.db_number, obj_store_id, ObjectStoreMetadataType.StoreName))
+        self._obj_store_names = tuple(names)
+        # pre-compile object store wrappers as there's little overhead
+        self._obj_stores = tuple(
+            WrappedObjectStore(
+                self._raw_db, self.db_number, i) for i in range(1, self.object_store_count + 1))
+
+    @property
+    def name(self) -> str:
+        """
+        :return: the name of this WrappedDatabase
+        """
+        return self._dbid.name
+
+    @property
+    def origin(self) -> str:
+        """
+        :return: the origin (host name) for this WrappedDatabase
+        """
+        return self._dbid.origin
+
+    @property
+    def db_number(self) -> int:
+        """
+        :return: the numerical ID assigned to this WrappedDatabase
+        """
+        return self._dbid.dbid_no
+
+    @property
+    def object_store_count(self) -> int:
+        """
+        :return: the "MaximumObjectStoreId" value fot this database; NB this may not be the *actual* number of object
+            stores which can be read - it is possible that some object stores may be deleted. Use len() to check the
+            number of object stores you can actually access
+        """
+        # NB obj store ids are enumerated from 1.
+        return self._raw_db.get_database_metadata(
+            self.db_number,
+            DatabaseMetadataType.MaximumObjectStoreId) or 0  # returns None if there are none.
+
+    @property
+    def object_store_names(self) -> typing.Iterable[str]:
+        """
+        :return: yields the names of the object stores in this WrappedDatabase
+        """
+        yield from self._obj_store_names
+
+    def get_object_store_by_id(self, obj_store_id: int) -> WrappedObjectStore:
+        """
+        :param obj_store_id: the numerical ID for an object store in this WrappedDatabase
+        :return: the WrappedObjectStore with the ID provided
+        """
+        if obj_store_id > 0 and obj_store_id <= self.object_store_count:
+            return self._obj_stores[obj_store_id - 1]
+        raise ValueError("obj_store_id must be greater than zero and less or equal to object_store_count "
+                         "NB object stores are enumerated from 1 - there is no store with id 0")
+
+    def get_object_store_by_name(self, name: str) -> WrappedObjectStore:
+        """
+        :param name: the name of an object store in this WrappedDatabase
+        :return: the WrappedObjectStore with the name provided
+        """
+        if name in self:
+            return self.get_object_store_by_id(self._obj_store_names.index(name) + 1)
+        raise KeyError(f"{name} is not an object store in this database")
+
+    def __iter__(self) -> typing.Iterable[WrappedObjectStore]:
+        """
+        :return: yields the object stores in this WrappedDatabase
+        """
+        yield from self._obj_stores
+
+    def __len__(self) -> int:
+        """
+        :return: the number of object stores accessible in this WrappedDatabase
+        """
+        return len(self._obj_stores)
+
+    def __contains__(self, item: str) -> bool:
+        """
+        :param item: the name of an object store in this WrappedDatabase
+        :return: True if the name provided matches one of the Object stores in this WrappedDatabase
+        """
+        return item in self._obj_store_names
+
+    def __getitem__(self, item: typing.Union[int, str]) -> WrappedObjectStore:
+        """
+        :param item: either the numerical ID of an object store (as an int) or the name of an object store in this
+            WrappedDatabase
+        :return:
+        """
+        if isinstance(item, int):
+            return self.get_object_store_by_id(item)
+        elif isinstance(item, str):
+            return self.get_object_store_by_name(item)
+        raise TypeError("Key can only be str (name) or int (id number)")
+
+    def __repr__(self):
+        return f"<WrappedDatabase: id={self.db_number}; name={self.name}; origin={self.origin}>"
+
+
+class WrappedIndexDB:
+    """
+    A wrapper object around the "raw" IndexedDb class. This should be used in most cases as the code required to use it
+    is simpler and more pythonic.
+    """
+    def __init__(self, leveldb_dir: os.PathLike, leveldb_blob_dir: os.PathLike = None):
+        self._raw_db = IndexedDb(leveldb_dir, leveldb_blob_dir)
+        self._multiple_origins = len(set(x.origin for x in self._raw_db.global_metadata.db_ids)) > 1
+
+        self._db_number_lookup = {
+            x.dbid_no: WrappedDatabase(self._raw_db, x)
+            for x in self._raw_db.global_metadata.db_ids}
+        # set origin to 0 if there's only 1, and we'll ignore it in all lookups
+        self._db_name_lookup = {
+            (x.name, x.origin if self.has_multiple_origins else 0): x
+            for x in self._db_number_lookup.values()}
+
+    def close(self):
+        self._raw_db.close()
+
+    @property
+    def database_count(self) -> int:
+        """
+        :return: The number of databases in this IndexedDB
+        """
+        return len(self._db_number_lookup)
+
+    @property
+    def database_ids(self) -> typing.Iterable[DatabaseId]:
+        """
+        :return: yields DatabaseId objects which define the databases in this IndexedDb
+        """
+        yield from self._raw_db.global_metadata.db_ids
+
+    @property
+    def has_multiple_origins(self) -> bool:
+        return self._multiple_origins
+
+    def __len__(self):
+        """
+        :return: the number of databases in this IndexedDb
+        """
+        len(self._db_number_lookup)
+
+    def __contains__(self, item: typing.Union[str, int, tuple[str, str]]):
+        """
+        :param item: either a database id number, the name of a database (as a string), or (if the database has multiple
+            origins), a tuple of database name and origin
+        :return: True if this IndexedDb contains the referenced database identifier
+        """
+        if isinstance(item, str):
+            if self.has_multiple_origins:
+                raise ValueError(
+                    "Database contains multiple origins, lookups must be provided as a tuple of (name, origin)")
+            return (item, 0) in self._db_name_lookup
+        elif isinstance(item, tuple) and len(item) == 2:
+            name, origin = item
+            if not self.has_multiple_origins:
+                origin = 0  # origin ignored if not needed
+            return (name, origin) in self._db_name_lookup
+        elif isinstance(item, int):
+            return item in self._db_number_lookup
+        elif isinstance(item, DatabaseId):
+            hit = self._db_number_lookup.get(item.dbid_no)
+            return hit is not None and hit.name == item.name and hit.origin == item.origin
+        else:
+            raise TypeError("keys must be provided as a tuple of (name, origin) or a str (if only single origin) or int")
+
+    def __getitem__(self, item: typing.Union[DatabaseId, int, str, typing.Tuple[str, str]]) -> WrappedDatabase:
+        """
+        :param item: either a DatabaseID object database id number, the name of a database (as a string), or
+        (if the database has multiple origins), a tuple of database name and origin
+        :return: the WrappedDatabase referenced by the id in item
+        """
+        if isinstance(item, DatabaseId):
+            if item.dbid_no in self._db_number_lookup:
+                result = self._db_number_lookup[item.dbid_no]
+                if result.name == item.name and result.origin == item.origin:
+                    return result
+                else:
+                    raise KeyError(item)
+            else:
+                raise KeyError(item)
+        if isinstance(item, int):
+            if item in self._db_number_lookup:
+                return self._db_number_lookup[item]
+            else:
+                raise KeyError(item)
+        elif isinstance(item, str):
+            if self.has_multiple_origins:
+                raise ValueError(
+                    "Database contains multiple origins, indexes must be provided as a tuple of (name, origin)")
+            if item in self:
+                return self._db_name_lookup[item, 0]
+            else:
+                raise KeyError(item)
+        elif isinstance(item, tuple) and len(item) == 2:
+            name, origin = item
+            if not self.has_multiple_origins:
+                origin = 0  # origin ignored if not needed
+            if (name, origin) in self:
+                return self._db_name_lookup[name, origin]
+            else:
+                raise KeyError(item)
+
+        raise TypeError("Lookups must be one of int, str or tuple of name and origin")
+
+    def __repr__(self):
+        return f"<WrappedIndexDB: {self._raw_db.database_path}>"
+
+    def __enter__(self) -> "WrappedIndexDB":
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()