dfindexeddb 20241031__py3-none-any.whl → 20251109__py3-none-any.whl

dfindexeddb/leveldb/descriptor.py

@@ -14,14 +14,13 @@
 # limitations under the License.
 """Parser for LevelDB Descriptor (MANIFEST) files."""
 from __future__ import annotations
+
 from collections import defaultdict
 from dataclasses import dataclass, field
-from typing import Generator, Optional
+from typing import Dict, Generator, Optional
 
 from dfindexeddb import errors
-from dfindexeddb.leveldb import definitions
-from dfindexeddb.leveldb import log
-from dfindexeddb.leveldb import utils
+from dfindexeddb.leveldb import definitions, log, utils
 
 
 @dataclass
@@ -34,6 +33,7 @@ class InternalKey:
     sequence_number: the sequence number.
     key_type: the key type.
   """
+
   offset: int
   user_key: bytes
   sequence_number: int
@@ -41,7 +41,8 @@ class InternalKey:
 
   @classmethod
   def FromDecoder(
-      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0) -> InternalKey:
+      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
+  ) -> InternalKey:
     """Decodes an InternalKey from the current position of a LevelDBDecoder.
 
     Args:
@@ -54,20 +55,22 @@ class InternalKey:
     offset, slice_bytes = decoder.DecodeLengthPrefixedSlice()
 
     if len(slice_bytes) < definitions.PACKED_SEQUENCE_AND_TYPE_LENGTH:
-      raise errors.ParserError('Insufficient bytes to parse InternalKey')
+      raise errors.ParserError("Insufficient bytes to parse InternalKey")
 
-    user_key = slice_bytes[:-definitions.SEQUENCE_LENGTH]
+    user_key = slice_bytes[: -definitions.SEQUENCE_LENGTH]
     sequence_number = int.from_bytes(
-        slice_bytes[-definitions.SEQUENCE_LENGTH:],
-        byteorder='little',
-        signed=False)
+        slice_bytes[-definitions.SEQUENCE_LENGTH :],
+        byteorder="little",
+        signed=False,
+    )
     key_type = slice_bytes[-definitions.PACKED_SEQUENCE_AND_TYPE_LENGTH]
 
     return cls(
         offset=base_offset + offset,
         user_key=user_key,
         sequence_number=sequence_number,
-        key_type=key_type)
+        key_type=key_type,
+    )
 
 
 @dataclass
@@ -82,6 +85,7 @@ class NewFile(utils.FromDecoderMixin):
     smallest: the smallest internal key.
     largest: the largest internal key.
   """
+
   offset: int
   level: int
   number: int
@@ -91,7 +95,8 @@ class NewFile(utils.FromDecoderMixin):
 
   @classmethod
   def FromDecoder(
-      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0) -> NewFile:
+      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
+  ) -> NewFile:
     """Decodes a NewFile from the current position of a LevelDBDecoder.
 
     Args:
@@ -113,7 +118,8 @@ class NewFile(utils.FromDecoderMixin):
         number=number,
         file_size=file_size,
         smallest=smallest,
-        largest=largest)
+        largest=largest,
+    )
 
 
 @dataclass
@@ -125,6 +131,7 @@ class CompactPointer(utils.FromDecoderMixin):
     level: the level.
     key: the key bytes.
   """
+
   offset: int
   level: int
   key: bytes
@@ -156,13 +163,15 @@ class DeletedFile(utils.FromDecoderMixin):
     level: the level.
     number: the file number.
   """
+
   offset: int
   level: int
   number: int
 
   @classmethod
   def FromDecoder(
-      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0) -> DeletedFile:
+      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
+  ) -> DeletedFile:
     """Decodes a DeletedFile from the current position of a LevelDBDecoder.
 
     Args:
@@ -192,6 +201,7 @@ class VersionEdit(utils.FromDecoderMixin):
     deleted_files: the list of DeletedFiles.
     new_files: the list of NewFiles.
   """
+
   offset: int
   comparator: Optional[bytes] = None
   log_number: Optional[int] = None
@@ -204,7 +214,8 @@ class VersionEdit(utils.FromDecoderMixin):
 
   @classmethod
   def FromDecoder(
-      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0) -> VersionEdit:
+      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
+  ) -> VersionEdit:
     """Decodes a VersionEdit from the current position of a LevelDBDecoder.
 
     Args:
@@ -225,7 +236,8 @@ class VersionEdit(utils.FromDecoderMixin):
         tag = definitions.VersionEditTags(tag_byte)
       except TypeError as error:
         raise errors.ParserError(
-            f'Invalid VersionEditTag at offset {offset}') from error
+            f"Invalid VersionEditTag at offset {offset}"
+        ) from error
 
       if tag == definitions.VersionEditTags.COMPARATOR:
         _, version_edit.comparator = decoder.DecodeLengthPrefixedSlice()
@@ -239,15 +251,18 @@ class VersionEdit(utils.FromDecoderMixin):
         _, version_edit.last_sequence = decoder.DecodeUint64Varint()
       elif tag == definitions.VersionEditTags.COMPACT_POINTER:
         compact_pointer = CompactPointer.FromDecoder(
-            decoder=decoder, base_offset=base_offset + offset)
+            decoder=decoder, base_offset=base_offset + offset
+        )
         version_edit.compact_pointers.append(compact_pointer)
       elif tag == definitions.VersionEditTags.DELETED_FILE:
         deleted_file = DeletedFile.FromDecoder(
-            decoder=decoder, base_offset=base_offset + offset)
+            decoder=decoder, base_offset=base_offset + offset
+        )
         version_edit.deleted_files.append(deleted_file)
       elif tag == definitions.VersionEditTags.NEW_FILE:
         file_metadata = NewFile.FromDecoder(
-            decoder=decoder, base_offset=base_offset + offset)
+            decoder=decoder, base_offset=base_offset + offset
+        )
         version_edit.new_files.append(file_metadata)
 
       if decoder.NumRemainingBytes() == 0:
@@ -272,11 +287,12 @@ class LevelDBVersion:
   "Deleted files" will typically no longer exist but may be forensically
   recoverable.
   """
-  current_log: str
+
+  current_log: Optional[str]
   version_edit_offset: int
-  last_sequence: int
-  active_files: dict[int, dict[int, NewFile]]
-  deleted_files: dict[int, dict[int, DeletedFile]]
+  last_sequence: Optional[int]
+  active_files: dict[int, dict[str, NewFile]]
+  deleted_files: dict[int, dict[str, DeletedFile]]
 
 
 class FileReader:
@@ -288,6 +304,7 @@ class FileReader:
   * records (PhysicalRecord)
   * version edits (VersionEdit)
   """
+
   def __init__(self, filename: str):
     """Initializes the Descriptor a.k.a. MANIFEST file.
 
@@ -296,7 +313,6 @@ class FileReader:
     """
     self.filename = filename
 
-
   def GetBlocks(self) -> Generator[log.Block, None, None]:
     """Returns an iterator of Block instances.
 
@@ -305,7 +321,7 @@ class FileReader:
     Yields:
       Block
     """
-    with open(self.filename, 'rb') as fh:
+    with open(self.filename, "rb") as fh:
       block = log.Block.FromStream(fh)
       while block:
         yield block
@@ -332,23 +348,33 @@ class FileReader:
       VersionEdit
     """
     buffer = bytearray()
+    offset = 0
     for physical_record in self.GetPhysicalRecords():
-      if (physical_record.record_type ==
-          definitions.LogFilePhysicalRecordType.FULL):
-        buffer = physical_record.contents
+      if (
+          physical_record.record_type
+          == definitions.LogFilePhysicalRecordType.FULL
+      ):
         offset = physical_record.contents_offset + physical_record.base_offset
-        version_edit = VersionEdit.FromBytes(buffer, base_offset=offset)
+        version_edit = VersionEdit.FromBytes(
+            physical_record.contents, base_offset=offset
+        )
         yield version_edit
         buffer = bytearray()
-      elif (physical_record.record_type ==
-            definitions.LogFilePhysicalRecordType.FIRST):
+      elif (
+          physical_record.record_type
+          == definitions.LogFilePhysicalRecordType.FIRST
+      ):
         offset = physical_record.contents_offset + physical_record.base_offset
         buffer = bytearray(physical_record.contents)
-      elif (physical_record.record_type ==
-            definitions.LogFilePhysicalRecordType.MIDDLE):
+      elif (
+          physical_record.record_type
+          == definitions.LogFilePhysicalRecordType.MIDDLE
+      ):
         buffer.extend(bytearray(physical_record.contents))
-      elif (physical_record.record_type ==
-            definitions.LogFilePhysicalRecordType.LAST):
+      elif (
+          physical_record.record_type
+          == definitions.LogFilePhysicalRecordType.LAST
+      ):
         buffer.extend(bytearray(physical_record.contents))
         version_edit = VersionEdit.FromBytes(buffer, base_offset=offset)
         yield version_edit
@@ -360,30 +386,34 @@ class FileReader:
     Yields:
       LevelDBVersion
     """
-    active_files = defaultdict(dict)
-    deleted_files = defaultdict(set)
+    active_files: Dict[int, dict[str, NewFile]] = defaultdict(dict)
+    deleted_files: Dict[int, dict[str, DeletedFile]] = defaultdict(dict)
     current_log = None
 
     for version_edit in self.GetVersionEdits():
       if version_edit.log_number:
-        current_log = f'{version_edit.log_number:06d}.log'
+        current_log = f"{version_edit.log_number:06d}.log"
 
       for new_file in version_edit.new_files:
-        active_files[new_file.level][f'{new_file.number:06d}.ldb'] = new_file
+        active_files[new_file.level][f"{new_file.number:06d}.ldb"] = new_file
 
       for deleted_file in version_edit.deleted_files:
-        active_files[deleted_file.level].pop(f'{deleted_file.number:06d}.ldb')
-        deleted_files[deleted_file.level].add(f'{deleted_file.number:06d}.ldb')
+        active_files[deleted_file.level].pop(f"{deleted_file.number:06d}.ldb")
+        deleted_files[deleted_file.level][
+            f"{deleted_file.number:06d}.ldb"
+        ] = deleted_file
 
       yield LevelDBVersion(
           current_log=current_log,
-          active_files=dict(active_files),
-          deleted_files=dict(deleted_files),
+          active_files=active_files,
+          deleted_files=deleted_files,
           version_edit_offset=version_edit.offset,
-          last_sequence=version_edit.last_sequence)
+          last_sequence=version_edit.last_sequence,
+      )
 
-  def GetLatestVersion(self) -> LevelDBVersion:
+  def GetLatestVersion(self) -> Optional[LevelDBVersion]:
     """Returns the latest LevelDBVersion instance."""
+    latest = None
     for version in self.GetVersions():
       latest = version
     return latest

dfindexeddb/leveldb/ldb.py

@@ -15,16 +15,15 @@
 """Parser for LevelDB Table (.ldb) files."""
 from __future__ import annotations
 
-from dataclasses import dataclass, field
 import io
 import os
+from dataclasses import dataclass, field
 from typing import BinaryIO, Iterable, Tuple
 
 import snappy
 import zstd
 
-from dfindexeddb.leveldb import definitions
-from dfindexeddb.leveldb import utils
+from dfindexeddb.leveldb import definitions, utils
 
 
 @dataclass
@@ -38,6 +37,7 @@ class KeyValueRecord:
     sequence_number: the sequence number of the record.
     record_type: the type of the record.
   """
+
   offset: int
   key: bytes
   value: bytes
@@ -65,17 +65,24 @@ class KeyValueRecord:
     _, value = decoder.ReadBytes(value_length)
 
     shared_key = shared_key[:shared_bytes] + key_delta
-    key = shared_key[:-definitions.PACKED_SEQUENCE_AND_TYPE_LENGTH]
+    key = shared_key[: -definitions.PACKED_SEQUENCE_AND_TYPE_LENGTH]
     sequence_number = int.from_bytes(
-        key[-definitions.SEQUENCE_LENGTH:], byteorder='little', signed=False)
+        shared_key[-definitions.SEQUENCE_LENGTH :],
+        byteorder="little",
+        signed=False,
+    )
     key_type = shared_key[-definitions.PACKED_SEQUENCE_AND_TYPE_LENGTH]
     record_type = definitions.InternalRecordType(key_type)
-    return cls(
-        offset=offset + block_offset,
-        key=key,
-        value=value,
-        sequence_number=sequence_number,
-        record_type=record_type), shared_key
+    return (
+        cls(
+            offset=offset + block_offset,
+            key=key,
+            value=value,
+            sequence_number=sequence_number,
+            record_type=record_type,
+        ),
+        shared_key,
+    )
 
 
 @dataclass
@@ -86,6 +93,7 @@ class Block:
     offset: the offset of the block.
     block_offset:
   """
+
   offset: int
   block_offset: int
   length: int
@@ -103,9 +111,9 @@ class Block:
   def GetBuffer(self) -> bytes:
     """Returns the block buffer, decompressing if required."""
     if self.IsSnappyCompressed():
-      return snappy.decompress(self.data)
+      return bytes(snappy.decompress(self.data))
     if self.IsZstdCompressed():
-      return zstd.decompress(self.data)
+      return bytes(zstd.decompress(self.data))
     return self.data
 
   def GetRecords(self) -> Iterable[KeyValueRecord]:
@@ -123,18 +131,19 @@ class Block:
     #    num_restarts: uint32
     decoder.stream.seek(-definitions.BLOCK_RESTART_ENTRY_LENGTH, os.SEEK_END)
     _, num_restarts = decoder.DecodeUint32()
-    restarts_offset = (
-        decoder.stream.tell()) - (
-            (num_restarts + 1) * definitions.BLOCK_RESTART_ENTRY_LENGTH)
+    restarts_offset = (decoder.stream.tell()) - (
+        (num_restarts + 1) * definitions.BLOCK_RESTART_ENTRY_LENGTH
+    )
 
     decoder.stream.seek(restarts_offset)
     _, offset = decoder.DecodeUint32()
     decoder.stream.seek(offset)
-    key = b''
+    key = b""
 
     while decoder.stream.tell() < restarts_offset:
       key_value_record, key = KeyValueRecord.FromDecoder(
-          decoder, self.block_offset, key)
+          decoder, self.block_offset, key
+      )
       yield key_value_record
 
     # TODO: parse trailer of the block for restart points (where the full
@@ -151,6 +160,7 @@ class BlockHandle(utils.FromDecoderMixin):
     block_offset: the offset of the block.
     length: the length of the block.
   """
+
   offset: int
   block_offset: int
   length: int
@@ -170,20 +180,18 @@ class BlockHandle(utils.FromDecoderMixin):
     stream.seek(self.block_offset, os.SEEK_SET)
     data = stream.read(self.length)
     if len(data) != self.length:
-      raise ValueError('Could not read all of the block')
+      raise ValueError("Could not read all of the block")
 
     footer = stream.read(definitions.BLOCK_TRAILER_SIZE)
     if len(footer) != definitions.BLOCK_TRAILER_SIZE:
-      raise ValueError('Could not read all of the block footer')
+      raise ValueError("Could not read all of the block footer")
 
     return Block(self.offset, self.block_offset, self.length, data, footer)
 
   @classmethod
   def FromDecoder(
-      cls: BlockHandle,
-      decoder: utils.LevelDBDecoder,
-      base_offset: int = 0
-    ) -> BlockHandle:
+      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
+  ) -> BlockHandle:
     """Decodes a BlockHandle from the current position of a LevelDBDecoder.
 
     Args:
@@ -217,10 +225,10 @@ class FileReader:
       ValueError if the file has an invalid magic number at the end.
     """
     self.filename = filename
-    with open(self.filename, 'rb') as fh:
+    with open(self.filename, "rb") as fh:
       fh.seek(-len(definitions.TABLE_MAGIC), os.SEEK_END)
       if fh.read(len(definitions.TABLE_MAGIC)) != definitions.TABLE_MAGIC:
-        raise ValueError(f'Invalid magic number in {self.filename}')
+        raise ValueError(f"Invalid magic number in {self.filename}")
 
       fh.seek(-definitions.TABLE_FOOTER_SIZE, os.SEEK_END)
       # meta_handle, need to read first due to variable integers
@@ -236,11 +244,12 @@ class FileReader:
     Yields:
       Block.
     """
-    with open(self.filename, 'rb') as fh:
+    with open(self.filename, "rb") as fh:
       for key_value_record in self.index_block.GetRecords():
         block_handle = BlockHandle.FromStream(
             io.BytesIO(key_value_record.value),
-            base_offset=key_value_record.offset)
+            base_offset=key_value_record.offset,
+        )
         yield block_handle.Load(fh)
 
   def GetKeyValueRecords(self) -> Iterable[KeyValueRecord]:
@@ -252,11 +261,11 @@ class FileReader:
     for block in self.GetBlocks():
       yield from block.GetRecords()
 
-  def RangeIter(self) -> Iterable[Tuple[bytes, bytes]]:  #pylint: disable=C0103
+  def RangeIter(self) -> Iterable[Tuple[bytes, bytes]]:  # pylint: disable=C0103
     """Returns an iterator of key-value pairs.
 
     Yields:
       A tuple of key and value as bytes.
     """
     for record in self.GetKeyValueRecords():
-      yield (record.key, record.value)
+      yield record.key, record.value

dfindexeddb/leveldb/log.py

@@ -15,13 +15,12 @@
 """Parser for LevelDB Log (.log) files."""
 from __future__ import annotations
 
-from dataclasses import dataclass
 import io
+from dataclasses import dataclass
 from typing import BinaryIO, Generator, Iterable, Optional
 
 from dfindexeddb import errors
-from dfindexeddb.leveldb import definitions
-from dfindexeddb.leveldb import utils
+from dfindexeddb.leveldb import definitions, utils
 
 
 @dataclass
@@ -36,6 +35,7 @@ class ParsedInternalKey:
     key: the record key.
     value: the record value.
   """
+
   offset: int
   record_type: definitions.InternalRecordType
   sequence_number: int
@@ -70,15 +70,16 @@ class ParsedInternalKey:
     if record_type == definitions.InternalRecordType.VALUE:
       _, value = decoder.DecodeBlobWithLength()
     elif record_type == definitions.InternalRecordType.DELETED:
-      value =  b''
+      value = b""
     else:
-      raise ValueError(f'Invalid record type {record_type}')
+      raise ValueError(f"Invalid record type {record_type}")
     return cls(
         offset=base_offset + offset,
         record_type=record_type,
         key=key,
         value=value,
-        sequence_number=sequence_number)
+        sequence_number=sequence_number,
+    )
 
 
 @dataclass
@@ -91,6 +92,7 @@ class WriteBatch(utils.FromDecoderMixin):
     count: the number of ParsedInternalKey in the batch.
     records: the ParsedInternalKey parsed from the batch.
   """
+
   offset: int
   sequence_number: int
   count: int
@@ -98,7 +100,7 @@ class WriteBatch(utils.FromDecoderMixin):
 
   @classmethod
   def FromDecoder(
-    cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
+      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
   ) -> WriteBatch:
     """Parses a WriteBatch from a binary stream.
 
@@ -116,15 +118,17 @@ class WriteBatch(utils.FromDecoderMixin):
     records = []
     for relative_sequence_number in range(count):
       record = ParsedInternalKey.FromDecoder(
-          decoder, base_offset + offset,
-          relative_sequence_number + sequence_number
+          decoder,
+          base_offset + offset,
+          relative_sequence_number + sequence_number,
       )
       records.append(record)
     return cls(
         offset=base_offset + offset,
         sequence_number=sequence_number,
         count=count,
-        records=records)
+        records=records,
+    )
 
 
 @dataclass
@@ -132,13 +136,15 @@ class PhysicalRecord(utils.FromDecoderMixin):
   """A physical record from a leveldb log file.
 
   Attributes:
-    offset: the record offset.
+    base_offset: the base offset.
     checksum: the record checksum.
     length: the length of the record in bytes.
-    type: the record type.
+    offset: the record offset.
+    record_type: the record type.
     contents: the record contents.
     contents_offset: the offset of where the record contents are stored.
   """
+
   base_offset: int
   offset: int
   checksum: int
@@ -172,8 +178,9 @@ class PhysicalRecord(utils.FromDecoderMixin):
       record_type = definitions.LogFilePhysicalRecordType(record_type_byte)
     except ValueError as error:
       raise errors.ParserError(
-          f'Error parsing record type of Physical Record at offset '
-          f'{offset + base_offset}') from error
+          f"Error parsing record type of Physical Record at offset "
+          f"{offset + base_offset}"
+      ) from error
     contents_offset, contents = decoder.ReadBytes(length)
     return cls(
         base_offset=base_offset,
@@ -182,7 +189,8 @@ class PhysicalRecord(utils.FromDecoderMixin):
         length=length,
         record_type=record_type,
         contents=contents,
-        contents_offset=contents_offset)
+        contents_offset=contents_offset,
+    )
 
 
 @dataclass
@@ -193,6 +201,7 @@ class Block:
     offset: the block offset.
     data: the block data.
   """
+
   offset: int
   data: bytes
 
@@ -237,7 +246,7 @@ class FileReader:
   A Log FileReader provides read-only sequential iteration of serialized
   structures in a leveldb logfile.  These structures include:
   * blocks (Block)
-  * phyiscal records (PhysicalRecord)
+  * physical records (PhysicalRecord)
   * batches (WriteBatch) and
   * key/value records (ParsedInternalKey).
 
@@ -261,7 +270,7 @@ class FileReader:
     Yields:
       a Block
     """
-    with open(self.filename, 'rb') as fh:
+    with open(self.filename, "rb") as fh:
       block = Block.FromStream(fh)
       while block:
         yield block
@@ -288,22 +297,30 @@ class FileReader:
       WriteBatch
     """
     buffer = bytearray()
+    offset = 0
     for physical_record in self.GetPhysicalRecords():
-      if (physical_record.record_type ==
-         definitions.LogFilePhysicalRecordType.FULL):
-        buffer = physical_record.contents
+      if (
+          physical_record.record_type
+          == definitions.LogFilePhysicalRecordType.FULL
+      ):
         offset = physical_record.contents_offset + physical_record.base_offset
-        yield WriteBatch.FromBytes(buffer, base_offset=offset)
+        yield WriteBatch.FromBytes(physical_record.contents, base_offset=offset)
         buffer = bytearray()
-      elif (physical_record.record_type
-            == definitions.LogFilePhysicalRecordType.FIRST):
+      elif (
+          physical_record.record_type
+          == definitions.LogFilePhysicalRecordType.FIRST
+      ):
         offset = physical_record.contents_offset + physical_record.base_offset
         buffer = bytearray(physical_record.contents)
-      elif (physical_record.record_type ==
-            definitions.LogFilePhysicalRecordType.MIDDLE):
+      elif (
+          physical_record.record_type
+          == definitions.LogFilePhysicalRecordType.MIDDLE
+      ):
         buffer.extend(bytearray(physical_record.contents))
-      elif (physical_record.record_type ==
-            definitions.LogFilePhysicalRecordType.LAST):
+      elif (
+          physical_record.record_type
+          == definitions.LogFilePhysicalRecordType.LAST
+      ):
         buffer.extend(bytearray(physical_record.contents))
         yield WriteBatch.FromBytes(buffer, base_offset=offset)
         buffer = bytearray()