dfindexeddb 20240301__py3-none-any.whl → 20240324__py3-none-any.whl

dfindexeddb/leveldb/log.py

@@ -15,20 +15,13 @@
 """Parser for LevelDB Log (.log) files."""
 from __future__ import annotations
 
-from dataclasses import dataclass, field
-from enum import IntEnum
+from dataclasses import dataclass
 import io
 from typing import BinaryIO, Generator, Iterable, Optional
 
-from dfindexeddb import utils
-
-
-class LogFilePhysicalRecordType(IntEnum):
-  """LevelDB log file physical record types."""
-  FULL = 1
-  FIRST = 2
-  MIDDLE = 3
-  LAST = 4
+from dfindexeddb import errors
+from dfindexeddb.leveldb import definitions
+from dfindexeddb.leveldb import utils
 
 
 @dataclass
@@ -37,12 +30,15 @@ class ParsedInternalKey:
 
   Attributes:
     offset: the offset of the record.
-    type: the record type.
+    record_type: the record type.
+    sequence_number: the sequence number (inferred from the relative location
+        the ParsedInternalKey in a WriteBatch.)
     key: the record key.
     value: the record value.
   """
   offset: int
-  type: int
+  record_type: definitions.InternalRecordType
+  sequence_number: int
   key: bytes
   value: bytes
 
@@ -50,37 +46,47 @@ class ParsedInternalKey:
   def FromDecoder(
       cls,
       decoder: utils.LevelDBDecoder,
-      base_offset: int = 0
+      base_offset: int = 0,
+      sequence_number: int = 0,
   ) -> ParsedInternalKey:
     """Decodes an internal key value record.
 
     Args:
       decoder: the leveldb decoder.
-      base_offset: the base offset for the parsed key value record.
+      base_offset: the base offset for the parsed internal key value record.
+      sequence_number: the sequence number for the parsed internal key value
+          record.
 
     Returns:
-      a ParsedInternalKey
+      A ParsedInternalKey
 
     Raises:
       ValueError: if there is an invalid record type encountered.
     """
     offset, record_type = decoder.DecodeUint8()
     _, key = decoder.DecodeBlobWithLength()
-    if record_type == 1:
+    record_type = definitions.InternalRecordType(record_type)
+
+    if record_type == definitions.InternalRecordType.VALUE:
       _, value = decoder.DecodeBlobWithLength()
-    elif record_type == 0:
+    elif record_type == definitions.InternalRecordType.DELETED:
       value =  b''
     else:
       raise ValueError(f'Invalid record type {record_type}')
-    return cls(base_offset + offset, record_type, key, value)
+    return cls(
+        offset=base_offset + offset,
+        record_type=record_type,
+        key=key,
+        value=value,
+        sequence_number=sequence_number)
 
 
 @dataclass
-class WriteBatch:
+class WriteBatch(utils.FromDecoderMixin):
   """A write batch from a leveldb log file.
 
   Attributes:
-    offset: the batch offset.
+    offset: the write batch offset.
     sequence_number: the batch sequence number.
     count: the number of ParsedInternalKey in the batch.
     records: the ParsedInternalKey parsed from the batch.
@@ -88,49 +94,41 @@ class WriteBatch:
   offset: int
   sequence_number: int
   count: int
-  records: Iterable[ParsedInternalKey] = field(repr=False)
+  records: Iterable[ParsedInternalKey]
 
   @classmethod
-  def FromStream(
-    cls, stream: BinaryIO, base_offset: int = 0
+  def FromDecoder(
+    cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
   ) -> WriteBatch:
     """Parses a WriteBatch from a binary stream.
 
     Args:
-      stream: the binary stream to be parsed.
+      decoder: the LevelDBDecoder
       base_offset: the base offset of the Block from which the data is
           read from.
 
     Returns:
       A WriteBatch.
     """
-    decoder = utils.LevelDBDecoder(stream)
-    _, sequence_number = decoder.DecodeUint64()
+    offset, sequence_number = decoder.DecodeUint64()
     _, count = decoder.DecodeUint32()
 
     records = []
-    for _ in range(count):
-      record = ParsedInternalKey.FromDecoder(decoder, base_offset)
+    for relative_sequence_number in range(count):
+      record = ParsedInternalKey.FromDecoder(
+          decoder, base_offset + offset,
+          relative_sequence_number + sequence_number
+      )
       records.append(record)
-    return cls(base_offset, sequence_number, count, records)
-
-  @classmethod
-  def FromBytes(cls, data: bytes, base_offset: int = 0) -> WriteBatch:
-    """Parses a WriteBatch from bytes.
-
-    Args:
-      data: the bytes to be parsed.
-      base_offset: the base offset of the Block from which the data is
-          read from.
-
-    Returns:
-      A WriteBatch.
-    """
-    return cls.FromStream(io.BytesIO(data), base_offset)
+    return cls(
+        offset=base_offset + offset,
+        sequence_number=sequence_number,
+        count=count,
+        records=records)
 
 
 @dataclass
-class PhysicalRecord:
+class PhysicalRecord(utils.FromDecoderMixin):
   """A physical record from a leveldb log file.
 
   Attributes:
@@ -145,27 +143,35 @@ class PhysicalRecord:
   offset: int
   checksum: int
   length: int
-  record_type: LogFilePhysicalRecordType
-  contents: bytes = field(repr=False)
+  record_type: definitions.LogFilePhysicalRecordType
+  contents: bytes
   contents_offset: int
 
+  PHYSICAL_HEADER_LENGTH = 7
+
   @classmethod
-  def FromStream(
-      cls, stream: BinaryIO, base_offset: int = 0) -> PhysicalRecord:
-    """Parses a PhysicalRecord from a binary stream.
+  def FromDecoder(
+      cls, decoder: utils.LevelDBDecoder, base_offset: int = 0
+  ) -> PhysicalRecord:
+    """Decodes a PhysicalRecord from the current position of a LevelDBDecoder.
 
     Args:
-      stream: the binary stream to be parsed.
+      decoder: the LevelDBDecoder.
       base_offset: the base offset of the WriteBatch from which the data is
           read from.
 
     Returns:
       A PhysicalRecord.
     """
-    decoder = utils.StreamDecoder(stream)
     offset, checksum = decoder.DecodeUint32()
     _, length = decoder.DecodeUint16()
-    record_type = LogFilePhysicalRecordType(decoder.DecodeUint8()[1])
+    _, record_type_byte = decoder.DecodeUint8()
+    try:
+      record_type = definitions.LogFilePhysicalRecordType(record_type_byte)
+    except ValueError as error:
+      raise errors.ParserError(
+          f'Error parsing record type of Physical Record at offset '
+          f'{offset + base_offset}') from error
     contents_offset, contents = decoder.ReadBytes(length)
     return cls(
         base_offset=base_offset,
@@ -186,7 +192,7 @@ class Block:
     data: the block data.
   """
   offset: int
-  data: bytes = field(repr=False)
+  data: bytes
 
   BLOCK_SIZE = 32768
 
@@ -199,7 +205,7 @@ class Block:
     buffer = io.BytesIO(self.data)
     buffer_length = len(self.data)
 
-    while buffer.tell() < buffer_length:
+    while buffer.tell() + PhysicalRecord.PHYSICAL_HEADER_LENGTH < buffer_length:
       yield PhysicalRecord.FromStream(buffer, base_offset=self.offset)
 
   @classmethod
@@ -219,10 +225,10 @@ class Block:
     return cls(offset, data)
 
 
-class LogFileReader:
+class FileReader:
   """A leveldb log file reader.
 
-  A LogFileReader provides read-only sequential iteration of serialized
+  A Log FileReader provides read-only sequential iteration of serialized
   structures in a leveldb logfile.  These structures include:
   * blocks (Block)
   * phyiscal records (PhysicalRecord)
@@ -250,11 +256,10 @@ class LogFileReader:
       a Block
     """
     with open(self.filename, 'rb') as fh:
-      while True:
-        block = Block.FromStream(fh)
-        if not block:
-          break
+      block = Block.FromStream(fh)
+      while block:
         yield block
+        block = Block.FromStream(fh)
 
   def GetPhysicalRecords(self) -> Generator[PhysicalRecord, None, None]:
     """Returns an iterator of PhysicalRecord instances.
@@ -278,28 +283,32 @@ class LogFileReader:
     """
     buffer = bytearray()
     for physical_record in self.GetPhysicalRecords():
-      if physical_record.record_type == LogFilePhysicalRecordType.FULL:
+      if (physical_record.record_type ==
+         definitions.LogFilePhysicalRecordType.FULL):
         buffer = physical_record.contents
         offset = physical_record.contents_offset + physical_record.base_offset
         yield WriteBatch.FromBytes(buffer, base_offset=offset)
         buffer = bytearray()
-      elif physical_record.record_type == LogFilePhysicalRecordType.FIRST:
+      elif (physical_record.record_type
+            == definitions.LogFilePhysicalRecordType.FIRST):
         offset = physical_record.contents_offset + physical_record.base_offset
         buffer = bytearray(physical_record.contents)
-      elif physical_record.record_type == LogFilePhysicalRecordType.MIDDLE:
+      elif (physical_record.record_type ==
+            definitions.LogFilePhysicalRecordType.MIDDLE):
         buffer.extend(bytearray(physical_record.contents))
-      elif physical_record.record_type == LogFilePhysicalRecordType.LAST:
+      elif (physical_record.record_type ==
+            definitions.LogFilePhysicalRecordType.LAST):
         buffer.extend(bytearray(physical_record.contents))
         yield WriteBatch.FromBytes(buffer, base_offset=offset)
         buffer = bytearray()
 
-  def GetKeyValueRecords(self) -> Generator[ParsedInternalKey, None, None]:
-    """Returns an iterator of KeyValueRecord instances.
+  def GetParsedInternalKeys(self) -> Generator[ParsedInternalKey, None, None]:
+    """Returns an iterator of ParsedInternalKey instances.
 
-    A batch can contain on or more key value records.
+    A batch can contain one or more key value records.
 
     Yields:
-      KeyValueRecord
+      ParsedInternalKey
     """
     for batch in self.GetWriteBatches():
       yield from batch.records

dfindexeddb/leveldb/record.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A module for records from LevelDB files."""
+from __future__ import annotations
+import dataclasses
+import pathlib
+import sys
+from typing import Any, Generator, Union
+
+from dfindexeddb.leveldb import descriptor
+from dfindexeddb.leveldb import ldb
+from dfindexeddb.leveldb import log
+
+
+@dataclasses.dataclass
+class LevelDBRecord:
+  """A leveldb record.
+
+  A record can come from a log file, a table (ldb) file or a descriptor
+  (MANIFEST) file.
+
+  Attributes:
+    path: the file path where the record was parsed from.
+    record: the leveldb record.
+  """
+  path: str
+  record: Union[
+      ldb.KeyValueRecord,
+      log.ParsedInternalKey,
+      descriptor.VersionEdit]
+
+  @classmethod
+  def FromFile(
+      cls,
+      file_path: pathlib.Path,
+      include_versionedit: bool = False
+  ) -> Generator[LevelDBRecord, Any, Any]:
+    """Yields leveldb records from the given path.
+
+    Yields:
+      LevelDBRecords
+
+    Args:
+      file_path: the file path.
+      include_versionedit: include VersionEdit records from descriptor files.
+    """
+    if file_path.name.endswith('.log'):
+      for record in log.FileReader(
+          file_path.as_posix()).GetParsedInternalKeys():
+        yield cls(path=file_path.as_posix(), record=record)
+    elif file_path.name.endswith('.ldb'):
+      for record in ldb.FileReader(file_path.as_posix()).GetKeyValueRecords():
+        yield cls(path=file_path.as_posix(), record=record)
+    elif file_path.name.startswith('MANIFEST'):
+      if not include_versionedit:
+        print(f'Ignoring {file_path.as_posix()}', file=sys.stderr)
+        return
+      for record in descriptor.FileReader(
+          file_path.as_posix()).GetVersionEdits():
+        yield cls(path=file_path.as_posix(), record=record)
+    elif file_path.name in ('LOCK', 'CURRENT', 'LOG', 'LOG.old'):
+      print(f'Ignoring {file_path.as_posix()}', file=sys.stderr)
+    else:
+      print(f'Unsupported file type {file_path.as_posix()}', file=sys.stderr)
+
+  @classmethod
+  def FromDir(
+      cls,
+      path: pathlib.Path,
+      include_versionedit: bool = False
+  ) -> Generator[LevelDBRecord, Any, Any]:
+    """Yields LevelDBRecords from the given directory.
+
+    Args:
+      path: the file path.
+      include_versionedit: include VersionEdit records from descriptor files.
+
+    Yields:
+      LevelDBRecords
+
+    Raises:
+      ValueError: if path is not a directory.
+    """
+    if path.is_dir():
+      for file_path in path.iterdir():
+        yield from cls.FromFile(
+            file_path=file_path,
+            include_versionedit=include_versionedit)
+    else:
+      raise ValueError(f'{path} is not a directory')

dfindexeddb/leveldb/utils.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Helper/utility classes for LevelDB."""
+from __future__ import annotations
+import io
+from typing import BinaryIO, Tuple, Type, TypeVar
+
+from dfindexeddb import errors
+from dfindexeddb import utils
+
+
+class LevelDBDecoder(utils.StreamDecoder):
+  """A helper class to decode data types from LevelDB files."""
+
+  def DecodeBool(self) -> Tuple[int, bool]:
+    """Returns a Tuple of the offset of decoding and the bool value."""
+    offset, buffer = self.ReadBytes(1)
+    return offset, buffer[0] is not None
+
+  def DecodeString(self) -> Tuple[int, str]:
+    """Returns a tuple of the offset of decoding and the string value.
+
+    Raises:
+      errors.DecoderError: when the parsed string buffer is not even (i.e.
+          cannot be decoded as a UTF-16-BE string.
+    """
+    offset = self.stream.tell()
+    buffer = self.stream.read()
+    if len(buffer) % 2:
+      raise errors.DecoderError(
+          f'Odd number of bytes encountered at offset {offset}')
+    return offset, buffer.decode('utf-16-be')
+
+  def DecodeLengthPrefixedSlice(self) -> Tuple[int, bytes]:
+    """Returns a tuple of the offset of decoding and the byte 'slice'."""
+    offset, num_bytes = self.DecodeUint32Varint()
+    _, blob = self.ReadBytes(num_bytes)
+    return offset, blob
+
+  def DecodeBlobWithLength(self) -> Tuple[int, bytes]:
+    """Returns a tuple of a the offset of decoding and the binary blob."""
+    offset, num_bytes = self.DecodeUint64Varint()
+    _, blob = self.ReadBytes(num_bytes)
+    return offset, blob
+
+  def DecodeStringWithLength(self, encoding='utf-16-be') -> Tuple[int, str]:
+    """Returns a tuple of the offset of decoding and the string value."""
+    offset, length = self.DecodeUint64Varint()
+    _, buffer = self.ReadBytes(length*2)
+    return offset, buffer.decode(encoding=encoding)
+
+
+T = TypeVar('T')
+
+
+class FromDecoderMixin:
+  """A mixin for parsing dataclass attributes using a LevelDBDecoder."""
+
+  @classmethod
+  def FromDecoder(
+      cls: Type[T], decoder: LevelDBDecoder, base_offset: int = 0) -> T:
+    """Decodes a class type from the current position of a LevelDBDecoder.
+
+    Args:
+      decoder: the LevelDBDecoder.
+      base_offset: the base offset.
+
+    Returns:
+      The class instance.
+
+    Raises:
+      NotImplementedError if the child class does not implement this method.
+    """
+    raise NotImplementedError
+
+  @classmethod
+  def FromStream(
+      cls: Type[T], stream: BinaryIO, base_offset: int = 0) -> T:
+    """Decodes a class type from the current position of a binary stream.
+
+    Args:
+      stream: the binary stream.
+      base_offset: the base offset of the binary stream.
+
+    Returns:
+      The class instance.
+    """
+    decoder = LevelDBDecoder(stream)
+    return cls.FromDecoder(decoder=decoder, base_offset=base_offset)
+
+  @classmethod
+  def FromBytes(
+      cls: Type[T], raw_data: bytes, base_offset: int = 0) -> T:
+    """Parses a class type from raw bytes.
+
+    Args:
+      raw_data: the raw data.
+      base_offset: the base offset of the raw data.
+
+    Returns:
+      The class instance.
+    """
+    stream = io.BytesIO(raw_data)
+    return cls.FromStream(stream=stream, base_offset=base_offset)

dfindexeddb/utils.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Utilities for dfindexeddb."""
+from __future__ import annotations
 import io
 import os
 import struct
@@ -206,55 +207,19 @@ class StreamDecoder:
     return self.DecodeZigzagVarint(max_bytes=10)
 
 
-
-class LevelDBDecoder(StreamDecoder):
-  """A helper class to decode data types from LevelDB files."""
-
-  def DecodeBool(self) -> Tuple[int, bool]:
-    """Returns a Tuple of the offset of decoding and the bool value."""
-    offset, buffer = self.ReadBytes(1)
-    return offset, buffer[0] is not None
-
-  def DecodeString(self) -> Tuple[int, str]:
-    """Returns a tuple of the offset of decoding and the string value.
-
-    Raises:
-      errors.DecoderError: when the parsed string buffer is not even (i.e.
-          cannot be decoded as a UTF-16-BE string.
-    """
-    offset = self.stream.tell()
-    buffer = self.stream.read()
-    if len(buffer) % 2:
-      raise errors.DecoderError(
-          f'Odd number of bytes encountered at offset {offset}')
-    return offset, buffer.decode('utf-16-be')
-
-  def DecodeBlobWithLength(self) -> Tuple[int, bytes]:
-    """Returns a tuple of a the offset of decoding and the binary blob."""
-    offset, num_bytes = self.DecodeUint64Varint()
-    _, blob = self.ReadBytes(num_bytes)
-    return offset, blob
-
-  def DecodeStringWithLength(self) -> Tuple[int, str]:
-    """Returns a tuple of the offset of decoding and the string value."""
-    offset, length = self.DecodeUint64Varint()
-    _, buffer = self.ReadBytes(length*2)
-    return offset, buffer.decode('utf-16-be')
-
-
 T = TypeVar('T')
 
 
-class FromStreamMixin:  # TODO: refactor leveldb parsers
-  """A mixin for dataclasses parsing their attributes from a binary stream."""
+class FromDecoderMixin:
+  """A mixin for parsing dataclass attributes using a LevelDBDecoder."""
 
   @classmethod
   def FromDecoder(
-      cls: Type[T], decoder: LevelDBDecoder, base_offset: int = 0) -> T:
-    """Decodes a class type from the current position of a LevelDBDecoder.
+      cls: Type[T], decoder: StreamDecoder, base_offset: int = 0) -> T:
+    """Decodes a class type from the current position of a StreamDecoder.
 
     Args:
-      decoder: the LevelDBDecoder.
+      decoder: the StreamDecoder.
       base_offset: the base offset.
 
     Returns:
@@ -277,8 +242,8 @@ class FromStreamMixin:  # TODO: refactor leveldb parsers
     Returns:
       The class instance.
     """
-    decoder = LevelDBDecoder(stream)
-    return cls.FromDecoder(decoder, base_offset)
+    decoder = StreamDecoder(stream)
+    return cls.FromDecoder(decoder=decoder, base_offset=base_offset)
 
   @classmethod
   def FromBytes(

dfindexeddb/version.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 """Version information for dfIndexeddb."""
 
-__version__ = "20240301"
+__version__ = "20240324"
 
 
 def GetVersion():