dfindexeddb 20240305__py3-none-any.whl → 20240324__py3-none-any.whl

dfindexeddb/leveldb/log.py

@@ -15,12 +15,13 @@
 """Parser for LevelDB Log (.log) files."""
 from __future__ import annotations
 
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 import io
 from typing import BinaryIO, Generator, Iterable, Optional
 
-from dfindexeddb import utils
+from dfindexeddb import errors
 from dfindexeddb.leveldb import definitions
+from dfindexeddb.leveldb import utils
 
 
 @dataclass
@@ -29,18 +30,17 @@ class ParsedInternalKey:
 
   Attributes:
     offset: the offset of the record.
-    type: the record type.
+    record_type: the record type.
     sequence_number: the sequence number (inferred from the relative location
         the ParsedInternalKey in a WriteBatch.)
     key: the record key.
     value: the record value.
   """
   offset: int
-  type: int
+  record_type: definitions.InternalRecordType
   sequence_number: int
   key: bytes
   value: bytes
-  __type__: str = 'ParsedInternalKey'
 
   @classmethod
   def FromDecoder(
@@ -65,15 +65,17 @@ class ParsedInternalKey:
     """
     offset, record_type = decoder.DecodeUint8()
     _, key = decoder.DecodeBlobWithLength()
-    if record_type == 1:
+    record_type = definitions.InternalRecordType(record_type)
+
+    if record_type == definitions.InternalRecordType.VALUE:
       _, value = decoder.DecodeBlobWithLength()
-    elif record_type == 0:
+    elif record_type == definitions.InternalRecordType.DELETED:
       value =  b''
     else:
       raise ValueError(f'Invalid record type {record_type}')
     return cls(
         offset=base_offset + offset,
-        type=record_type,
+        record_type=record_type,
         key=key,
         value=value,
         sequence_number=sequence_number)
@@ -92,7 +94,7 @@ class WriteBatch(utils.FromDecoderMixin):
   offset: int
   sequence_number: int
   count: int
-  records: Iterable[ParsedInternalKey] = field(repr=False)
+  records: Iterable[ParsedInternalKey]
 
   @classmethod
   def FromDecoder(
@@ -142,7 +144,7 @@ class PhysicalRecord(utils.FromDecoderMixin):
   checksum: int
   length: int
   record_type: definitions.LogFilePhysicalRecordType
-  contents: bytes = field(repr=False)
+  contents: bytes
   contents_offset: int
 
   PHYSICAL_HEADER_LENGTH = 7
@@ -163,8 +165,13 @@ class PhysicalRecord(utils.FromDecoderMixin):
     """
     offset, checksum = decoder.DecodeUint32()
     _, length = decoder.DecodeUint16()
-    record_type = definitions.LogFilePhysicalRecordType(
-        decoder.DecodeUint8()[1])
+    _, record_type_byte = decoder.DecodeUint8()
+    try:
+      record_type = definitions.LogFilePhysicalRecordType(record_type_byte)
+    except ValueError as error:
+      raise errors.ParserError(
+          f'Error parsing record type of Physical Record at offset '
+          f'{offset + base_offset}') from error
     contents_offset, contents = decoder.ReadBytes(length)
     return cls(
         base_offset=base_offset,
@@ -185,7 +192,7 @@ class Block:
     data: the block data.
   """
   offset: int
-  data: bytes = field(repr=False)
+  data: bytes
 
   BLOCK_SIZE = 32768
 
@@ -276,7 +283,7 @@ class FileReader:
     """
     buffer = bytearray()
     for physical_record in self.GetPhysicalRecords():
-      if(physical_record.record_type ==
+      if (physical_record.record_type ==
          definitions.LogFilePhysicalRecordType.FULL):
         buffer = physical_record.contents
         offset = physical_record.contents_offset + physical_record.base_offset
@@ -295,13 +302,13 @@ class FileReader:
         yield WriteBatch.FromBytes(buffer, base_offset=offset)
         buffer = bytearray()
 
-  def GetKeyValueRecords(self) -> Generator[ParsedInternalKey, None, None]:
-    """Returns an iterator of KeyValueRecord instances.
+  def GetParsedInternalKeys(self) -> Generator[ParsedInternalKey, None, None]:
+    """Returns an iterator of ParsedInternalKey instances.
 
-    A batch can contain on or more key value records.
+    A batch can contain one or more key value records.
 
     Yields:
-      KeyValueRecord
+      ParsedInternalKey
     """
     for batch in self.GetWriteBatches():
       yield from batch.records

dfindexeddb/leveldb/record.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A module for records from LevelDB files."""
+from __future__ import annotations
+import dataclasses
+import pathlib
+import sys
+from typing import Any, Generator, Union
+
+from dfindexeddb.leveldb import descriptor
+from dfindexeddb.leveldb import ldb
+from dfindexeddb.leveldb import log
+
+
+@dataclasses.dataclass
+class LevelDBRecord:
+  """A leveldb record.
+
+  A record can come from a log file, a table (ldb) file or a descriptor
+  (MANIFEST) file.
+
+  Attributes:
+    path: the file path where the record was parsed from.
+    record: the leveldb record.
+  """
+  path: str
+  record: Union[
+      ldb.KeyValueRecord,
+      log.ParsedInternalKey,
+      descriptor.VersionEdit]
+
+  @classmethod
+  def FromFile(
+      cls,
+      file_path: pathlib.Path,
+      include_versionedit: bool = False
+  ) -> Generator[LevelDBRecord, Any, Any]:
+    """Yields leveldb records from the given path.
+
+    Yields:
+      LevelDBRecords
+
+    Args:
+      file_path: the file path.
+      include_versionedit: include VersionEdit records from descriptor files.
+    """
+    if file_path.name.endswith('.log'):
+      for record in log.FileReader(
+          file_path.as_posix()).GetParsedInternalKeys():
+        yield cls(path=file_path.as_posix(), record=record)
+    elif file_path.name.endswith('.ldb'):
+      for record in ldb.FileReader(file_path.as_posix()).GetKeyValueRecords():
+        yield cls(path=file_path.as_posix(), record=record)
+    elif file_path.name.startswith('MANIFEST'):
+      if not include_versionedit:
+        print(f'Ignoring {file_path.as_posix()}', file=sys.stderr)
+        return
+      for record in descriptor.FileReader(
+          file_path.as_posix()).GetVersionEdits():
+        yield cls(path=file_path.as_posix(), record=record)
+    elif file_path.name in ('LOCK', 'CURRENT', 'LOG', 'LOG.old'):
+      print(f'Ignoring {file_path.as_posix()}', file=sys.stderr)
+    else:
+      print(f'Unsupported file type {file_path.as_posix()}', file=sys.stderr)
+
+  @classmethod
+  def FromDir(
+      cls,
+      path: pathlib.Path,
+      include_versionedit: bool = False
+  ) -> Generator[LevelDBRecord, Any, Any]:
+    """Yields LevelDBRecords from the given directory.
+
+    Args:
+      path: the file path.
+      include_versionedit: include VersionEdit records from descriptor files.
+
+    Yields:
+      LevelDBRecords
+
+    Raises:
+      ValueError: if path is not a directory.
+    """
+    if path.is_dir():
+      for file_path in path.iterdir():
+        yield from cls.FromFile(
+            file_path=file_path,
+            include_versionedit=include_versionedit)
+    else:
+      raise ValueError(f'{path} is not a directory')

dfindexeddb/leveldb/utils.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Helper/utility classes for LevelDB."""
+from __future__ import annotations
+import io
+from typing import BinaryIO, Tuple, Type, TypeVar
+
+from dfindexeddb import errors
+from dfindexeddb import utils
+
+
+class LevelDBDecoder(utils.StreamDecoder):
+  """A helper class to decode data types from LevelDB files."""
+
+  def DecodeBool(self) -> Tuple[int, bool]:
+    """Returns a Tuple of the offset of decoding and the bool value."""
+    offset, buffer = self.ReadBytes(1)
+    return offset, buffer[0] is not None
+
+  def DecodeString(self) -> Tuple[int, str]:
+    """Returns a tuple of the offset of decoding and the string value.
+
+    Raises:
+      errors.DecoderError: when the parsed string buffer is not even (i.e.
+          cannot be decoded as a UTF-16-BE string.
+    """
+    offset = self.stream.tell()
+    buffer = self.stream.read()
+    if len(buffer) % 2:
+      raise errors.DecoderError(
+          f'Odd number of bytes encountered at offset {offset}')
+    return offset, buffer.decode('utf-16-be')
+
+  def DecodeLengthPrefixedSlice(self) -> Tuple[int, bytes]:
+    """Returns a tuple of the offset of decoding and the byte 'slice'."""
+    offset, num_bytes = self.DecodeUint32Varint()
+    _, blob = self.ReadBytes(num_bytes)
+    return offset, blob
+
+  def DecodeBlobWithLength(self) -> Tuple[int, bytes]:
+    """Returns a tuple of a the offset of decoding and the binary blob."""
+    offset, num_bytes = self.DecodeUint64Varint()
+    _, blob = self.ReadBytes(num_bytes)
+    return offset, blob
+
+  def DecodeStringWithLength(self, encoding='utf-16-be') -> Tuple[int, str]:
+    """Returns a tuple of the offset of decoding and the string value."""
+    offset, length = self.DecodeUint64Varint()
+    _, buffer = self.ReadBytes(length*2)
+    return offset, buffer.decode(encoding=encoding)
+
+
+T = TypeVar('T')
+
+
+class FromDecoderMixin:
+  """A mixin for parsing dataclass attributes using a LevelDBDecoder."""
+
+  @classmethod
+  def FromDecoder(
+      cls: Type[T], decoder: LevelDBDecoder, base_offset: int = 0) -> T:
+    """Decodes a class type from the current position of a LevelDBDecoder.
+
+    Args:
+      decoder: the LevelDBDecoder.
+      base_offset: the base offset.
+
+    Returns:
+      The class instance.
+
+    Raises:
+      NotImplementedError if the child class does not implement this method.
+    """
+    raise NotImplementedError
+
+  @classmethod
+  def FromStream(
+      cls: Type[T], stream: BinaryIO, base_offset: int = 0) -> T:
+    """Decodes a class type from the current position of a binary stream.
+
+    Args:
+      stream: the binary stream.
+      base_offset: the base offset of the binary stream.
+
+    Returns:
+      The class instance.
+    """
+    decoder = LevelDBDecoder(stream)
+    return cls.FromDecoder(decoder=decoder, base_offset=base_offset)
+
+  @classmethod
+  def FromBytes(
+      cls: Type[T], raw_data: bytes, base_offset: int = 0) -> T:
+    """Parses a class type from raw bytes.
+
+    Args:
+      raw_data: the raw data.
+      base_offset: the base offset of the raw data.
+
+    Returns:
+      The class instance.
+    """
+    stream = io.BytesIO(raw_data)
+    return cls.FromStream(stream=stream, base_offset=base_offset)

dfindexeddb/utils.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Utilities for dfindexeddb."""
+from __future__ import annotations
 import io
 import os
 import struct
@@ -206,48 +207,6 @@ class StreamDecoder:
     return self.DecodeZigzagVarint(max_bytes=10)
 
 
-
-class LevelDBDecoder(StreamDecoder):
-  """A helper class to decode data types from LevelDB files."""
-
-  def DecodeBool(self) -> Tuple[int, bool]:
-    """Returns a Tuple of the offset of decoding and the bool value."""
-    offset, buffer = self.ReadBytes(1)
-    return offset, buffer[0] is not None
-
-  def DecodeString(self) -> Tuple[int, str]:
-    """Returns a tuple of the offset of decoding and the string value.
-
-    Raises:
-      errors.DecoderError: when the parsed string buffer is not even (i.e.
-          cannot be decoded as a UTF-16-BE string.
-    """
-    offset = self.stream.tell()
-    buffer = self.stream.read()
-    if len(buffer) % 2:
-      raise errors.DecoderError(
-          f'Odd number of bytes encountered at offset {offset}')
-    return offset, buffer.decode('utf-16-be')
-
-  def DecodeLengthPrefixedSlice(self) -> Tuple[int, bytes]:
-    """Returns a tuple of the offset of decoding and the byte 'slice'."""
-    offset, num_bytes = self.DecodeUint32Varint()
-    _, blob = self.ReadBytes(num_bytes)
-    return offset, blob
-
-  def DecodeBlobWithLength(self) -> Tuple[int, bytes]:
-    """Returns a tuple of a the offset of decoding and the binary blob."""
-    offset, num_bytes = self.DecodeUint64Varint()
-    _, blob = self.ReadBytes(num_bytes)
-    return offset, blob
-
-  def DecodeStringWithLength(self, encoding='utf-16-be') -> Tuple[int, str]:
-    """Returns a tuple of the offset of decoding and the string value."""
-    offset, length = self.DecodeUint64Varint()
-    _, buffer = self.ReadBytes(length*2)
-    return offset, buffer.decode(encoding=encoding)
-
-
 T = TypeVar('T')
 
 
@@ -256,11 +215,11 @@ class FromDecoderMixin:
 
   @classmethod
   def FromDecoder(
-      cls: Type[T], decoder: LevelDBDecoder, base_offset: int = 0) -> T:
-    """Decodes a class type from the current position of a LevelDBDecoder.
+      cls: Type[T], decoder: StreamDecoder, base_offset: int = 0) -> T:
+    """Decodes a class type from the current position of a StreamDecoder.
 
     Args:
-      decoder: the LevelDBDecoder.
+      decoder: the StreamDecoder.
       base_offset: the base offset.
 
     Returns:
@@ -283,7 +242,7 @@ class FromDecoderMixin:
     Returns:
       The class instance.
     """
-    decoder = LevelDBDecoder(stream)
+    decoder = StreamDecoder(stream)
     return cls.FromDecoder(decoder=decoder, base_offset=base_offset)
 
   @classmethod

dfindexeddb/version.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 """Version information for dfIndexeddb."""
 
-__version__ = "20240305"
+__version__ = "20240324"
 
 
 def GetVersion():

{dfindexeddb-20240305.dist-info → dfindexeddb-20240324.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dfindexeddb
-Version: 20240305
+Version: 20240324
 Summary: dfindexeddb is an experimental Python tool for performing digital forensic analysis of IndexedDB and leveldb files.
 Author-email: Syd Pleno <sydp@google.com>
 Maintainer-email: dfIndexeddb Developers <dfindexeddb-dev@googlegroups.com>
@@ -226,7 +226,9 @@ dfindexeddb is an experimental Python tool for performing digital forensic
 analysis of IndexedDB and leveldb files.
 
 It parses leveldb, IndexedDB and javascript structures from these files without
-requiring native libraries.
+requiring native libraries.  (Note: only a subset of IndexedDB key types and
+Javascript types for Chromium-based browsers are currently supported.  Safari
+and Firefox are under development).
 
 The content of IndexedDB files is dependent on what a web application stores
 locally/offline using the web browser's
@@ -236,25 +238,34 @@ include:
 * emails and contact information from an e-mail application,
 * images and metadata from a photo gallery application
 
+
 ## Installation
 
+1. [Linux] Install the snappy compression development package
+
 ```
-$ pip install dfindexeddb
+    $ sudo apt install libsnappy-dev
 ```
 
-## Installation from source
+2. Create a virtual environment and install the package
 
-### Linux
+```
+    $ python3 -m venv .venv
+    $ source .venv/bin/activate
+    $ pip install dfindexeddb
+```
 
-1. Install the snappy compression development package
+## Installation from source
+
+1. [Linux] Install the snappy compression development package
 
 ```
     $ sudo apt install libsnappy-dev
 ```
 
-2. Clone or download the repository to your local machine.
+2. Clone or download/unzip the repository to your local machine.
 
-3. Create a virutal environemnt and install the package
+3. Create a virtual environment and install the package
 
 ```
     $ python3 -m venv .venv
@@ -264,55 +275,58 @@ $ pip install dfindexeddb
 
 ## Usage
 
-A CLI tool is available after installation:
+Two CLI tools for parsing IndexedDB/leveldb files are available after
+installation:
+
+
+### IndexedDB
 
 ```
 $ dfindexeddb -h
-usage: dfindexeddb [-h] -s SOURCE [--json] {log,ldb,indexeddb} ...
-
-A cli tool for the dfindexeddb package
+usage: dfindexeddb [-h] -s SOURCE [--json]
 
-positional arguments:
-  {log,ldb,indexeddb}
+A cli tool for parsing indexeddb files
 
 options:
+  -h, --help            show this help message and exit
   -s SOURCE, --source SOURCE
-                        The source leveldb file
+                        The source leveldb folder
   --json                Output as JSON
 ```
 
-To parse a LevelDB .log file:
+### LevelDB
 
 ```
-$ dfindexeddb -s <SOURCE> log -h
-usage: dfindexeddb log [-h] {blocks,physical_records,write_batches,parsed_internal_key,records}
+$ dfleveldb -h
+usage: dfleveldb [-h] {db,log,ldb,descriptor} ...
+
+A cli tool for parsing leveldb files
 
 positional arguments:
-  {blocks,physical_records,write_batches,parsed_internal_key,records}
+  {db,log,ldb,descriptor}
+    db                  Parse a directory as leveldb.
+    log                 Parse a leveldb log file.
+    ldb                 Parse a leveldb table (.ldb) file.
+    descriptor          Parse a leveldb descriptor (MANIFEST) file.
 
 options:
   -h, --help            show this help message and exit
 ```
 
-To parse a LevelDB .ldb file:
+To parse records from a LevelDB log (.log) file, use the following command:
 
 ```
-$ dfindexeddb -s <SOURCE> ldb -h
-usage: dfindexeddb ldb [-h] {blocks,records}
+$ dfleveldb log -s <SOURCE> [--json]
+```
 
-positional arguments:
-  {blocks,records}
+To parse records from a LevelDB table (.ldb) file, use the following command:
 
-options:
-  -h, --help        show this help message and exit
+```
+$ dfleveldb ldb -s <SOURCE> [--json]
 ```
 
-To parse a LevelDB .ldb or .log file as IndexedDB:
+To parse version edit records from a Descriptor (MANIFEST) file:
 
 ```
-$ dfindexeddb -s <SOURCE> indexeddb -h
-usage: dfindexeddb indexeddb [-h]
-
-options:
-  -h, --help  show this help message and exit
+$ dfleveldb descriptor -s <SOURCE> [--json]
 ```

dfindexeddb-20240324.dist-info/RECORD

@@ -0,0 +1,26 @@
+dfindexeddb/__init__.py,sha256=KPYL9__l8od6_OyDfGRTgaJ6iy_fqIgZ-dS2S-e3Rac,599
+dfindexeddb/errors.py,sha256=PNpwyf_lrPc4TE77oAakX3mu5D_YcP3f80wq8Y1LkvY,749
+dfindexeddb/utils.py,sha256=pV2blFnMxDwk3kBRK6UVji66ctkYpm6wfH9p0jCC7Nk,8797
+dfindexeddb/version.py,sha256=EVJT5s12vbz8xf-Q1XokuIHYzjSFiD4qJS1qBmpWjdc,750
+dfindexeddb/indexeddb/__init__.py,sha256=kExXSVBCTKCD5BZJkdMfUMqGksH-DMJxP2_lI0gq-BE,575
+dfindexeddb/indexeddb/blink.py,sha256=voC5P33MkyF9gMYJL6XVu2jDk68auTj3qxsCZw_x9rs,3548
+dfindexeddb/indexeddb/chromium.py,sha256=wczfAYSVVWwfzrSf3O_DxZmjVy4b6fzBom6mr9nKZHA,44713
+dfindexeddb/indexeddb/cli.py,sha256=drTSV2Eu-DKQev8xRqgifX68vTAjW0Nt_kUqT79-ZWo,3235
+dfindexeddb/indexeddb/definitions.py,sha256=yline3y3gmZx6s-dwjpPDNs5HO4zT6KZqPWQfEsHDoM,7413
+dfindexeddb/indexeddb/utils.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dfindexeddb/indexeddb/v8.py,sha256=ldqpc9T1kG7BOdjnHjQ5hNO9OCXZ3_Zd6vRSpC-NrEA,21893
+dfindexeddb/leveldb/__init__.py,sha256=KPYL9__l8od6_OyDfGRTgaJ6iy_fqIgZ-dS2S-e3Rac,599
+dfindexeddb/leveldb/cli.py,sha256=tqu8Vpw45d38zVGLvTNDiNtppHoakj_4IrzURzQsEv8,6702
+dfindexeddb/leveldb/definitions.py,sha256=wwm0uySOeI0-2fG9KvrY-StE0NP3iW5mCEuKYQL4ahg,1436
+dfindexeddb/leveldb/descriptor.py,sha256=cw7A_KDiKf3ZyYThPbvC-LtnPM7Jvn3zr2jPswRBTq4,10426
+dfindexeddb/leveldb/ldb.py,sha256=mN-M7PLtE_VLZCbCbzRgjkSezbMUhgDjgWgPgIxJ1jM,8087
+dfindexeddb/leveldb/log.py,sha256=QeH8oESOPEZUjANGiDRSmXZa2SuoKlPFBJY7SxTV1lg,9209
+dfindexeddb/leveldb/record.py,sha256=AnM4kQb81igmJla5q3rQUYvlzPwZaEHTvauxOC_dtM8,3217
+dfindexeddb/leveldb/utils.py,sha256=RgEEZ7Z35m3CcOUypAiViQSzKjBgSXZ3aeJhQjY3H9w,3748
+dfindexeddb-20240324.dist-info/AUTHORS,sha256=QbvjbAom57fpEkekkCVFUj0B9KUMGraR510aUMBC-PE,286
+dfindexeddb-20240324.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+dfindexeddb-20240324.dist-info/METADATA,sha256=YT7g3UbA5SXyZZLKSYwwL8Yra8Vi40xh4eJ_G7rD1M4,16473
+dfindexeddb-20240324.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+dfindexeddb-20240324.dist-info/entry_points.txt,sha256=WG9YNLZ9lBx4Q9QF6wS4dZdZfADT3Zs4_-MV5TcA0ls,102
+dfindexeddb-20240324.dist-info/top_level.txt,sha256=X9OTaub1c8S_JJ7g-f8JdkhhdiZ4x1j4eni1hdUCwE4,12
+dfindexeddb-20240324.dist-info/RECORD,,

{dfindexeddb-20240305.dist-info → dfindexeddb-20240324.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.42.0)
+Generator: bdist_wheel (0.43.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

dfindexeddb-20240324.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+dfindexeddb = dfindexeddb.indexeddb.cli:App
+dfleveldb = dfindexeddb.leveldb.cli:App