dfindexeddb 20240305__tar.gz → 20240324__tar.gz

{dfindexeddb-20240305/dfindexeddb.egg-info → dfindexeddb-20240324}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dfindexeddb
-Version: 20240305
+Version: 20240324
 Summary: dfindexeddb is an experimental Python tool for performing digital forensic analysis of IndexedDB and leveldb files.
 Author-email: Syd Pleno <sydp@google.com>
 Maintainer-email: dfIndexeddb Developers <dfindexeddb-dev@googlegroups.com>
@@ -226,7 +226,9 @@ dfindexeddb is an experimental Python tool for performing digital forensic
 analysis of IndexedDB and leveldb files.
 
 It parses leveldb, IndexedDB and javascript structures from these files without
-requiring native libraries.
+requiring native libraries.  (Note: only a subset of IndexedDB key types and
+Javascript types for Chromium-based browsers are currently supported.  Safari
+and Firefox are under development).
 
 The content of IndexedDB files is dependent on what a web application stores
 locally/offline using the web browser's
@@ -236,25 +238,34 @@ include:
 * emails and contact information from an e-mail application,
 * images and metadata from a photo gallery application
 
+
 ## Installation
 
+1. [Linux] Install the snappy compression development package
+
 ```
-$ pip install dfindexeddb
+    $ sudo apt install libsnappy-dev
 ```
 
-## Installation from source
+2. Create a virtual environment and install the package
 
-### Linux
+```
+    $ python3 -m venv .venv
+    $ source .venv/bin/activate
+    $ pip install dfindexeddb
+```
 
-1. Install the snappy compression development package
+## Installation from source
+
+1. [Linux] Install the snappy compression development package
 
 ```
     $ sudo apt install libsnappy-dev
 ```
 
-2. Clone or download the repository to your local machine.
+2. Clone or download/unzip the repository to your local machine.
 
-3. Create a virutal environemnt and install the package
+3. Create a virtual environment and install the package
 
 ```
     $ python3 -m venv .venv
@@ -264,55 +275,58 @@ $ pip install dfindexeddb
 
 ## Usage
 
-A CLI tool is available after installation:
+Two CLI tools for parsing IndexedDB/leveldb files are available after
+installation:
+
+
+### IndexedDB
 
 ```
 $ dfindexeddb -h
-usage: dfindexeddb [-h] -s SOURCE [--json] {log,ldb,indexeddb} ...
-
-A cli tool for the dfindexeddb package
+usage: dfindexeddb [-h] -s SOURCE [--json]
 
-positional arguments:
-  {log,ldb,indexeddb}
+A cli tool for parsing indexeddb files
 
 options:
+  -h, --help            show this help message and exit
   -s SOURCE, --source SOURCE
-                        The source leveldb file
+                        The source leveldb folder
   --json                Output as JSON
 ```
 
-To parse a LevelDB .log file:
+### LevelDB
 
 ```
-$ dfindexeddb -s <SOURCE> log -h
-usage: dfindexeddb log [-h] {blocks,physical_records,write_batches,parsed_internal_key,records}
+$ dfleveldb -h
+usage: dfleveldb [-h] {db,log,ldb,descriptor} ...
+
+A cli tool for parsing leveldb files
 
 positional arguments:
-  {blocks,physical_records,write_batches,parsed_internal_key,records}
+  {db,log,ldb,descriptor}
+    db                  Parse a directory as leveldb.
+    log                 Parse a leveldb log file.
+    ldb                 Parse a leveldb table (.ldb) file.
+    descriptor          Parse a leveldb descriptor (MANIFEST) file.
 
 options:
   -h, --help            show this help message and exit
 ```
 
-To parse a LevelDB .ldb file:
+To parse records from a LevelDB log (.log) file, use the following command:
 
 ```
-$ dfindexeddb -s <SOURCE> ldb -h
-usage: dfindexeddb ldb [-h] {blocks,records}
+$ dfleveldb log -s <SOURCE> [--json]
+```
 
-positional arguments:
-  {blocks,records}
+To parse records from a LevelDB table (.ldb) file, use the following command:
 
-options:
-  -h, --help        show this help message and exit
+```
+$ dfleveldb ldb -s <SOURCE> [--json]
 ```
 
-To parse a LevelDB .ldb or .log file as IndexedDB:
+To parse version edit records from a Descriptor (MANIFEST) file:
 
 ```
-$ dfindexeddb -s <SOURCE> indexeddb -h
-usage: dfindexeddb indexeddb [-h]
-
-options:
-  -h, --help  show this help message and exit
+$ dfleveldb descriptor -s <SOURCE> [--json]
 ```

dfindexeddb-20240324/README.md

@@ -0,0 +1,110 @@
+# dfIndexeddb
+
+dfindexeddb is an experimental Python tool for performing digital forensic
+analysis of IndexedDB and leveldb files.
+
+It parses leveldb, IndexedDB and javascript structures from these files without
+requiring native libraries.  (Note: only a subset of IndexedDB key types and
+Javascript types for Chromium-based browsers are currently supported.  Safari
+and Firefox are under development).
+
+The content of IndexedDB files is dependent on what a web application stores
+locally/offline using the web browser's
+[IndexedDB API](https://www.w3.org/TR/IndexedDB/).  Examples of content might
+include:
+* text from a text/source-code editor application,
+* emails and contact information from an e-mail application,
+* images and metadata from a photo gallery application
+
+
+## Installation
+
+1. [Linux] Install the snappy compression development package
+
+```
+    $ sudo apt install libsnappy-dev
+```
+
+2. Create a virtual environment and install the package
+
+```
+    $ python3 -m venv .venv
+    $ source .venv/bin/activate
+    $ pip install dfindexeddb
+```
+
+## Installation from source
+
+1. [Linux] Install the snappy compression development package
+
+```
+    $ sudo apt install libsnappy-dev
+```
+
+2. Clone or download/unzip the repository to your local machine.
+
+3. Create a virtual environment and install the package
+
+```
+    $ python3 -m venv .venv
+    $ source .venv/bin/activate
+    $ pip install .
+```
+
+## Usage
+
+Two CLI tools for parsing IndexedDB/leveldb files are available after
+installation:
+
+
+### IndexedDB
+
+```
+$ dfindexeddb -h
+usage: dfindexeddb [-h] -s SOURCE [--json]
+
+A cli tool for parsing indexeddb files
+
+options:
+  -h, --help            show this help message and exit
+  -s SOURCE, --source SOURCE
+                        The source leveldb folder
+  --json                Output as JSON
+```
+
+### LevelDB
+
+```
+$ dfleveldb -h
+usage: dfleveldb [-h] {db,log,ldb,descriptor} ...
+
+A cli tool for parsing leveldb files
+
+positional arguments:
+  {db,log,ldb,descriptor}
+    db                  Parse a directory as leveldb.
+    log                 Parse a leveldb log file.
+    ldb                 Parse a leveldb table (.ldb) file.
+    descriptor          Parse a leveldb descriptor (MANIFEST) file.
+
+options:
+  -h, --help            show this help message and exit
+```
+
+To parse records from a LevelDB log (.log) file, use the following command:
+
+```
+$ dfleveldb log -s <SOURCE> [--json]
+```
+
+To parse records from a LevelDB table (.ldb) file, use the following command:
+
+```
+$ dfleveldb ldb -s <SOURCE> [--json]
+```
+
+To parse version edit records from a Descriptor (MANIFEST) file:
+
+```
+$ dfleveldb descriptor -s <SOURCE> [--json]
+```

{dfindexeddb-20240305 → dfindexeddb-20240324}/dfindexeddb/indexeddb/blink.py

@@ -86,7 +86,8 @@ class V8ScriptValueDecoder:
       NotImplementedError: when called.
     """
     tag = self.ReadTag()
-    raise NotImplementedError(f'V8ScriptValueDecoder.ReadHostObject - {tag}')
+    raise NotImplementedError(
+        f'V8ScriptValueDecoder.ReadHostObject - {tag.name}')
 
   def Deserialize(self) -> Any:
     """Deserializes a Blink SSV.

{dfindexeddb-20240305 → dfindexeddb-20240324}/dfindexeddb/indexeddb/chromium.py

@@ -20,11 +20,11 @@ import io
 from typing import Any, BinaryIO, Optional, Tuple, Type, TypeVar, Union
 
 from dfindexeddb import errors
-from dfindexeddb import utils
 from dfindexeddb.indexeddb import blink
 from dfindexeddb.indexeddb import definitions
 from dfindexeddb.leveldb import ldb
 from dfindexeddb.leveldb import log
+from dfindexeddb.leveldb import utils
 
 
 T = TypeVar('T')
@@ -570,7 +570,7 @@ class EarlistCompactionTimeKey(BaseIndexedDBKey):
 class ScopesPrefixKey(BaseIndexedDBKey):
   """A scopes prefix IndexedDB key."""
 
-  def DecodeValue(self, decoder: utils.StreamDecoder) -> Optional[bytes]:
+  def DecodeValue(self, decoder: utils.LevelDBDecoder) -> Optional[bytes]:
     """Decodes the scopes prefix value."""
     if decoder.NumRemainingBytes:
       return decoder.ReadBytes()[1]
@@ -578,7 +578,7 @@ class ScopesPrefixKey(BaseIndexedDBKey):
 
   @classmethod
   def FromDecoder(
-      cls, decoder: utils.StreamDecoder, key_prefix: KeyPrefix,
+      cls, decoder: utils.LevelDBDecoder, key_prefix: KeyPrefix,
       base_offset: int = 0
   ) -> ScopesPrefixKey:
     """Decodes the scopes prefix key."""
@@ -1357,4 +1357,4 @@ class IndexedDBRecord:
       value=idb_value,
       sequence_number=record.sequence_number if hasattr(
           record, 'sequence_number') else None,
-      type=record.type)
+      type=record.record_type)

dfindexeddb-20240324/dfindexeddb/indexeddb/cli.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A CLI tool for dfindexeddb."""
+import argparse
+import dataclasses
+from datetime import datetime
+import json
+import pathlib
+import sys
+import traceback
+
+from dfindexeddb import errors
+from dfindexeddb import version
+from dfindexeddb.leveldb import record as leveldb_record
+from dfindexeddb.indexeddb import chromium
+from dfindexeddb.indexeddb import v8
+
+
+_VALID_PRINTABLE_CHARACTERS = (
+    ' abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' +
+    '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~.')
+
+
+class Encoder(json.JSONEncoder):
+  """A JSON encoder class for dfindexeddb fields."""
+  def default(self, o):
+    if dataclasses.is_dataclass(o):
+      o_dict = dataclasses.asdict(o)
+      return o_dict
+    if isinstance(o, bytes):
+      out = []
+      for x in o:
+        if chr(x) not in _VALID_PRINTABLE_CHARACTERS:
+          out.append(f'\\x{x:02X}')
+        else:
+          out.append(chr(x))
+      return ''.join(out)
+    if isinstance(o, datetime):
+      return o.isoformat()
+    if isinstance(o, v8.Undefined):
+      return "<undefined>"
+    if isinstance(o, v8.Null):
+      return "<null>"
+    if isinstance(o, set):
+      return list(o)
+    if isinstance(o, v8.RegExp):
+      return str(o)
+    return json.JSONEncoder.default(self, o)
+
+
+def _Output(structure, to_json=False):
+  """Helper method to output parsed structure to stdout."""
+  if to_json:
+    print(json.dumps(structure, indent=2, cls=Encoder))
+  else:
+    print(structure)
+
+
+def IndexeddbCommand(args):
+  """The CLI for processing a log/ldb file as indexeddb."""
+  for db_record in leveldb_record.LevelDBRecord.FromDir(args.source):
+    record = db_record.record
+    try:
+      db_record.record = chromium.IndexedDBRecord.FromLevelDBRecord(record)
+    except(
+        errors.ParserError,
+        errors.DecoderError,
+        NotImplementedError) as err:
+      print(
+          (f'Error parsing blink value: {err} for {record.__class__.__name__} '
+           f'at offset {record.offset} in {db_record.path}'), file=sys.stderr)
+      print(f'Traceback: {traceback.format_exc()}', file=sys.stderr)
+    _Output(db_record, to_json=args.json)
+
+
+def App():
+  """The CLI app entrypoint for dfindexeddb."""
+  parser = argparse.ArgumentParser(
+      prog='dfindexeddb',
+      description='A cli tool for parsing indexeddb files',
+      epilog=f'Version {version.GetVersion()}')
+  parser.add_argument(
+      '-s', '--source', required=True, type=pathlib.Path,
+      help='The source leveldb folder')
+  parser.add_argument('--json', action='store_true', help='Output as JSON')
+  parser.set_defaults(func=IndexeddbCommand)
+
+  args = parser.parse_args()
+  args.func(args)

dfindexeddb-20240324/dfindexeddb/leveldb/cli.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A CLI tool for leveldb files."""
+import argparse
+import dataclasses
+from datetime import datetime
+import json
+import pathlib
+
+from dfindexeddb import version
+from dfindexeddb.leveldb import descriptor
+from dfindexeddb.leveldb import ldb
+from dfindexeddb.leveldb import log
+from dfindexeddb.leveldb import record
+
+
+_VALID_PRINTABLE_CHARACTERS = (
+    ' abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' +
+    '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~.')
+
+
+class Encoder(json.JSONEncoder):
+  """A JSON encoder class for dfleveldb fields."""
+
+  def default(self, o):
+    """Returns a serializable object for o."""
+    if dataclasses.is_dataclass(o):
+      o_dict = dataclasses.asdict(o)
+      return o_dict
+    if isinstance(o, bytes):
+      out = []
+      for x in o:
+        if chr(x) not in _VALID_PRINTABLE_CHARACTERS:
+          out.append(f'\\x{x:02X}')
+        else:
+          out.append(chr(x))
+      return ''.join(out)
+    if isinstance(o, datetime):
+      return o.isoformat()
+    if isinstance(o, set):
+      return list(o)
+    return json.JSONEncoder.default(self, o)
+
+
+def _Output(structure, to_json=False):
+  """Helper method to output parsed structure to stdout."""
+  if to_json:
+    print(json.dumps(structure, indent=2, cls=Encoder))
+  else:
+    print(structure)
+
+
+def DbCommand(args):
+  """The CLI for processing leveldb folders."""
+  for rec in record.LevelDBRecord.FromDir(args.source):
+    _Output(rec, to_json=args.json)
+
+
+def LdbCommand(args):
+  """The CLI for processing ldb files."""
+  ldb_file = ldb.FileReader(args.source)
+
+  if args.structure_type == 'blocks':
+    # Prints block information.
+    for block in ldb_file.GetBlocks():
+      _Output(block, to_json=args.json)
+
+  elif args.structure_type == 'records' or not args.structure_type:
+    # Prints key value record information.
+    for key_value_record in ldb_file.GetKeyValueRecords():
+      _Output(key_value_record, to_json=args.json)
+
+  else:
+    print(f'{args.structure_type} is not supported for ldb files.')
+
+
+def LogCommand(args):
+  """The CLI for processing log files."""
+  log_file = log.FileReader(args.source)
+
+  if args.structure_type == 'blocks':
+    # Prints block information.
+    for block in log_file.GetBlocks():
+      _Output(block, to_json=args.json)
+
+  elif args.structure_type == 'physical_records':
+    # Prints log file physical record information.
+    for log_file_record in log_file.GetPhysicalRecords():
+      _Output(log_file_record, to_json=args.json)
+
+  elif args.structure_type == 'write_batches':
+    # Prints log file batch information.
+    for batch in log_file.GetWriteBatches():
+      _Output(batch, to_json=args.json)
+
+  elif (args.structure_type in ('parsed_internal_key', 'records')
+        or not args.structure_type):
+    # Prints key value record information.
+    for internal_key_record in log_file.GetParsedInternalKeys():
+      _Output(internal_key_record, to_json=args.json)
+
+  else:
+    print(f'{args.structure_type} is not supported for log files.')
+
+
+def DescriptorCommand(args):
+  """The CLI for processing descriptor (MANIFEST) files."""
+  manifest_file = descriptor.FileReader(args.source)
+
+  if args.structure_type == 'blocks':
+    # Prints block information.
+    for block in manifest_file.GetBlocks():
+      _Output(block, to_json=args.json)
+
+  elif args.structure_type == 'physical_records':
+    # Prints log file physical record information.
+    for log_file_record in manifest_file.GetPhysicalRecords():
+      _Output(log_file_record, to_json=args.json)
+
+  elif (args.structure_type == 'versionedit'
+        or not args.structure_type):
+    for version_edit in manifest_file.GetVersionEdits():
+      _Output(version_edit, to_json=args.json)
+
+  else:
+    print(f'{args.structure_type} is not supported for descriptor files.')
+
+def App():
+  """The CLI app entrypoint for parsing leveldb files."""
+  parser = argparse.ArgumentParser(
+      prog='dfleveldb',
+      description='A cli tool for parsing leveldb files',
+      epilog=f'Version {version.GetVersion()}')
+
+  subparsers = parser.add_subparsers()
+
+  parser_db = subparsers.add_parser(
+      'db', help='Parse a directory as leveldb.')
+  parser_db.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source leveldb directory')
+  parser_db.add_argument(
+      '--json', action='store_true', help='Output as JSON')
+  parser_db.set_defaults(func=DbCommand)
+
+  parser_log = subparsers.add_parser(
+      'log', help='Parse a leveldb log file.')
+  parser_log.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source leveldb file')
+  parser_log.add_argument(
+      '--json', action='store_true', help='Output as JSON')
+  parser_log.add_argument(
+      '-t',
+      '--structure_type',
+      choices=[
+          'blocks',
+          'physical_records',
+          'write_batches',
+          'parsed_internal_key'])
+  parser_log.set_defaults(func=LogCommand)
+
+  parser_ldb = subparsers.add_parser(
+      'ldb', help='Parse a leveldb table (.ldb) file.')
+  parser_ldb.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source leveldb file')
+  parser_ldb.add_argument(
+      '--json', action='store_true', help='Output as JSON')
+  parser_ldb.add_argument(
+      '-t',
+      '--structure_type',
+      choices=[
+          'blocks',
+          'records'])
+  parser_ldb.set_defaults(func=LdbCommand)
+
+  parser_descriptor = subparsers.add_parser(
+      'descriptor', help='Parse a leveldb descriptor (MANIFEST) file.')
+  parser_descriptor.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source leveldb file')
+  parser_descriptor.add_argument(
+      '--json', action='store_true', help='Output as JSON')
+  parser_descriptor.add_argument(
+      '-t',
+      '--structure_type',
+      choices=[
+          'blocks', 'physical_records', 'versionedit'])
+  parser_descriptor.set_defaults(func=DescriptorCommand)
+
+  args = parser.parse_args()
+
+  if not hasattr(args, 'func'):
+    parser.print_usage()
+  else:
+    args.func(args)

{dfindexeddb-20240305 → dfindexeddb-20240324}/dfindexeddb/leveldb/definitions.py

@@ -16,12 +16,22 @@
 
 import enum
 
+BLOCK_RESTART_ENTRY_LENGTH = 4
+BLOCK_TRAILER_SIZE = 5
+TABLE_FOOTER_SIZE = 48
+TABLE_MAGIC = b'\x57\xfb\x80\x8b\x24\x75\x47\xdb'
 
 PACKED_SEQUENCE_AND_TYPE_LENGTH = 8
 SEQUENCE_LENGTH = 7
 TYPE_LENGTH = 1
 
 
+class BlockCompressionType(enum.IntEnum):
+  """Block compression types."""
+  SNAPPY = 1
+  ZSTD = 2
+
+
 class VersionEditTags(enum.IntEnum):
   """VersionEdit tags."""
   COMPARATOR = 1
@@ -41,3 +51,9 @@ class LogFilePhysicalRecordType(enum.IntEnum):
   FIRST = 2
   MIDDLE = 3
   LAST = 4
+
+
+class InternalRecordType(enum.IntEnum):
+  """Internal record types."""
+  DELETED = 0
+  VALUE = 1