dfindexeddb 20240225__py3-none-any.whl → 20240301__py3-none-any.whl

dfindexeddb/cli.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A CLI tool for dfindexeddb."""
+import argparse
+import dataclasses
+from datetime import datetime
+import json
+import pathlib
+import sys
+import traceback
+
+from dfindexeddb.leveldb import log
+from dfindexeddb.leveldb import ldb
+from dfindexeddb.indexeddb import chromium
+from dfindexeddb import errors
+from dfindexeddb.indexeddb import v8
+
+
+class Encoder(json.JSONEncoder):
+  """A JSON encoder class for dfindexeddb fields."""
+  def default(self, o):
+    if isinstance(o, bytes):
+      return o.decode(encoding='ascii', errors='backslashreplace')
+    if isinstance(o, datetime):
+      return o.isoformat()
+    if isinstance(o, v8.Undefined):
+      return "<undefined>"
+    if isinstance(o, v8.Null):
+      return "<null>"
+    if isinstance(o, set):
+      return list(o)
+    if isinstance(o, v8.RegExp):
+      return str(o)
+    return json.JSONEncoder.default(self, o)
+
+
+def _Output(structure, to_json=False):
+  """Helper method to output parsed structure to stdout."""
+  if to_json:
+    structure_dict = dataclasses.asdict(structure)
+    print(json.dumps(structure_dict, indent=2, cls=Encoder))
+  else:
+    print(structure)
+
+
+def IndexeddbCommand(args):
+  """The CLI for processing a log/ldb file as indexeddb."""
+  if args.source.name.endswith('.log'):
+    records = list(
+        log.LogFileReader(args.source).GetKeyValueRecords())
+  elif args.source.name.endswith('.ldb'):
+    records = list(
+        ldb.LdbFileReader(args.source).GetKeyValueRecords())
+  else:
+    print('Unsupported file type.', file=sys.stderr)
+    return
+
+  for record in records:
+    try:
+      record = chromium.IndexedDBRecord.FromLevelDBRecord(record)
+    except (errors.ParserError, errors.DecoderError) as err:
+      print(
+          (f'Error parsing blink value: {err} for {record.__class__.__name__} '
+           f'at offset {record.offset}'), file=sys.stderr)
+      print(f'Traceback: {traceback.format_exc()}', file=sys.stderr)
+    _Output(record, to_json=args.json)
+
+
+def LdbCommand(args):
+  """The CLI for processing ldb files."""
+  ldb_file = ldb.LdbFileReader(args.source)
+
+  if args.structure_type == 'blocks':
+    # Prints block information.
+    for block in ldb_file.GetBlocks():
+      _Output(block, to_json=args.json)
+
+  elif args.structure_type == 'records':
+    # Prints key value record information.
+    for record in ldb_file.GetKeyValueRecords():
+      _Output(record, to_json=args.json)
+
+
+def LogCommand(args):
+  """The CLI for processing log files."""
+  log_file = log.LogFileReader(args.source)
+
+  if args.structure_type == 'blocks':
+    # Prints block information.
+    for block in log_file.GetBlocks():
+      _Output(block, to_json=args.json)
+
+  elif args.structure_type == 'physical_records':
+    # Prints log file physical record information.
+    for log_file_record in log_file.GetPhysicalRecords():
+      _Output(log_file_record, to_json=args.json)
+
+  elif args.structure_type == 'write_batches':
+    # Prints log file batch information.
+    for batch in log_file.GetWriteBatches():
+      _Output(batch, to_json=args.json)
+
+  elif args.structure_type in ('parsed_internal_key', 'records'):
+    # Prints key value record information.
+    for record in log_file.GetKeyValueRecords():
+      _Output(record, to_json=args.json)
+
+
+def App():
+  """The CLI app entrypoint."""
+  parser = argparse.ArgumentParser(
+      prog='dfindexeddb',
+      description='A cli tool for the dfindexeddb package')
+
+  parser.add_argument(
+      '-s', '--source', required=True, type=pathlib.Path, 
+      help='The source leveldb file')
+  parser.add_argument('--json', action='store_true', help='Output as JSON')
+  subparsers = parser.add_subparsers(required=True)
+
+  parser_log = subparsers.add_parser('log')
+  parser_log.add_argument(
+      'structure_type', choices=[
+          'blocks',
+          'physical_records',
+          'write_batches',
+          'parsed_internal_key',
+          'records'])
+  parser_log.set_defaults(func=LogCommand)
+
+  parser_log = subparsers.add_parser('ldb')
+  parser_log.add_argument(
+      'structure_type', choices=[
+          'blocks',
+          'records'])
+  parser_log.set_defaults(func=LdbCommand)
+
+  parser_log = subparsers.add_parser('indexeddb')
+  parser_log.set_defaults(func=IndexeddbCommand)
+
+  args = parser.parse_args()
+
+  args.func(args)

dfindexeddb/indexeddb/chromium.py

@@ -21,7 +21,10 @@ from typing import Any, BinaryIO, Optional, Tuple, Type, TypeVar, Union
 
 from dfindexeddb import errors
 from dfindexeddb import utils
+from dfindexeddb.indexeddb import blink
 from dfindexeddb.indexeddb import definitions
+from dfindexeddb.leveldb import ldb
+from dfindexeddb.leveldb import log
 
 
 T = TypeVar('T')
@@ -401,7 +404,7 @@ class BaseIndexedDBKey:
       The decoded key.
     """
     decoder = utils.LevelDBDecoder(stream)
-    key_prefix = KeyPrefix.FromDecoder(decoder)
+    key_prefix = KeyPrefix.FromDecoder(decoder, base_offset=base_offset)
     return cls.FromDecoder(
         decoder=decoder, key_prefix=key_prefix, base_offset=base_offset)
 
@@ -958,6 +961,23 @@ class ObjectStoreMetaDataKey(BaseIndexedDBKey):
         offset=base_offset + offset, key_prefix=key_prefix,
         object_store_id=object_store_id, metadata_type=metadata_type)
 
+@dataclass
+class ObjectStoreDataValue:
+  """The parsed values from an ObjectStoreDataKey.
+
+  Attributes:
+    unknown: an unknown integer (possibly a sequence number?).
+    is_wrapped: True if the value was wrapped.
+    blob_size: the blob size, only valid if wrapped.
+    blob_offset: the blob offset, only valid if wrapped.
+    value: the blink serialized value, only valid if not wrapped.
+  """
+  unkown: int
+  is_wrapped: bool
+  blob_size: Optional[int]
+  blob_offset: Optional[int]
+  value: Any
+
 
 @dataclass
 class ObjectStoreDataKey(BaseIndexedDBKey):
@@ -969,11 +989,33 @@ class ObjectStoreDataKey(BaseIndexedDBKey):
   encoded_user_key: IDBKey
 
   def DecodeValue(
-      self, decoder: utils.LevelDBDecoder) -> Tuple[int, bytes]:
+      self, decoder: utils.LevelDBDecoder) -> ObjectStoreDataValue:
     """Decodes the object store data value."""
-    _, version = decoder.DecodeVarint()
-    _, encoded_string = decoder.ReadBytes()
-    return version, encoded_string
+    _, unknown_integer = decoder.DecodeVarint()
+
+    _, wrapped_header_bytes = decoder.PeekBytes(3)
+    if len(wrapped_header_bytes) != 3:
+      raise errors.DecoderError('Insufficient bytes')
+
+    if (wrapped_header_bytes[0] == definitions.BlinkSerializationTag.VERSION and
+        wrapped_header_bytes[1] == 0x11 and
+        wrapped_header_bytes[2] == 0x01):
+      _, blob_size = decoder.DecodeVarint()
+      _, blob_offset = decoder.DecodeVarint()
+      return ObjectStoreDataValue(
+          unkown=unknown_integer,
+          is_wrapped=True,
+          blob_size=blob_size,
+          blob_offset=blob_offset,
+          value=None)
+    _, blink_bytes = decoder.ReadBytes()
+    blink_value = blink.V8ScriptValueDecoder.FromBytes(blink_bytes)
+    return ObjectStoreDataValue(
+        unkown=unknown_integer,
+        is_wrapped=False,
+        blob_size=None,
+        blob_offset=None,
+        value=blink_value)
 
   @classmethod
   def FromDecoder(
@@ -985,7 +1027,7 @@ class ObjectStoreDataKey(BaseIndexedDBKey):
         definitions.KeyPrefixType.OBJECT_STORE_DATA):
       raise errors.ParserError('Invalid KeyPrefix for ObjectStoreDataKey')
     offset = decoder.stream.tell()
-    encoded_user_key = IDBKey.FromDecoder(decoder, base_offset)
+    encoded_user_key = IDBKey.FromDecoder(decoder, offset)
     return cls(
         offset=base_offset + offset,
         key_prefix=key_prefix, encoded_user_key=encoded_user_key)
@@ -1012,7 +1054,7 @@ class ExistsEntryKey(BaseIndexedDBKey):
   ) -> ExistsEntryKey:
     """Decodes the exists entry key."""
     offset = decoder.stream.tell()
-    encoded_user_key = IDBKey.FromDecoder(decoder, base_offset)
+    encoded_user_key = IDBKey.FromDecoder(decoder, offset)
 
     return cls(
         offset=base_offset + offset,
@@ -1043,7 +1085,7 @@ class IndexDataKey(BaseIndexedDBKey):
                  base_offset: int = 0) -> IndexDataKey:
     """Decodes the index data key."""
     offset = decoder.stream.tell()
-    encoded_user_key = IDBKey.FromDecoder(decoder, base_offset)
+    encoded_user_key = IDBKey.FromDecoder(decoder, offset)
 
     if decoder.NumRemainingBytes() > 0:
       _, sequence_number = decoder.DecodeVarint()
@@ -1051,7 +1093,9 @@ class IndexDataKey(BaseIndexedDBKey):
       sequence_number = None
 
     if decoder.NumRemainingBytes() > 0:
-      encoded_primary_key = IDBKey.FromDecoder(decoder, base_offset)
+      encoded_primary_key_offset = decoder.stream.tell()
+      encoded_primary_key = IDBKey.FromDecoder(
+          decoder, encoded_primary_key_offset)
     else:
       encoded_primary_key = None
 
@@ -1084,7 +1128,7 @@ class BlobEntryKey(BaseIndexedDBKey):
   ) -> BlobEntryKey:
     """Decodes the blob entry key."""
     offset = decoder.stream.tell()
-    user_key = IDBKey.FromDecoder(decoder, base_offset)
+    user_key = IDBKey.FromDecoder(decoder, offset)
 
     return cls(key_prefix=key_prefix, user_key=user_key,
                offset=base_offset + offset)
@@ -1281,3 +1325,36 @@ class ObjectStore:
   id: int
   name: str
   records: list = field(default_factory=list, repr=False)
+
+
+@dataclass
+class IndexedDBRecord:
+  """An IndexedDB Record.
+
+  Attributes:
+    offset: the offset of the record.
+    key: the key of the record.
+    value: the value of the record.
+    sequence_number: if available, the sequence number of the record.
+    type: the type of the record.
+  """
+  offset: int
+  key: Any
+  value: Any
+  sequence_number: int
+  type: int
+
+  @classmethod
+  def FromLevelDBRecord(
+      cls, record: Union[ldb.LdbKeyValueRecord, log.ParsedInternalKey]
+  ) -> IndexedDBRecord:
+    """Returns an IndexedDBRecord from a ParsedInternalKey."""
+    idb_key = IndexedDbKey.FromBytes(record.key, base_offset=record.offset)
+    idb_value = idb_key.ParseValue(record.value)
+    return cls(
+      offset=record.offset,
+      key=idb_key,
+      value=idb_value,
+      sequence_number=record.sequence_number if hasattr(
+          record, 'sequence_number') else None,
+      type=record.type)

dfindexeddb/version.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 """Version information for dfIndexeddb."""
 
-__version__ = "20240225"
+__version__ = "20240301"
 
 
 def GetVersion():

{dfindexeddb-20240225.dist-info → dfindexeddb-20240301.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dfindexeddb
-Version: 20240225
+Version: 20240301
 Summary: dfindexeddb is an experimental Python tool for performing digital forensic analysis of IndexedDB and leveldb files.
 Author-email: Syd Pleno <sydp@google.com>
 Maintainer-email: dfIndexeddb Developers <dfindexeddb-dev@googlegroups.com>
@@ -236,6 +236,12 @@ include:
 * emails and contact information from an e-mail application,
 * images and metadata from a photo gallery application
 
+## Installation
+
+```
+$ pip install dfindexeddb
+```
+
 ## Installation from source
 
 ### Linux
@@ -256,23 +262,57 @@ include:
     $ pip install .
 ```
 
-## Tools
+## Usage
+
+A CLI tool is available after installation:
 
-This repository contains a number of scripts which demonstrate how one can use
-this library.  To run these tools, please install the `click` python package.
+```
+$ dfindexeddb -h
+usage: dfindexeddb [-h] -s SOURCE [--json] {log,ldb,indexeddb} ...
 
-* `tools/indexeddb_dump.py` - parses structures from an IndexedDB and prints
-them to standard output.
-  - Optionally, you can also install the `leveldb` python package if you
-  would prefer to use a native leveldb library instead of the leveldb parser in
-  this repository.
-* `tools/ldb_dump.py` - parses structures from a LevelDB .ldb file and prints
-them to standard output.
-* `tools/log_dump.py` - parses structures from a LevelDB .log file and prints
-them to standard output.
+A cli tool for the dfindexeddb package
 
+positional arguments:
+  {log,ldb,indexeddb}
+
+options:
+  -s SOURCE, --source SOURCE
+                        The source leveldb file
+  --json                Output as JSON
+```
+
+To parse a LevelDB .log file:
+
+```
+$ dfindexeddb -s <SOURCE> log -h
+usage: dfindexeddb log [-h] {blocks,physical_records,write_batches,parsed_internal_key,records}
 
+positional arguments:
+  {blocks,physical_records,write_batches,parsed_internal_key,records}
 
+options:
+  -h, --help            show this help message and exit
 ```
-    $ pip install click leveldb
+
+To parse a LevelDB .ldb file:
+
+```
+$ dfindexeddb -s <SOURCE> ldb -h
+usage: dfindexeddb ldb [-h] {blocks,records}
+
+positional arguments:
+  {blocks,records}
+
+options:
+  -h, --help        show this help message and exit
+```
+
+To parse a LevelDB .ldb or .log file as IndexedDB:
+
+```
+$ dfindexeddb -s <SOURCE> indexeddb -h
+usage: dfindexeddb indexeddb [-h]
+
+options:
+  -h, --help  show this help message and exit
 ```

{dfindexeddb-20240225.dist-info → dfindexeddb-20240301.dist-info}/RECORD

@@ -1,18 +1,20 @@
 dfindexeddb/__init__.py,sha256=KPYL9__l8od6_OyDfGRTgaJ6iy_fqIgZ-dS2S-e3Rac,599
+dfindexeddb/cli.py,sha256=QsFPC_B_PW6E42kBPUO0Ny3oRspYJsXlDCQDHmKuDHQ,4866
 dfindexeddb/errors.py,sha256=PNpwyf_lrPc4TE77oAakX3mu5D_YcP3f80wq8Y1LkvY,749
 dfindexeddb/utils.py,sha256=g9iiGRX4DB1wFBSBHa6b9lg7JzAdE0SN0DrdB2aS_Co,10091
-dfindexeddb/version.py,sha256=PiFacIFljjKL8Q1cSjdiapC9lqsgpPS4GDZDrPlBy4o,750
+dfindexeddb/version.py,sha256=d2fDd3V2U_CkGi3PawWMR77qU0DzHEC3_bk7Ywh3xlM,750
 dfindexeddb/indexeddb/__init__.py,sha256=kExXSVBCTKCD5BZJkdMfUMqGksH-DMJxP2_lI0gq-BE,575
 dfindexeddb/indexeddb/blink.py,sha256=MblpYfv-ByG7n_fjYKu2EUhpfVJdUveoW4oSAg5T4tY,3534
-dfindexeddb/indexeddb/chromium.py,sha256=N1aCoJETNqLER8T_C4bmfrxiNr1csJhUJ4-14qrl0nc,42291
+dfindexeddb/indexeddb/chromium.py,sha256=oHHyGuy7BanRWRoJu4zqXo5QvxO2j_qLk2KMHBBwZvs,44692
 dfindexeddb/indexeddb/definitions.py,sha256=yline3y3gmZx6s-dwjpPDNs5HO4zT6KZqPWQfEsHDoM,7413
 dfindexeddb/indexeddb/v8.py,sha256=ldqpc9T1kG7BOdjnHjQ5hNO9OCXZ3_Zd6vRSpC-NrEA,21893
 dfindexeddb/leveldb/__init__.py,sha256=KPYL9__l8od6_OyDfGRTgaJ6iy_fqIgZ-dS2S-e3Rac,599
 dfindexeddb/leveldb/ldb.py,sha256=uShhXjQe4Sz3dn54IXbGxRtE6D8RNpu1NDy5Zb0P9LA,7927
 dfindexeddb/leveldb/log.py,sha256=cyMfjDz5a6gfGb5NonxC1Y72OmHYBWzYK8UMVzP_umw,8532
-dfindexeddb-20240225.dist-info/AUTHORS,sha256=QbvjbAom57fpEkekkCVFUj0B9KUMGraR510aUMBC-PE,286
-dfindexeddb-20240225.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-dfindexeddb-20240225.dist-info/METADATA,sha256=ZRr-aOFy5SlgZ2nTgeaYpLCjauP7qzW5XSSsFCyeFOQ,15474
-dfindexeddb-20240225.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-dfindexeddb-20240225.dist-info/top_level.txt,sha256=X9OTaub1c8S_JJ7g-f8JdkhhdiZ4x1j4eni1hdUCwE4,12
-dfindexeddb-20240225.dist-info/RECORD,,
+dfindexeddb-20240301.dist-info/AUTHORS,sha256=QbvjbAom57fpEkekkCVFUj0B9KUMGraR510aUMBC-PE,286
+dfindexeddb-20240301.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+dfindexeddb-20240301.dist-info/METADATA,sha256=AYggbgjQv3dNPwO-f3e7tK3EPvKsnlNTKM2nPTbAQYg,15933
+dfindexeddb-20240301.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+dfindexeddb-20240301.dist-info/entry_points.txt,sha256=UsfPLLhTiVAAtZ8Rq3ZR7JNFGMuHqJy-tugGWonQWtc,52
+dfindexeddb-20240301.dist-info/top_level.txt,sha256=X9OTaub1c8S_JJ7g-f8JdkhhdiZ4x1j4eni1hdUCwE4,12
+dfindexeddb-20240301.dist-info/RECORD,,

dfindexeddb-20240301.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+dfindexeddb = dfindexeddb.cli:App