dfindexeddb 20240301__py3-none-any.whl → 20240324__py3-none-any.whl

dfindexeddb/indexeddb/blink.py

@@ -86,7 +86,8 @@ class V8ScriptValueDecoder:
       NotImplementedError: when called.
     """
     tag = self.ReadTag()
-    raise NotImplementedError(f'V8ScriptValueDecoder.ReadHostObject - {tag}')
+    raise NotImplementedError(
+        f'V8ScriptValueDecoder.ReadHostObject - {tag.name}')
 
   def Deserialize(self) -> Any:
     """Deserializes a Blink SSV.

dfindexeddb/indexeddb/chromium.py

@@ -20,18 +20,18 @@ import io
 from typing import Any, BinaryIO, Optional, Tuple, Type, TypeVar, Union
 
 from dfindexeddb import errors
-from dfindexeddb import utils
 from dfindexeddb.indexeddb import blink
 from dfindexeddb.indexeddb import definitions
 from dfindexeddb.leveldb import ldb
 from dfindexeddb.leveldb import log
+from dfindexeddb.leveldb import utils
 
 
 T = TypeVar('T')
 
 
 @dataclass
-class KeyPrefix(utils.FromStreamMixin):
+class KeyPrefix(utils.FromDecoderMixin):
   """The IndexedDB key prefix.
 
   Attributes:
@@ -111,7 +111,7 @@ class KeyPrefix(utils.FromStreamMixin):
 
 
 @dataclass
-class IDBKey(utils.FromStreamMixin):
+class IDBKey(utils.FromDecoderMixin):
   """An IDBKey.
 
   Attributes:
@@ -199,7 +199,7 @@ class IDBKey(utils.FromStreamMixin):
 
 
 @dataclass
-class IDBKeyPath(utils.FromStreamMixin):
+class IDBKeyPath(utils.FromDecoderMixin):
   """An IDBKeyPath.
 
   Arguments:
@@ -256,7 +256,7 @@ class IDBKeyPath(utils.FromStreamMixin):
 
 
 @dataclass
-class BlobJournalEntry(utils.FromStreamMixin):
+class BlobJournalEntry(utils.FromDecoderMixin):
   """A blob journal entry.
 
   Attributes:
@@ -287,7 +287,7 @@ class BlobJournalEntry(utils.FromStreamMixin):
 
 
 @dataclass
-class BlobJournal(utils.FromStreamMixin):
+class BlobJournal(utils.FromDecoderMixin):
   """A BlobJournal.
 
   Attributes:
@@ -570,7 +570,7 @@ class EarlistCompactionTimeKey(BaseIndexedDBKey):
 class ScopesPrefixKey(BaseIndexedDBKey):
   """A scopes prefix IndexedDB key."""
 
-  def DecodeValue(self, decoder: utils.StreamDecoder) -> Optional[bytes]:
+  def DecodeValue(self, decoder: utils.LevelDBDecoder) -> Optional[bytes]:
     """Decodes the scopes prefix value."""
     if decoder.NumRemainingBytes:
       return decoder.ReadBytes()[1]
@@ -578,7 +578,7 @@ class ScopesPrefixKey(BaseIndexedDBKey):
 
   @classmethod
   def FromDecoder(
-      cls, decoder: utils.StreamDecoder, key_prefix: KeyPrefix,
+      cls, decoder: utils.LevelDBDecoder, key_prefix: KeyPrefix,
       base_offset: int = 0
   ) -> ScopesPrefixKey:
     """Decodes the scopes prefix key."""
@@ -1226,7 +1226,7 @@ class IndexMetaDataKey(BaseIndexedDBKey):
 
 
 @dataclass
-class ExternalObjectEntry(utils.FromStreamMixin):
+class ExternalObjectEntry(utils.FromDecoderMixin):
   """An IndexedDB external object entry.
 
   Args:
@@ -1287,7 +1287,7 @@ class ExternalObjectEntry(utils.FromStreamMixin):
 
 
 @dataclass
-class IndexedDBExternalObject(utils.FromStreamMixin):
+class IndexedDBExternalObject(utils.FromDecoderMixin):
   """An IndexedDB external object.
 
   Args:
@@ -1346,7 +1346,7 @@ class IndexedDBRecord:
 
   @classmethod
   def FromLevelDBRecord(
-      cls, record: Union[ldb.LdbKeyValueRecord, log.ParsedInternalKey]
+      cls, record: Union[ldb.KeyValueRecord, log.ParsedInternalKey]
   ) -> IndexedDBRecord:
     """Returns an IndexedDBRecord from a ParsedInternalKey."""
     idb_key = IndexedDbKey.FromBytes(record.key, base_offset=record.offset)
@@ -1357,4 +1357,4 @@ class IndexedDBRecord:
       value=idb_value,
       sequence_number=record.sequence_number if hasattr(
           record, 'sequence_number') else None,
-      type=record.type)
+      type=record.record_type)

dfindexeddb/indexeddb/cli.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A CLI tool for dfindexeddb."""
+import argparse
+import dataclasses
+from datetime import datetime
+import json
+import pathlib
+import sys
+import traceback
+
+from dfindexeddb import errors
+from dfindexeddb import version
+from dfindexeddb.leveldb import record as leveldb_record
+from dfindexeddb.indexeddb import chromium
+from dfindexeddb.indexeddb import v8
+
+
+_VALID_PRINTABLE_CHARACTERS = (
+    ' abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' +
+    '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~.')
+
+
+class Encoder(json.JSONEncoder):
+  """A JSON encoder class for dfindexeddb fields."""
+  def default(self, o):
+    if dataclasses.is_dataclass(o):
+      o_dict = dataclasses.asdict(o)
+      return o_dict
+    if isinstance(o, bytes):
+      out = []
+      for x in o:
+        if chr(x) not in _VALID_PRINTABLE_CHARACTERS:
+          out.append(f'\\x{x:02X}')
+        else:
+          out.append(chr(x))
+      return ''.join(out)
+    if isinstance(o, datetime):
+      return o.isoformat()
+    if isinstance(o, v8.Undefined):
+      return "<undefined>"
+    if isinstance(o, v8.Null):
+      return "<null>"
+    if isinstance(o, set):
+      return list(o)
+    if isinstance(o, v8.RegExp):
+      return str(o)
+    return json.JSONEncoder.default(self, o)
+
+
+def _Output(structure, to_json=False):
+  """Helper method to output parsed structure to stdout."""
+  if to_json:
+    print(json.dumps(structure, indent=2, cls=Encoder))
+  else:
+    print(structure)
+
+
+def IndexeddbCommand(args):
+  """The CLI for processing a log/ldb file as indexeddb."""
+  for db_record in leveldb_record.LevelDBRecord.FromDir(args.source):
+    record = db_record.record
+    try:
+      db_record.record = chromium.IndexedDBRecord.FromLevelDBRecord(record)
+    except(
+        errors.ParserError,
+        errors.DecoderError,
+        NotImplementedError) as err:
+      print(
+          (f'Error parsing blink value: {err} for {record.__class__.__name__} '
+           f'at offset {record.offset} in {db_record.path}'), file=sys.stderr)
+      print(f'Traceback: {traceback.format_exc()}', file=sys.stderr)
+    _Output(db_record, to_json=args.json)
+
+
+def App():
+  """The CLI app entrypoint for dfindexeddb."""
+  parser = argparse.ArgumentParser(
+      prog='dfindexeddb',
+      description='A cli tool for parsing indexeddb files',
+      epilog=f'Version {version.GetVersion()}')
+  parser.add_argument(
+      '-s', '--source', required=True, type=pathlib.Path,
+      help='The source leveldb folder')
+  parser.add_argument('--json', action='store_true', help='Output as JSON')
+  parser.set_defaults(func=IndexeddbCommand)
+
+  args = parser.parse_args()
+  args.func(args)

dfindexeddb/leveldb/cli.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A CLI tool for leveldb files."""
+import argparse
+import dataclasses
+from datetime import datetime
+import json
+import pathlib
+
+from dfindexeddb import version
+from dfindexeddb.leveldb import descriptor
+from dfindexeddb.leveldb import ldb
+from dfindexeddb.leveldb import log
+from dfindexeddb.leveldb import record
+
+
+_VALID_PRINTABLE_CHARACTERS = (
+    ' abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' +
+    '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~.')
+
+
+class Encoder(json.JSONEncoder):
+  """A JSON encoder class for dfleveldb fields."""
+
+  def default(self, o):
+    """Returns a serializable object for o."""
+    if dataclasses.is_dataclass(o):
+      o_dict = dataclasses.asdict(o)
+      return o_dict
+    if isinstance(o, bytes):
+      out = []
+      for x in o:
+        if chr(x) not in _VALID_PRINTABLE_CHARACTERS:
+          out.append(f'\\x{x:02X}')
+        else:
+          out.append(chr(x))
+      return ''.join(out)
+    if isinstance(o, datetime):
+      return o.isoformat()
+    if isinstance(o, set):
+      return list(o)
+    return json.JSONEncoder.default(self, o)
+
+
+def _Output(structure, to_json=False):
+  """Helper method to output parsed structure to stdout."""
+  if to_json:
+    print(json.dumps(structure, indent=2, cls=Encoder))
+  else:
+    print(structure)
+
+
+def DbCommand(args):
+  """The CLI for processing leveldb folders."""
+  for rec in record.LevelDBRecord.FromDir(args.source):
+    _Output(rec, to_json=args.json)
+
+
+def LdbCommand(args):
+  """The CLI for processing ldb files."""
+  ldb_file = ldb.FileReader(args.source)
+
+  if args.structure_type == 'blocks':
+    # Prints block information.
+    for block in ldb_file.GetBlocks():
+      _Output(block, to_json=args.json)
+
+  elif args.structure_type == 'records' or not args.structure_type:
+    # Prints key value record information.
+    for key_value_record in ldb_file.GetKeyValueRecords():
+      _Output(key_value_record, to_json=args.json)
+
+  else:
+    print(f'{args.structure_type} is not supported for ldb files.')
+
+
+def LogCommand(args):
+  """The CLI for processing log files."""
+  log_file = log.FileReader(args.source)
+
+  if args.structure_type == 'blocks':
+    # Prints block information.
+    for block in log_file.GetBlocks():
+      _Output(block, to_json=args.json)
+
+  elif args.structure_type == 'physical_records':
+    # Prints log file physical record information.
+    for log_file_record in log_file.GetPhysicalRecords():
+      _Output(log_file_record, to_json=args.json)
+
+  elif args.structure_type == 'write_batches':
+    # Prints log file batch information.
+    for batch in log_file.GetWriteBatches():
+      _Output(batch, to_json=args.json)
+
+  elif (args.structure_type in ('parsed_internal_key', 'records')
+        or not args.structure_type):
+    # Prints key value record information.
+    for internal_key_record in log_file.GetParsedInternalKeys():
+      _Output(internal_key_record, to_json=args.json)
+
+  else:
+    print(f'{args.structure_type} is not supported for log files.')
+
+
+def DescriptorCommand(args):
+  """The CLI for processing descriptor (MANIFEST) files."""
+  manifest_file = descriptor.FileReader(args.source)
+
+  if args.structure_type == 'blocks':
+    # Prints block information.
+    for block in manifest_file.GetBlocks():
+      _Output(block, to_json=args.json)
+
+  elif args.structure_type == 'physical_records':
+    # Prints log file physical record information.
+    for log_file_record in manifest_file.GetPhysicalRecords():
+      _Output(log_file_record, to_json=args.json)
+
+  elif (args.structure_type == 'versionedit'
+        or not args.structure_type):
+    for version_edit in manifest_file.GetVersionEdits():
+      _Output(version_edit, to_json=args.json)
+
+  else:
+    print(f'{args.structure_type} is not supported for descriptor files.')
+
+def App():
+  """The CLI app entrypoint for parsing leveldb files."""
+  parser = argparse.ArgumentParser(
+      prog='dfleveldb',
+      description='A cli tool for parsing leveldb files',
+      epilog=f'Version {version.GetVersion()}')
+
+  subparsers = parser.add_subparsers()
+
+  parser_db = subparsers.add_parser(
+      'db', help='Parse a directory as leveldb.')
+  parser_db.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source leveldb directory')
+  parser_db.add_argument(
+      '--json', action='store_true', help='Output as JSON')
+  parser_db.set_defaults(func=DbCommand)
+
+  parser_log = subparsers.add_parser(
+      'log', help='Parse a leveldb log file.')
+  parser_log.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source leveldb file')
+  parser_log.add_argument(
+      '--json', action='store_true', help='Output as JSON')
+  parser_log.add_argument(
+      '-t',
+      '--structure_type',
+      choices=[
+          'blocks',
+          'physical_records',
+          'write_batches',
+          'parsed_internal_key'])
+  parser_log.set_defaults(func=LogCommand)
+
+  parser_ldb = subparsers.add_parser(
+      'ldb', help='Parse a leveldb table (.ldb) file.')
+  parser_ldb.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source leveldb file')
+  parser_ldb.add_argument(
+      '--json', action='store_true', help='Output as JSON')
+  parser_ldb.add_argument(
+      '-t',
+      '--structure_type',
+      choices=[
+          'blocks',
+          'records'])
+  parser_ldb.set_defaults(func=LdbCommand)
+
+  parser_descriptor = subparsers.add_parser(
+      'descriptor', help='Parse a leveldb descriptor (MANIFEST) file.')
+  parser_descriptor.add_argument(
+      '-s', '--source',
+      required=True,
+      type=pathlib.Path,
+      help='The source leveldb file')
+  parser_descriptor.add_argument(
+      '--json', action='store_true', help='Output as JSON')
+  parser_descriptor.add_argument(
+      '-t',
+      '--structure_type',
+      choices=[
+          'blocks', 'physical_records', 'versionedit'])
+  parser_descriptor.set_defaults(func=DescriptorCommand)
+
+  args = parser.parse_args()
+
+  if not hasattr(args, 'func'):
+    parser.print_usage()
+  else:
+    args.func(args)

dfindexeddb/leveldb/definitions.py

@@ -0,0 +1,59 @@
+# -*- coding: utf-8 -*-
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Definitions for LevelDB."""
+
+import enum
+
+BLOCK_RESTART_ENTRY_LENGTH = 4
+BLOCK_TRAILER_SIZE = 5
+TABLE_FOOTER_SIZE = 48
+TABLE_MAGIC = b'\x57\xfb\x80\x8b\x24\x75\x47\xdb'
+
+PACKED_SEQUENCE_AND_TYPE_LENGTH = 8
+SEQUENCE_LENGTH = 7
+TYPE_LENGTH = 1
+
+
+class BlockCompressionType(enum.IntEnum):
+  """Block compression types."""
+  SNAPPY = 1
+  ZSTD = 2
+
+
+class VersionEditTags(enum.IntEnum):
+  """VersionEdit tags."""
+  COMPARATOR = 1
+  LOG_NUMBER = 2
+  NEXT_FILE_NUMBER = 3
+  LAST_SEQUENCE = 4
+  COMPACT_POINTER = 5
+  DELETED_FILE = 6
+  NEW_FILE = 7
+  # 8 was used for large value refs
+  PREV_LOG_NUMBER = 9
+
+
+class LogFilePhysicalRecordType(enum.IntEnum):
+  """Log file physical record types."""
+  FULL = 1
+  FIRST = 2
+  MIDDLE = 3
+  LAST = 4
+
+
+class InternalRecordType(enum.IntEnum):
+  """Internal record types."""
+  DELETED = 0
+  VALUE = 1